jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add enum for Airbyte Json Schemas (#543)

Browse Source
This commit is contained in:
Michel Tricot
2020-10-12 10:07:18 -07:00
committed by GitHub
parent a07b8edfd9
commit 364eb1eafc
8 changed files with 185 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java
View File

@@ -26,6 +26,13 @@ package io.airbyte.commons.json;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.airbyte.commons.io.IOs;
import io.airbyte.commons.resources.MoreResources;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
public class JsonSchemas {
@@ -43,4 +50,31 @@ public class JsonSchemas {
}
}
/*
* JsonReferenceProcessor relies on all of the json in consumes being in a file system (not in a
* jar). This method copies all of the json configs out of the jar into a temporary directory so
* that JsonReferenceProcessor can find them.
*/
@SuppressWarnings("UnstableApiUsage")
public static <T> Path prepareSchemas(final String resourceDir, Class<T> klass) {
try {
final List<String> filenames = MoreResources.listResources(klass, resourceDir)
.map(p -> p.getFileName().toString())
.filter(p -> p.endsWith(".yaml"))
.collect(Collectors.toList());
final Path configRoot = Files.createTempDirectory("schemas");
for (String filename : filenames) {
IOs.writeFile(
configRoot,
filename,
MoreResources.readResource(String.format("%s/%s", resourceDir, filename)));
}
return configRoot;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

37
airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java
View File

@@ -24,14 +24,9 @@
package io.airbyte.config;
import io.airbyte.commons.io.IOs;
import io.airbyte.commons.resources.MoreResources;
import io.airbyte.commons.json.JsonSchemas;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
public enum ConfigSchema {
@@ -56,35 +51,7 @@ public enum ConfigSchema {
STATE("State.yaml");
static final Path KNOWN_SCHEMAS_ROOT = prepareSchemas();
private static final String RESOURCE_DIR = "types";
/*
* JsonReferenceProcessor relies on all of the json in consumes being in a file system (not in a
* jar). This method copies all of the json configs out of the jar into a temporary directory so
* that JsonReferenceProcessor can find them.
*/
@SuppressWarnings("UnstableApiUsage")
private static Path prepareSchemas() {
try {
final List<String> filenames = MoreResources.listResources(ConfigSchema.class, RESOURCE_DIR)
.map(p -> p.getFileName().toString())
.filter(p -> p.endsWith(".yaml"))
.collect(Collectors.toList());
final Path configRoot = Files.createTempDirectory("schemas");
for (String filename : filenames) {
IOs.writeFile(
configRoot,
filename,
MoreResources.readResource(String.format("%s/%s", RESOURCE_DIR, filename)));
}
return configRoot;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
static final Path KNOWN_SCHEMAS_ROOT = JsonSchemas.prepareSchemas("types", ConfigSchema.class);
private final String schemaFilename;

48
airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/AirbyteProtocolSchema.java Normal file
View File

@@ -0,0 +1,48 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package io.airbyte.protocol.models;
import io.airbyte.commons.json.JsonSchemas;
import java.io.File;
import java.nio.file.Path;
public enum AirbyteProtocolSchema {
MESSAGE("airbyte_message.yaml"),
CATALOG("airbyte_catalog.yaml");
static final Path KNOWN_SCHEMAS_ROOT = JsonSchemas.prepareSchemas("airbyte_protocol", AirbyteProtocolSchema.class);
private final String schemaFilename;
AirbyteProtocolSchema(final String schemaFilename) {
this.schemaFilename = schemaFilename;
}
public File getFile() {
return KNOWN_SCHEMAS_ROOT.resolve(schemaFilename).toFile();
}
}

49
airbyte-protocol/models/src/test/java/io/airbyte/protocol/models/AirbyteProtocolSchemaTest.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package io.airbyte.protocol.models;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import org.junit.jupiter.api.Test;
class AirbyteProtocolSchemaTest {
@Test
void testFile() throws IOException {
final String schema = Files.readString(AirbyteProtocolSchema.MESSAGE.getFile().toPath(), StandardCharsets.UTF_8);
assertTrue(schema.contains("title"));
}
@Test
void testPrepareKnownSchemas() {
for (AirbyteProtocolSchema value : AirbyteProtocolSchema.values()) {
assertTrue(Files.exists(value.getFile().toPath()));
}
}
}

82
airbyte-singer/src/main/java/io/airbyte/singer/SingerConfigSchema.java
View File

@@ -1,82 +0,0 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package io.airbyte.singer;
import io.airbyte.commons.io.IOs;
import io.airbyte.commons.resources.MoreResources;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
// todo (cgardens) - dedupe this with ConfigSchema.
public enum SingerConfigSchema {
SINGER_MESSAGE("SingerMessage.yaml");
static final Path KNOWN_SCHEMAS_ROOT = prepareSchemas();
private static final String RESOURCE_DIR = "singer_types";
/*
* JsonReferenceProcessor relies on all of the json in consumes being in a file system (not in a
* jar). This method copies all of the json configs out of the jar into a temporary directory so
* that JsonReferenceProcessor can find them.
*/
@SuppressWarnings("UnstableApiUsage")
private static Path prepareSchemas() {
try {
final List<String> filenames = MoreResources.listResources(SingerConfigSchema.class, RESOURCE_DIR)
.map(p -> p.getFileName().toString())
.peek(x -> System.out.println("x = " + x))
.filter(p -> p.endsWith(".yaml"))
.collect(Collectors.toList());
final Path configRoot = Files.createTempDirectory("schemas");
for (String filename : filenames) {
IOs.writeFile(
configRoot,
filename,
MoreResources.readResource(String.format("%s/%s", RESOURCE_DIR, filename)));
}
return configRoot;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private final String schemaFilename;
SingerConfigSchema(final String schemaFilename) {
this.schemaFilename = schemaFilename;
}
public File getFile() {
return KNOWN_SCHEMAS_ROOT.resolve(schemaFilename).toFile();
}
}

47
airbyte-singer/src/main/java/io/airbyte/singer/SingerProtocolSchema.java Normal file
View File

@@ -0,0 +1,47 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package io.airbyte.singer;
import io.airbyte.commons.json.JsonSchemas;
import java.io.File;
import java.nio.file.Path;
public enum SingerProtocolSchema {
SINGER_MESSAGE("SingerMessage.yaml");
static final Path KNOWN_SCHEMAS_ROOT = JsonSchemas.prepareSchemas("singer_types", SingerProtocolSchema.class);
private final String schemaFilename;
SingerProtocolSchema(final String schemaFilename) {
this.schemaFilename = schemaFilename;
}
public File getFile() {
return KNOWN_SCHEMAS_ROOT.resolve(schemaFilename).toFile();
}
}

6
airbyte-singer/src/test/java/io/airbyte/singer/SingerConfigSchemaTest.java → airbyte-singer/src/test/java/io/airbyte/singer/SingerProtocolSchemaTest.java
View File

@@ -31,17 +31,17 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import org.junit.jupiter.api.Test;
public class SingerConfigSchemaTest {
public class SingerProtocolSchemaTest {
@Test
void testFile() throws IOException {
final String schema = Files.readString(SingerConfigSchema.SINGER_MESSAGE.getFile().toPath(), StandardCharsets.UTF_8);
final String schema = Files.readString(SingerProtocolSchema.SINGER_MESSAGE.getFile().toPath(), StandardCharsets.UTF_8);
assertTrue(schema.contains("title"));
}
@Test
void testPrepareKnownSchemas() {
for (SingerConfigSchema value : SingerConfigSchema.values()) {
for (SingerProtocolSchema value : SingerProtocolSchema.values()) {
assertTrue(Files.exists(value.getFile().toPath()));
}
}

4
airbyte-workers/src/main/java/io/airbyte/workers/protocols/singer/SingerProtocolPredicate.java
View File

@@ -26,7 +26,7 @@ package io.airbyte.workers.protocols.singer;
import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.commons.json.JsonSchemaValidator;
import io.airbyte.singer.SingerConfigSchema;
import io.airbyte.singer.SingerProtocolSchema;
import java.util.function.Predicate;
public class SingerProtocolPredicate implements Predicate<JsonNode> {
@@ -36,7 +36,7 @@ public class SingerProtocolPredicate implements Predicate<JsonNode> {
public SingerProtocolPredicate() {
jsonSchemaValidator = new JsonSchemaValidator();
schema = JsonSchemaValidator.getSchema(SingerConfigSchema.SINGER_MESSAGE.getFile());
schema = JsonSchemaValidator.getSchema(SingerProtocolSchema.SINGER_MESSAGE.getFile());
}
@Override