From 908f3949d206ecba249a27faef582877dfc19a85 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Thu, 13 Oct 2022 10:28:40 -0400 Subject: [PATCH] remove usages of YamlSeedConfigPersistence (#17895) --- .../java/io/airbyte/bootloader/BootloaderApp.java | 3 +-- .../io/airbyte/bootloader/BootloaderAppTest.java | 9 ++++++--- airbyte-config/config-persistence/readme.md | 2 +- .../config/init/LocalDefinitionsProvider.java | 2 ++ .../java/io/airbyte/config/init/SpecFormatTest.java | 12 +++--------- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java index 1b875c358e6..8c508fe5b15 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java @@ -17,7 +17,6 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.init.ApplyDefinitionsHelper; import io.airbyte.config.init.DefinitionsProvider; import io.airbyte.config.init.LocalDefinitionsProvider; -import io.airbyte.config.init.YamlSeedConfigPersistence; import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.DatabaseConfigPersistence; @@ -206,7 +205,7 @@ public class BootloaderApp { } private static DefinitionsProvider getLocalDefinitionsProvider() throws IOException { - return new LocalDefinitionsProvider(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); + return new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); } private static Database getJobDatabase(final DSLContext dslContext) throws IOException { diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java index 976e3d51853..cf2630467ca 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java @@ -24,7 +24,9 @@ import io.airbyte.config.Configs; import io.airbyte.config.Geography; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardWorkspace; -import io.airbyte.config.init.YamlSeedConfigPersistence; +import io.airbyte.config.init.DefinitionProviderToConfigPersistenceAdapter; +import io.airbyte.config.init.DefinitionsProvider; +import io.airbyte.config.init.LocalDefinitionsProvider; import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.DatabaseConfigPersistence; @@ -199,9 +201,10 @@ class BootloaderAppTest { val initBootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags, null, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway); initBootloader.load(); - final ConfigPersistence localSchema = new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); + final DefinitionsProvider localDefinitions = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase); - configRepository.loadDataNoSecrets(localSchema); + final ConfigPersistence localConfigPersistence = new DefinitionProviderToConfigPersistenceAdapter(localDefinitions); + configRepository.loadDataNoSecrets(localConfigPersistence); final String sourceSpecs = """ { diff --git a/airbyte-config/config-persistence/readme.md b/airbyte-config/config-persistence/readme.md index 0bfd8b20dcf..b314dd62bf3 100644 --- a/airbyte-config/config-persistence/readme.md +++ b/airbyte-config/config-persistence/readme.md @@ -3,5 +3,5 @@ This module contains the logic for accessing the config database. This database is primarily used by the `airbyte-server` but is also accessed from `airbyte-workers`. It contains all configuration information for Airbyte. ## Key files -* `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java` The only other one that is used is the `YamlSeedConfigPersistence.java` which is used for loading configs that ship with the app. +* `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java`. * `ConfigRepository.java` is what is most used for accessing the databases. The `ConfigPersistence` iface was hard to work with. `ConfigRepository` builds on top of it and houses any databases queries to keep them from proliferating throughout the codebase. diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java index cd0f841e704..bc8207c55c5 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java @@ -33,6 +33,8 @@ import java.util.stream.Collectors; */ final public class LocalDefinitionsProvider implements DefinitionsProvider { + public static final Class DEFAULT_SEED_DEFINITION_RESOURCE_CLASS = SeedType.class; + private final static String PROTOCOL_VERSION = "protocol_version"; private final static String SPEC = "spec"; diff --git a/airbyte-config/init/src/test/java/io/airbyte/config/init/SpecFormatTest.java b/airbyte-config/init/src/test/java/io/airbyte/config/init/SpecFormatTest.java index ef7d7c8af80..c8b3561c892 100644 --- a/airbyte-config/init/src/test/java/io/airbyte/config/init/SpecFormatTest.java +++ b/airbyte-config/init/src/test/java/io/airbyte/config/init/SpecFormatTest.java @@ -6,10 +6,6 @@ package io.airbyte.config.init; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.JsonSchemas; -import io.airbyte.config.ConfigSchema; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -25,16 +21,14 @@ class SpecFormatTest { @Test void testOnAllExistingConfig() throws IOException, JsonValidationException { - final ConfigPersistence configPersistence = new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); + final DefinitionsProvider definitionsProvider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); - final List sourceSpecs = configPersistence.listConfigs( - ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class) + final List sourceSpecs = definitionsProvider.getSourceDefinitions() .stream() .map(standardSourceDefinition -> standardSourceDefinition.getSpec().getConnectionSpecification()) .toList(); - final List destinationSpecs = configPersistence.listConfigs( - ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class) + final List destinationSpecs = definitionsProvider.getDestinationDefinitions() .stream() .map(standardDestinationDefinition -> standardDestinationDefinition.getSpec().getConnectionSpecification()) .toList();