jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2026-01-10 09:04:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

remove usages of YamlSeedConfigPersistence (#17895)

Browse Source
This commit is contained in:
Pedro S. Lopez
2022-10-13 10:28:40 -04:00
committed by GitHub
parent 3268cfe1c3
commit 908f3949d2
5 changed files with 13 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java
View File

@@ -17,7 +17,6 @@ import io.airbyte.config.StandardWorkspace;
import io.airbyte.config.init.ApplyDefinitionsHelper;
import io.airbyte.config.init.DefinitionsProvider;
import io.airbyte.config.init.LocalDefinitionsProvider;
import io.airbyte.config.init.YamlSeedConfigPersistence;
import io.airbyte.config.persistence.ConfigPersistence;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.config.persistence.DatabaseConfigPersistence;
@@ -206,7 +205,7 @@ public class BootloaderApp {
}
private static DefinitionsProvider getLocalDefinitionsProvider() throws IOException {
return new LocalDefinitionsProvider(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
return new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
}
private static Database getJobDatabase(final DSLContext dslContext) throws IOException {

9
airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java
View File

@@ -24,7 +24,9 @@ import io.airbyte.config.Configs;
import io.airbyte.config.Geography;
import io.airbyte.config.SourceConnection;
import io.airbyte.config.StandardWorkspace;
import io.airbyte.config.init.YamlSeedConfigPersistence;
import io.airbyte.config.init.DefinitionProviderToConfigPersistenceAdapter;
import io.airbyte.config.init.DefinitionsProvider;
import io.airbyte.config.init.LocalDefinitionsProvider;
import io.airbyte.config.persistence.ConfigPersistence;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.config.persistence.DatabaseConfigPersistence;
@@ -199,9 +201,10 @@ class BootloaderAppTest {
val initBootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags, null, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway);
initBootloader.load();
final ConfigPersistence localSchema = new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
final DefinitionsProvider localDefinitions = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase);
configRepository.loadDataNoSecrets(localSchema);
final ConfigPersistence localConfigPersistence = new DefinitionProviderToConfigPersistenceAdapter(localDefinitions);
configRepository.loadDataNoSecrets(localConfigPersistence);
final String sourceSpecs = """
{

2
airbyte-config/config-persistence/readme.md
View File

@@ -3,5 +3,5 @@
This module contains the logic for accessing the config database. This database is primarily used by the `airbyte-server` but is also accessed from `airbyte-workers`. It contains all configuration information for Airbyte.
## Key files
* `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java` The only other one that is used is the `YamlSeedConfigPersistence.java` which is used for loading configs that ship with the app.
* `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java`.
* `ConfigRepository.java` is what is most used for accessing the databases. The `ConfigPersistence` iface was hard to work with. `ConfigRepository` builds on top of it and houses any databases queries to keep them from proliferating throughout the codebase.

2
airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java
View File

@@ -33,6 +33,8 @@ import java.util.stream.Collectors;
*/
final public class LocalDefinitionsProvider implements DefinitionsProvider {
public static final Class<?> DEFAULT_SEED_DEFINITION_RESOURCE_CLASS = SeedType.class;
private final static String PROTOCOL_VERSION = "protocol_version";
private final static String SPEC = "spec";

12
airbyte-config/init/src/test/java/io/airbyte/config/init/SpecFormatTest.java
View File

@@ -6,10 +6,6 @@ package io.airbyte.config.init;
import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.commons.json.JsonSchemas;
import io.airbyte.config.ConfigSchema;
import io.airbyte.config.StandardDestinationDefinition;
import io.airbyte.config.StandardSourceDefinition;
import io.airbyte.config.persistence.ConfigPersistence;
import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor;
import io.airbyte.validation.json.JsonValidationException;
import java.io.IOException;
@@ -25,16 +21,14 @@ class SpecFormatTest {
@Test
void testOnAllExistingConfig() throws IOException, JsonValidationException {
final ConfigPersistence configPersistence = new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
final DefinitionsProvider definitionsProvider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
final List<JsonNode> sourceSpecs = configPersistence.listConfigs(
ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class)
final List<JsonNode> sourceSpecs = definitionsProvider.getSourceDefinitions()
.stream()
.map(standardSourceDefinition -> standardSourceDefinition.getSpec().getConnectionSpecification())
.toList();
final List<JsonNode> destinationSpecs = configPersistence.listConfigs(
ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)
final List<JsonNode> destinationSpecs = definitionsProvider.getDestinationDefinitions()
.stream()
.map(standardDestinationDefinition -> standardDestinationDefinition.getSpec().getConnectionSpecification())
.toList();