diff --git a/airbyte-bootloader/build.gradle b/airbyte-bootloader/build.gradle index 4fb63889cba..77cf5c872fd 100644 --- a/airbyte-bootloader/build.gradle +++ b/airbyte-bootloader/build.gradle @@ -14,9 +14,9 @@ dependencies { implementation project(':airbyte-scheduler:persistence') implementation 'io.temporal:temporal-sdk:1.8.1' - implementation "org.flywaydb:flyway-core:7.14.0" + implementation libs.flyway.core - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.testcontainers.postgresql testImplementation 'uk.org.webcompere:system-stubs-jupiter:1.2.0' } diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java index 9a89a9507b8..089f5c0f9d4 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java @@ -4,9 +4,9 @@ package io.airbyte.bootloader; -import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; @@ -19,6 +19,10 @@ import io.airbyte.config.persistence.DatabaseConfigPersistence; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.DatabaseMigrator; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; @@ -30,6 +34,10 @@ import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.Optional; import java.util.UUID; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,20 +57,18 @@ public class BootloaderApp { private static final Logger LOGGER = LoggerFactory.getLogger(BootloaderApp.class); private static final AirbyteVersion VERSION_BREAK = new AirbyteVersion("0.32.0-alpha"); + private static final String DRIVER_CLASS_NAME = DatabaseDriver.POSTGRESQL.getDriverClassName(); private final Configs configs; private final Runnable postLoadExecution; private final FeatureFlags featureFlags; - private SecretMigrator secretMigrator; + private final SecretMigrator secretMigrator; private ConfigPersistence configPersistence; private Database configDatabase; private Database jobDatabase; private JobPersistence jobPersistence; - - @VisibleForTesting - public BootloaderApp(final Configs configs, final FeatureFlags featureFlags) { - this(configs, () -> {}, featureFlags, null); - } + private final Flyway configsFlyway; + private final Flyway jobsFlyway; /** * This method is exposed for Airbyte Cloud consumption. This lets us override the seed loading @@ -71,81 +77,94 @@ public class BootloaderApp { * * @param configs * @param postLoadExecution + * @param featureFlags + * @param secretMigrator + * @param configsDslContext */ public BootloaderApp(final Configs configs, final Runnable postLoadExecution, final FeatureFlags featureFlags, - final SecretMigrator secretMigrator) { + final SecretMigrator secretMigrator, + final DSLContext configsDslContext, + final DSLContext jobsDslContext, + final Flyway configsFlyway, + final Flyway jobsFlyway) { this.configs = configs; this.postLoadExecution = postLoadExecution; this.featureFlags = featureFlags; this.secretMigrator = secretMigrator; + this.configsFlyway = configsFlyway; + this.jobsFlyway = jobsFlyway; - initPersistences(); + initPersistences(configsDslContext, jobsDslContext); } - public BootloaderApp(final Configs configs, final FeatureFlags featureFlags, final SecretMigrator secretMigrator) { + public BootloaderApp(final Configs configs, + final FeatureFlags featureFlags, + final SecretMigrator secretMigrator, + final DSLContext configsDslContext, + final DSLContext jobsDslContext, + final Flyway configsFlyway, + final Flyway jobsFlyway) { this.configs = configs; this.featureFlags = featureFlags; + this.secretMigrator = secretMigrator; + this.configsFlyway = configsFlyway; + this.jobsFlyway = jobsFlyway; - initPersistences(); + initPersistences(configsDslContext, jobsDslContext); postLoadExecution = () -> { try { configPersistence.loadData(YamlSeedConfigPersistence.getDefault()); if (featureFlags.forceSecretMigration() || !jobPersistence.isSecretMigrated()) { - secretMigrator.migrateSecrets(); + if (this.secretMigrator != null) { + this.secretMigrator.migrateSecrets(); + LOGGER.info("Secrets successfully migrated."); + } } LOGGER.info("Loaded seed data.."); } catch (final IOException | JsonValidationException e) { throw new RuntimeException(e); } }; - } public void load() throws Exception { - LOGGER.info("Setting up config database and default workspace.."); + LOGGER.info("Setting up config database and default workspace..."); + final JobPersistence jobPersistence = new DefaultJobPersistence(jobDatabase); + final AirbyteVersion currAirbyteVersion = configs.getAirbyteVersion(); + assertNonBreakingMigration(jobPersistence, currAirbyteVersion); - try { - LOGGER.info("Created initial jobs and configs database..."); + // TODO Will be converted to an injected singleton during DI migration + final DatabaseMigrator configDbMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway); + final DatabaseMigrator jobDbMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); - final JobPersistence jobPersistence = new DefaultJobPersistence(jobDatabase); - final AirbyteVersion currAirbyteVersion = configs.getAirbyteVersion(); - assertNonBreakingMigration(jobPersistence, currAirbyteVersion); + runFlywayMigration(configs, configDbMigrator, jobDbMigrator); + LOGGER.info("Ran Flyway migrations."); - runFlywayMigration(configs, configDatabase, jobDatabase); - LOGGER.info("Ran Flyway migrations..."); + final ConfigRepository configRepository = + new ConfigRepository(configPersistence, configDatabase); - final ConfigRepository configRepository = - new ConfigRepository(configPersistence, configDatabase); + createWorkspaceIfNoneExists(configRepository); + LOGGER.info("Default workspace created."); - createWorkspaceIfNoneExists(configRepository); - LOGGER.info("Default workspace created.."); + createDeploymentIfNoneExists(jobPersistence); + LOGGER.info("Default deployment created."); - createDeploymentIfNoneExists(jobPersistence); - LOGGER.info("Default deployment created.."); + jobPersistence.setVersion(currAirbyteVersion.serialize()); + LOGGER.info("Set version to {}", currAirbyteVersion); - jobPersistence.setVersion(currAirbyteVersion.serialize()); - LOGGER.info("Set version to {}", currAirbyteVersion); + postLoadExecution.run(); - postLoadExecution.run(); - } finally { - jobDatabase.close(); - configDatabase.close(); - } + LOGGER.info("Finished running post load Execution."); - LOGGER.info("Finished running post load Execution.."); - - LOGGER.info("Finished bootstrapping Airbyte environment.."); + LOGGER.info("Finished bootstrapping Airbyte environment."); } - private static Database getConfigDatabase(final Configs configs) throws IOException { - return new ConfigsDatabaseInstance( - configs.getConfigDatabaseUser(), - configs.getConfigDatabasePassword(), - configs.getConfigDatabaseUrl()).getAndInitialize(); + private static Database getConfigDatabase(final DSLContext dslContext) throws IOException { + return new ConfigsDatabaseInstance(dslContext).getAndInitialize(); } private static ConfigPersistence getConfigPersistence(final Database configDatabase) throws IOException { @@ -157,36 +176,56 @@ public class BootloaderApp { return DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor); } - private static Database getJobDatabase(final Configs configs) throws IOException { - return new JobsDatabaseInstance(configs.getDatabaseUser(), configs.getDatabasePassword(), configs.getDatabaseUrl()).getAndInitialize(); + private static Database getJobDatabase(final DSLContext dslContext) throws IOException { + return new JobsDatabaseInstance(dslContext).getAndInitialize(); } private static JobPersistence getJobPersistence(final Database jobDatabase) throws IOException { return new DefaultJobPersistence(jobDatabase); } - private void initPersistences() { + private void initPersistences(final DSLContext configsDslContext, final DSLContext jobsDslContext) { try { - configDatabase = getConfigDatabase(configs); + configDatabase = getConfigDatabase(configsDslContext); configPersistence = getConfigPersistence(configDatabase); - jobDatabase = getJobDatabase(configs); + jobDatabase = getJobDatabase(jobsDslContext); jobPersistence = getJobPersistence(jobDatabase); } catch (final IOException e) { - e.printStackTrace(); + LOGGER.error("Unable to initialize persistence.", e); } } public static void main(final String[] args) throws Exception { final Configs configs = new EnvConfigs(); final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); - final Database configDatabase = getConfigDatabase(configs); - final ConfigPersistence configPersistence = getConfigPersistence(configDatabase); - final Database jobDatabase = getJobDatabase(configs); - final JobPersistence jobPersistence = getJobPersistence(jobDatabase); - final SecretMigrator secretMigrator = new SecretMigrator(configPersistence, jobPersistence, SecretPersistence.getLongLived(configs)); - final Optional secretPersistence = SecretPersistence.getLongLived(configs); - final var bootloader = new BootloaderApp(configs, featureFlags, secretMigrator); - bootloader.load(); + + // Manual configuration that will be replaced by Dependency Injection in the future + final DataSource configsDataSource = DataSourceFactory.create(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), + DRIVER_CLASS_NAME, configs.getConfigDatabaseUrl()); + final DataSource jobsDataSource = + DataSourceFactory.create(configs.getDatabaseUser(), configs.getDatabasePassword(), DRIVER_CLASS_NAME, configs.getDatabaseUrl()); + + try (final DSLContext configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); + final DSLContext jobsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES)) { + + // TODO Will be converted to an injected singleton during DI migration + final Database configDatabase = getConfigDatabase(configsDslContext); + final ConfigPersistence configPersistence = getConfigPersistence(configDatabase); + final Database jobDatabase = getJobDatabase(jobsDslContext); + final JobPersistence jobPersistence = getJobPersistence(jobDatabase); + final SecretMigrator secretMigrator = + new SecretMigrator(configPersistence, jobPersistence, SecretPersistence.getLongLived(configsDslContext, configs)); + final Flyway configsFlyway = FlywayFactory.create(configsDataSource, BootloaderApp.class.getSimpleName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final Flyway jobsFlyway = FlywayFactory.create(jobsDataSource, BootloaderApp.class.getSimpleName(), JobsDatabaseMigrator.DB_IDENTIFIER, + JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); + + // Ensure that the database resources are closed on application shutdown + CloseableShutdownHook.registerRuntimeShutdownHook(configsDataSource, jobsDataSource, configsDslContext, jobsDslContext); + + final var bootloader = new BootloaderApp(configs, featureFlags, secretMigrator, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway); + bootloader.load(); + } } private static void createDeploymentIfNoneExists(final JobPersistence jobPersistence) throws IOException { @@ -253,10 +292,7 @@ public class BootloaderApp { return !isUpgradingThroughVersionBreak; } - private static void runFlywayMigration(final Configs configs, final Database configDatabase, final Database jobDatabase) { - final DatabaseMigrator configDbMigrator = new ConfigsDatabaseMigrator(configDatabase, BootloaderApp.class.getSimpleName()); - final DatabaseMigrator jobDbMigrator = new JobsDatabaseMigrator(jobDatabase, BootloaderApp.class.getSimpleName()); - + private static void runFlywayMigration(final Configs configs, final DatabaseMigrator configDbMigrator, final DatabaseMigrator jobDbMigrator) { configDbMigrator.createBaseline(); jobDbMigrator.createBaseline(); diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java index 439cd790a68..7436ac1ceb7 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java @@ -29,16 +29,25 @@ import io.airbyte.config.persistence.DatabaseConfigPersistence; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.jobs.JobsDatabaseInstance; import io.airbyte.db.instance.jobs.JobsDatabaseMigrator; import io.airbyte.scheduler.persistence.DefaultJobPersistence; +import java.io.Closeable; +import java.io.IOException; import java.util.Optional; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import javax.sql.DataSource; import lombok.val; +import org.flywaydb.core.Flyway; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; @@ -49,16 +58,36 @@ import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; @ExtendWith(SystemStubsExtension.class) public class BootloaderAppTest { + private PostgreSQLContainer container; + private DataSource configsDataSource; + private DataSource jobsDataSource; + + @BeforeEach + void setup() { + container = new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName("public") + .withUsername("docker") + .withPassword("docker"); + container.start(); + + configsDataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), container.getDriverClassName(), container.getJdbcUrl()); + jobsDataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), container.getDriverClassName(), container.getJdbcUrl()); + } + + @AfterEach + void cleanup() throws IOException { + closeDataSource(configsDataSource); + closeDataSource(jobsDataSource); + container.stop(); + } + @SystemStub private EnvironmentVariables environmentVariables; @Test void testBootloaderAppBlankDb() throws Exception { - val container = new PostgreSQLContainer<>("postgres:13-alpine") - .withDatabaseName("public") - .withUsername("docker") - .withPassword("docker"); - container.start(); val version = "0.33.0-alpha"; val mockedConfigs = mock(Configs.class); @@ -83,37 +112,33 @@ public class BootloaderAppTest { environmentVariables.set("DATABASE_PASSWORD", "docker"); environmentVariables.set("DATABASE_URL", container.getJdbcUrl()); - val bootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags); - bootloader.load(); + try (val configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); + val jobsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES)) { - val jobDatabase = new JobsDatabaseInstance( - container.getUsername(), - container.getPassword(), - container.getJdbcUrl()).getInitialized(); - val jobsMigrator = new JobsDatabaseMigrator(jobDatabase, this.getClass().getName()); - assertEquals("0.35.62.001", jobsMigrator.getLatestMigration().getVersion().getVersion()); + val configsFlyway = createConfigsFlyway(configsDataSource); + val jobsFlyway = createJobsFlyway(jobsDataSource); - val configDatabase = new ConfigsDatabaseInstance( - mockedConfigs.getConfigDatabaseUser(), - mockedConfigs.getConfigDatabasePassword(), - mockedConfigs.getConfigDatabaseUrl()) - .getAndInitialize(); - val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, this.getClass().getName()); - assertEquals("0.35.65.001", configsMigrator.getLatestMigration().getVersion().getVersion()); + val bootloader = + new BootloaderApp(mockedConfigs, mockedFeatureFlags, mockedSecretMigrator, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway); + bootloader.load(); - val jobsPersistence = new DefaultJobPersistence(jobDatabase); - assertEquals(version, jobsPersistence.getVersion().get()); + val jobDatabase = new JobsDatabaseInstance(jobsDslContext).getInitialized(); + val jobsMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); + assertEquals("0.35.62.001", jobsMigrator.getLatestMigration().getVersion().getVersion()); - assertNotEquals(Optional.empty(), jobsPersistence.getDeployment().get()); + val configDatabase = new ConfigsDatabaseInstance(configsDslContext).getAndInitialize(); + val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway); + assertEquals("0.35.65.001", configsMigrator.getLatestMigration().getVersion().getVersion()); + + val jobsPersistence = new DefaultJobPersistence(jobDatabase); + assertEquals(version, jobsPersistence.getVersion().get()); + + assertNotEquals(Optional.empty(), jobsPersistence.getDeployment().get()); + } } @Test void testBootloaderAppRunSecretMigration() throws Exception { - val container = new PostgreSQLContainer<>("postgres:13-alpine") - .withDatabaseName("public") - .withUsername("docker") - .withPassword("docker"); - container.start(); val version = "0.33.0-alpha"; val mockedConfigs = mock(Configs.class); @@ -134,85 +159,102 @@ public class BootloaderAppTest { .copySecrets(true) .maskSecrets(true) .build(); - final Database configDatabase = Databases.createPostgresDatabase(container.getUsername(), container.getPassword(), container.getJdbcUrl()); - final ConfigPersistence configPersistence = new DatabaseConfigPersistence(configDatabase, jsonSecretsProcessor); - val jobsPersistence = new DefaultJobPersistence(configDatabase); + try (val configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); + val jobsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES)) { - val spiedSecretMigrator = spy(new SecretMigrator(configPersistence, jobsPersistence, SecretPersistence.getLongLived(mockedConfigs))); + val configsFlyway = createConfigsFlyway(configsDataSource); + val jobsFlyway = createJobsFlyway(jobsDataSource); - // Although we are able to inject mocked configs into the Bootloader, a particular migration in the - // configs database - // requires the env var to be set. Flyway prevents injection, so we dynamically set this instead. - environmentVariables.set("DATABASE_USER", "docker"); - environmentVariables.set("DATABASE_PASSWORD", "docker"); - environmentVariables.set("DATABASE_URL", container.getJdbcUrl()); + final Database configDatabase = new Database(configsDslContext); + final ConfigPersistence configPersistence = new DatabaseConfigPersistence(configDatabase, jsonSecretsProcessor); - val initBootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags); - initBootloader.load(); + val jobsPersistence = new DefaultJobPersistence(configDatabase); - final ConfigPersistence localSchema = YamlSeedConfigPersistence.getDefault(); - final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase); - configRepository.loadDataNoSecrets(localSchema); + val spiedSecretMigrator = + spy(new SecretMigrator(configPersistence, jobsPersistence, SecretPersistence.getLongLived(configsDslContext, mockedConfigs))); - final String sourceSpecs = """ - { - "account_id": "1234567891234567", - "start_date": "2022-04-01T00:00:00Z", - "access_token": "nonhiddensecret", - "include_deleted": false, - "fetch_thumbnail_images": false - } + // Although we are able to inject mocked configs into the Bootloader, a particular migration in the + // configs database requires the env var to be set. Flyway prevents injection, so we dynamically set + // this instead. + environmentVariables.set("DATABASE_USER", "docker"); + environmentVariables.set("DATABASE_PASSWORD", "docker"); + environmentVariables.set("DATABASE_URL", container.getJdbcUrl()); - """; + // Bootstrap the database for the test + val initBootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags, null, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway); + initBootloader.load(); - final ObjectMapper mapper = new ObjectMapper(); + final ConfigPersistence localSchema = YamlSeedConfigPersistence.getDefault(); + final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase); + configRepository.loadDataNoSecrets(localSchema); - final UUID workspaceId = UUID.randomUUID(); - configRepository.writeStandardWorkspace(new StandardWorkspace() - .withWorkspaceId(workspaceId) - .withName("wName") - .withSlug("wSlug") - .withEmail("email@mail.com") - .withTombstone(false) - .withInitialSetupComplete(false)); - final UUID sourceId = UUID.randomUUID(); - configRepository.writeSourceConnectionNoSecrets(new SourceConnection() - .withSourceDefinitionId(UUID.fromString("e7778cfc-e97c-4458-9ecb-b4f2bba8946c")) // Facebook Marketing - .withSourceId(sourceId) - .withName("test source") - .withWorkspaceId(workspaceId) - .withConfiguration(mapper.readTree(sourceSpecs))); + final String sourceSpecs = """ + { + "account_id": "1234567891234567", + "start_date": "2022-04-01T00:00:00Z", + "access_token": "nonhiddensecret", + "include_deleted": false, + "fetch_thumbnail_images": false + } - when(mockedFeatureFlags.forceSecretMigration()).thenReturn(false); - var bootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags, spiedSecretMigrator); - boolean isMigrated = jobsPersistence.isSecretMigrated(); + """; - assertFalse(isMigrated); + final ObjectMapper mapper = new ObjectMapper(); - bootloader.load(); - verify(spiedSecretMigrator).migrateSecrets(); + final UUID workspaceId = UUID.randomUUID(); + configRepository.writeStandardWorkspace(new StandardWorkspace() + .withWorkspaceId(workspaceId) + .withName("wName") + .withSlug("wSlug") + .withEmail("email@mail.com") + .withTombstone(false) + .withInitialSetupComplete(false)); + final UUID sourceId = UUID.randomUUID(); + configRepository.writeSourceConnectionNoSecrets(new SourceConnection() + .withSourceDefinitionId(UUID.fromString("e7778cfc-e97c-4458-9ecb-b4f2bba8946c")) // Facebook Marketing + .withSourceId(sourceId) + .withName("test source") + .withWorkspaceId(workspaceId) + .withConfiguration(mapper.readTree(sourceSpecs))); - final SourceConnection sourceConnection = configRepository.getSourceConnection(sourceId); + when(mockedFeatureFlags.forceSecretMigration()).thenReturn(false); - assertFalse(sourceConnection.getConfiguration().toString().contains("nonhiddensecret")); - assertTrue(sourceConnection.getConfiguration().toString().contains("_secret")); + // Perform secrets migration + var bootloader = + new BootloaderApp(mockedConfigs, mockedFeatureFlags, spiedSecretMigrator, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway); + boolean isMigrated = jobsPersistence.isSecretMigrated(); - isMigrated = jobsPersistence.isSecretMigrated(); - assertTrue(isMigrated); + assertFalse(isMigrated); - reset(spiedSecretMigrator); - // We need to re-create the bootloader because it is closing the persistence after running load - bootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags, spiedSecretMigrator); - bootloader.load(); - verifyNoInteractions(spiedSecretMigrator); + bootloader.load(); + verify(spiedSecretMigrator).migrateSecrets(); - reset(spiedSecretMigrator); - when(mockedFeatureFlags.forceSecretMigration()).thenReturn(true); - // We need to re-create the bootloader because it is closing the persistence after running load - bootloader = new BootloaderApp(mockedConfigs, mockedFeatureFlags, spiedSecretMigrator); - bootloader.load(); - verify(spiedSecretMigrator).migrateSecrets(); + final SourceConnection sourceConnection = configRepository.getSourceConnection(sourceId); + + assertFalse(sourceConnection.getConfiguration().toString().contains("nonhiddensecret")); + assertTrue(sourceConnection.getConfiguration().toString().contains("_secret")); + + isMigrated = jobsPersistence.isSecretMigrated(); + assertTrue(isMigrated); + + // Verify that the migration does not happen if it has already been performed + reset(spiedSecretMigrator); + // We need to re-create the bootloader because it is closing the persistence after running load + bootloader = + new BootloaderApp(mockedConfigs, mockedFeatureFlags, spiedSecretMigrator, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway); + bootloader.load(); + verifyNoInteractions(spiedSecretMigrator); + + // Verify that the migration occurs if the force migration feature flag is enabled + reset(spiedSecretMigrator); + when(mockedFeatureFlags.forceSecretMigration()).thenReturn(true); + // We need to re-create the bootloader because it is closing the persistence after running load + bootloader = + new BootloaderApp(mockedConfigs, mockedFeatureFlags, spiedSecretMigrator, configsDslContext, jobsDslContext, configsFlyway, jobsFlyway); + bootloader.load(); + verify(spiedSecretMigrator).migrateSecrets(); + } } @Test @@ -240,12 +282,6 @@ public class BootloaderAppTest { @Test void testPostLoadExecutionExecutes() throws Exception { final var testTriggered = new AtomicBoolean(); - - val container = new PostgreSQLContainer<>("postgres:13-alpine") - .withDatabaseName("public") - .withUsername("docker") - .withPassword("docker"); - container.start(); val version = "0.33.0-alpha"; val mockedConfigs = mock(Configs.class); @@ -263,9 +299,34 @@ public class BootloaderAppTest { val mockedSecretMigrator = mock(SecretMigrator.class); - new BootloaderApp(mockedConfigs, () -> testTriggered.set(true), mockedFeatureFlags, mockedSecretMigrator).load(); + try (val configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); + val jobsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES)) { - assertTrue(testTriggered.get()); + val configsFlyway = createConfigsFlyway(configsDataSource); + val jobsFlyway = createJobsFlyway(jobsDataSource); + + new BootloaderApp(mockedConfigs, () -> testTriggered.set(true), mockedFeatureFlags, mockedSecretMigrator, configsDslContext, jobsDslContext, + configsFlyway, jobsFlyway) + .load(); + + assertTrue(testTriggered.get()); + } + } + + private Flyway createConfigsFlyway(final DataSource dataSource) { + return FlywayFactory.create(dataSource, getClass().getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + } + + private Flyway createJobsFlyway(final DataSource dataSource) { + return FlywayFactory.create(dataSource, getClass().getName(), JobsDatabaseMigrator.DB_IDENTIFIER, + JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); + } + + private void closeDataSource(final DataSource dataSource) throws IOException { + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java b/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java new file mode 100644 index 00000000000..2ac113181c5 --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.lang; + +import com.google.common.annotations.VisibleForTesting; +import java.io.Closeable; +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Registers a shutdown hook that calls the close method of the provided objects. If an object does + * not support either the {@link AutoCloseable} or {@link Closeable} interface, it will be ignored. + * + * This is a temporary class that is being provided to ensure that resources created by each + * application are properly closed on shutdown. This logic will no longer be necessary once an + * application framework is introduced to the project that can provide object lifecycle management. + */ +public class CloseableShutdownHook { + + private static final Logger LOGGER = LoggerFactory.getLogger(CloseableShutdownHook.class); + + /** + * Registers a runtime shutdown hook with the application for each provided closeable object. + * + * @param objects An array of objects to be closed on application shutdown. + */ + public static void registerRuntimeShutdownHook(final Object... objects) { + Runtime.getRuntime().addShutdownHook(buildShutdownHookThread(objects)); + } + + /** + * Builds the {@link Thread} that will be registered as an application shutdown hook. + * + * @param objects An array of objects to be closed on application shutdown. + * @return The application shutdown hook {@link Thread}. + */ + @VisibleForTesting + static Thread buildShutdownHookThread(final Object... objects) { + final Collection autoCloseables = Stream.of(objects) + .filter(o -> o != null) + .filter(o -> o instanceof AutoCloseable) + .map(o -> AutoCloseable.class.cast(o)) + .collect(Collectors.toList()); + + return new Thread(() -> { + autoCloseables.forEach(CloseableShutdownHook::close); + }); + } + + private static void close(final AutoCloseable autoCloseable) { + try { + autoCloseable.close(); + } catch (final Exception e) { + LOGGER.error("Unable to close object {}.", autoCloseable.getClass().getName(), e); + } + } + +} diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java new file mode 100644 index 00000000000..86e24bfab43 --- /dev/null +++ b/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.lang; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.InputStream; +import org.junit.jupiter.api.Test; + +public class CloseableShutdownHookTest { + + @Test + void testRegisteringShutdownHook() throws Exception { + final InputStream closeable = mock(InputStream.class); + final CloseableQueue autoCloseable = mock(CloseableQueue.class); + final String notCloseable = "Not closeable"; + + final Thread thread = CloseableShutdownHook.buildShutdownHookThread(closeable, autoCloseable, notCloseable, null); + thread.run(); + + verify(closeable, times(1)).close(); + verify(autoCloseable, times(1)).close(); + } + +} diff --git a/airbyte-config/persistence/build.gradle b/airbyte-config/persistence/build.gradle index 8fe422a4fa6..6c415f1e6af 100644 --- a/airbyte-config/persistence/build.gradle +++ b/airbyte-config/persistence/build.gradle @@ -16,7 +16,8 @@ dependencies { implementation 'com.google.cloud:google-cloud-secretmanager:2.0.5' testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.testcontainers.postgresql + testImplementation libs.flyway.core testImplementation project(':airbyte-test-utils') integrationTestJavaImplementation project(':airbyte-config:persistence') } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java index f90ec25b92d..c76ac26b563 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java @@ -9,6 +9,7 @@ import io.airbyte.db.Database; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import java.io.IOException; import java.util.Optional; +import org.jooq.DSLContext; /** * Provides the ability to read and write secrets to a backing store. Assumes that secret payloads @@ -21,15 +22,10 @@ public interface SecretPersistence extends ReadOnlySecretPersistence { void write(final SecretCoordinate coordinate, final String payload) throws IllegalArgumentException; - static Optional getLongLived(final Configs configs) throws IOException { + static Optional getLongLived(final DSLContext dslContext, final Configs configs) throws IOException { switch (configs.getSecretPersistenceType()) { case TESTING_CONFIG_DB_TABLE -> { - final Database configDatabase = new ConfigsDatabaseInstance( - configs.getConfigDatabaseUser(), - configs.getConfigDatabasePassword(), - configs.getConfigDatabaseUrl()) - .getAndInitialize(); - + final Database configDatabase = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); return Optional.of(new LocalTestingSecretPersistence(configDatabase)); } case GOOGLE_SECRET_MANAGER -> { @@ -41,8 +37,8 @@ public interface SecretPersistence extends ReadOnlySecretPersistence { } } - static SecretsHydrator getSecretsHydrator(final Configs configs) throws IOException { - final var persistence = getLongLived(configs); + static SecretsHydrator getSecretsHydrator(final DSLContext dslContext, final Configs configs) throws IOException { + final var persistence = getLongLived(dslContext, configs); if (persistence.isPresent()) { return new RealSecretsHydrator(persistence.get()); @@ -51,15 +47,10 @@ public interface SecretPersistence extends ReadOnlySecretPersistence { } } - static Optional getEphemeral(final Configs configs) throws IOException { + static Optional getEphemeral(final DSLContext dslContext, final Configs configs) throws IOException { switch (configs.getSecretPersistenceType()) { case TESTING_CONFIG_DB_TABLE -> { - final Database configDatabase = new ConfigsDatabaseInstance( - configs.getConfigDatabaseUser(), - configs.getConfigDatabasePassword(), - configs.getConfigDatabaseUrl()) - .getAndInitialize(); - + final Database configDatabase = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); return Optional.of(new LocalTestingSecretPersistence(configDatabase)); } case GOOGLE_SECRET_MANAGER -> { diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java index a2037db0154..a5e00f96d05 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java @@ -29,6 +29,9 @@ import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; import org.jooq.Record1; import org.jooq.Result; import org.jooq.Table; @@ -45,6 +48,9 @@ public abstract class BaseDatabaseConfigPersistenceTest { protected static Database database; protected static DatabaseConfigPersistence configPersistence; protected static JsonSecretsProcessor jsonSecretsProcessor; + protected static DataSource dataSource; + protected static DSLContext dslContext; + protected static Flyway flyway; @BeforeAll public static void dbSetup() { diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 29d1644e1d8..e1c40b9cbe3 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -28,6 +28,8 @@ import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.development.DevDatabaseMigrator; @@ -36,6 +38,7 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.sql.SQLException; @@ -47,6 +50,10 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -56,10 +63,13 @@ import org.testcontainers.containers.PostgreSQLContainer; public class ConfigRepositoryE2EReadWriteTest { private static PostgreSQLContainer container; + private DataSource dataSource; + private DSLContext dslContext; private Database database; private ConfigRepository configRepository; private DatabaseConfigPersistence configPersistence; private JsonSecretsProcessor jsonSecretsProcessor; + private Flyway flyway; @BeforeAll public static void dbSetup() { @@ -72,12 +82,16 @@ public class ConfigRepositoryE2EReadWriteTest { @BeforeEach void setup() throws IOException, JsonValidationException, SQLException { - database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceLoadDataTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); jsonSecretsProcessor = mock(JsonSecretsProcessor.class); configPersistence = spy(new DatabaseConfigPersistence(database, jsonSecretsProcessor)); configRepository = spy(new ConfigRepository(configPersistence, database)); final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + new ConfigsDatabaseMigrator(database, flyway); final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); MigrationDevHelper.runLastMigration(devDatabaseMigrator); for (final StandardWorkspace workspace : MockData.standardWorkspaces()) { diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java index f05fbb986ff..ad215afe6a4 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java @@ -24,13 +24,18 @@ import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardSyncState; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.WorkspaceServiceAccount; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.development.DevDatabaseMigrator; import io.airbyte.db.instance.development.MigrationDevHelper; +import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonValidationException; +import java.io.Closeable; import java.io.IOException; import java.util.List; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,19 +44,26 @@ public class DatabaseConfigPersistenceE2EReadWriteTest extends BaseDatabaseConfi @BeforeEach public void setup() throws Exception { - database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); + flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceLoadDataTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); configPersistence = spy(new DatabaseConfigPersistence(database, jsonSecretsProcessor)); final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + new ConfigsDatabaseMigrator(database, flyway); final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); MigrationDevHelper.runLastMigration(devDatabaseMigrator); truncateAllTables(); } @AfterEach - void tearDown() throws Exception { - database.close(); + void tearDown() throws IOException { + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } @Test diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java index 5033c50b706..1665be0f7d9 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java @@ -21,13 +21,19 @@ import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardWorkspace; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.development.DevDatabaseMigrator; import io.airbyte.db.instance.development.MigrationDevHelper; +import io.airbyte.test.utils.DatabaseConnectionHelper; +import java.io.Closeable; +import java.io.IOException; import java.util.Collections; import java.util.UUID; import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -47,18 +53,26 @@ public class DatabaseConfigPersistenceLoadDataTest extends BaseDatabaseConfigPer @BeforeAll public static void setup() throws Exception { - database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); + flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceLoadDataTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); configPersistence = spy(new DatabaseConfigPersistence(database, jsonSecretsProcessor)); final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + new ConfigsDatabaseMigrator(database, flyway); final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); MigrationDevHelper.runLastMigration(devDatabaseMigrator); truncateAllTables(); } @AfterAll - public static void tearDown() throws Exception { - database.close(); + public static void tearDown() throws IOException { + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } @BeforeEach diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java index 2bf79d927c8..a67fd909809 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java @@ -30,11 +30,16 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.ReleaseStage; import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.development.DevDatabaseMigrator; import io.airbyte.db.instance.development.MigrationDevHelper; import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.test.utils.DatabaseConnectionHelper; +import java.io.Closeable; +import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.time.LocalDate; @@ -47,6 +52,7 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import org.assertj.core.api.Assertions; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -59,18 +65,26 @@ public class DatabaseConfigPersistenceTest extends BaseDatabaseConfigPersistence @BeforeEach public void setup() throws Exception { - database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); + flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceLoadDataTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); configPersistence = spy(new DatabaseConfigPersistence(database, jsonSecretsProcessor)); final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + new ConfigsDatabaseMigrator(database, flyway); final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); MigrationDevHelper.runLastMigration(devDatabaseMigrator); truncateAllTables(); } @AfterEach - void tearDown() throws Exception { - database.close(); + void tearDown() throws IOException { + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } @Test diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java index 6b6261aed50..a92260af618 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java @@ -12,10 +12,14 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConfigSchema; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.development.DevDatabaseMigrator; import io.airbyte.db.instance.development.MigrationDevHelper; +import io.airbyte.test.utils.DatabaseConnectionHelper; +import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; import java.util.Collections; @@ -23,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -38,17 +43,25 @@ public class DatabaseConfigPersistenceUpdateConnectorDefinitionsTest extends Bas @BeforeAll public static void setup() throws Exception { - database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); + flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceLoadDataTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + database = new ConfigsDatabaseInstance(dslContext).getAndInitialize(); configPersistence = new DatabaseConfigPersistence(database, jsonSecretsProcessor); final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, DatabaseConfigPersistenceLoadDataTest.class.getName()); + new ConfigsDatabaseMigrator(database, flyway); final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); MigrationDevHelper.runLastMigration(devDatabaseMigrator); } @AfterAll - public static void tearDown() throws Exception { - database.close(); + public static void tearDown() throws IOException { + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } @BeforeEach diff --git a/airbyte-container-orchestrator/build.gradle b/airbyte-container-orchestrator/build.gradle index 6356b42c3be..ee0da304ba3 100644 --- a/airbyte-container-orchestrator/build.gradle +++ b/airbyte-container-orchestrator/build.gradle @@ -20,9 +20,9 @@ dependencies { implementation project(':airbyte-workers') testImplementation 'org.mockito:mockito-inline:2.13.0' - testImplementation 'org.postgresql:postgresql:42.2.18' - testImplementation 'org.testcontainers:testcontainers:1.15.3' - testImplementation 'org.testcontainers:postgresql:1.15.3' + testImplementation libs.postgresql + testImplementation libs.testcontainers + testImplementation libs.testcontainers.postgresql testImplementation project(':airbyte-commons-docker') } diff --git a/airbyte-db/jooq/build.gradle b/airbyte-db/jooq/build.gradle index 48041d1b1ac..24b41309eed 100644 --- a/airbyte-db/jooq/build.gradle +++ b/airbyte-db/jooq/build.gradle @@ -4,16 +4,15 @@ plugins { } dependencies { - implementation 'org.jooq:jooq-meta:3.13.4' - implementation 'org.jooq:jooq:3.13.4' - implementation 'org.postgresql:postgresql:42.3.1' - implementation "org.flywaydb:flyway-core:7.14.0" - + implementation libs.jooq.meta + implementation libs.jooq + implementation libs.postgresql + implementation libs.flyway.core implementation project(':airbyte-db:lib') // jOOQ code generation - implementation 'org.jooq:jooq-codegen:3.13.4' - implementation "org.testcontainers:postgresql:1.15.3" + implementation libs.jooq.codegen + implementation libs.testcontainers.postgresql // These are required because gradle might be using lower version of Jna from other // library transitive dependency. Can be removed if we can figure out which library is the cause. // Refer: https://github.com/testcontainers/testcontainers-java/issues/3834#issuecomment-825409079 diff --git a/airbyte-db/lib/build.gradle b/airbyte-db/lib/build.gradle index 3d52778abfc..352fce360b9 100644 --- a/airbyte-db/lib/build.gradle +++ b/airbyte-db/lib/build.gradle @@ -3,24 +3,22 @@ plugins { } dependencies { - api 'org.apache.commons:commons-dbcp2:2.7.0' - api 'com.zaxxer:HikariCP:5.0.1' - api 'org.jooq:jooq-meta:3.13.4' - api 'org.jooq:jooq:3.13.4' - api 'org.postgresql:postgresql:42.2.18' + api libs.hikaricp + api libs.jooq.meta + api libs.jooq + api libs.postgresql implementation project(':airbyte-protocol:models') implementation project(':airbyte-json-validation') implementation project(':airbyte-config:models') - implementation "org.flywaydb:flyway-core:7.14.0" - implementation "org.testcontainers:postgresql:1.15.3" + implementation libs.flyway.core + implementation libs.testcontainers.postgresql // These are required because gradle might be using lower version of Jna from other // library transitive dependency. Can be removed if we can figure out which library is the cause. // Refer: https://github.com/testcontainers/testcontainers-java/issues/3834#issuecomment-825409079 implementation 'net.java.dev.jna:jna:5.8.0' implementation 'net.java.dev.jna:jna-platform:5.8.0' - testImplementation project(':airbyte-test-utils') testImplementation 'org.apache.commons:commons-lang3:3.11' diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Database.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Database.java index 0f779909781..e620adef10a 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Database.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Database.java @@ -4,48 +4,84 @@ package io.airbyte.db; -import java.io.Closeable; +import io.airbyte.commons.lang.Exceptions; +import java.io.IOException; import java.sql.SQLException; -import javax.sql.DataSource; -import org.jooq.SQLDialect; +import java.util.function.Function; +import lombok.val; +import org.jooq.DSLContext; import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Database object for interacting with a Jooq connection. */ -public class Database implements AutoCloseable { +public class Database { - private final DataSource ds; - private final SQLDialect dialect; + private static final Logger LOGGER = LoggerFactory.getLogger(Database.class); - public Database(final DataSource ds, final SQLDialect dialect) { - this.ds = ds; - this.dialect = dialect; + private static final long DEFAULT_WAIT_MS = 5 * 1000; + + private final DSLContext dslContext; + + public Database(final DSLContext dslContext) { + this.dslContext = dslContext; } public T query(final ContextQueryFunction transform) throws SQLException { - return transform.query(DSL.using(ds, dialect)); + return transform.query(dslContext); } public T transaction(final ContextQueryFunction transform) throws SQLException { - return DSL.using(ds, dialect).transactionResult(configuration -> transform.query(DSL.using(configuration))); + return dslContext.transactionResult(configuration -> transform.query(DSL.using(configuration))); } - public DataSource getDataSource() { - return ds; + public static Database createWithRetry(final DSLContext dslContext, + final Function isDbReady) { + Database database = null; + while (database == null) { + try { + val infinity = Integer.MAX_VALUE; + database = createWithRetryTimeout(dslContext, isDbReady, infinity); + } catch (final IOException e) { + // This should theoretically never happen since we set the timeout to be a very high number. + } + } + + LOGGER.info("Database available!"); + return database; } - @Override - public void close() throws Exception { - // Just a safety in case we are using a datasource implementation that requires closing. - // BasicDataSource from apache does since it also provides a pooling mechanism to reuse connections. + public static Database createWithRetryTimeout(final DSLContext dslContext, + final Function isDbReady, + final long timeoutMs) + throws IOException { + Database database = null; + var totalTime = 0; + while (database == null) { + LOGGER.warn("Waiting for database to become available..."); + if (totalTime >= timeoutMs) { + throw new IOException("Unable to connect to database."); + } - if (ds instanceof AutoCloseable) { - ((AutoCloseable) ds).close(); - } - if (ds instanceof Closeable) { - ((Closeable) ds).close(); + try { + database = new Database(dslContext); + if (!isDbReady.apply(database)) { + LOGGER.info("Database is not ready yet. Please wait a moment, it might still be initializing..."); + database = null; + Exceptions.toRuntime(() -> Thread.sleep(DEFAULT_WAIT_MS)); + totalTime += DEFAULT_WAIT_MS; + } + } catch (final Exception e) { + // Ignore the exception because this likely means that the database server is still initializing. + LOGGER.warn("Ignoring exception while trying to request database:", e); + database = null; + Exceptions.toRuntime(() -> Thread.sleep(DEFAULT_WAIT_MS)); + totalTime += DEFAULT_WAIT_MS; + } } + return database; } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java index 2df0b624f07..63a04ce7f88 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java @@ -7,6 +7,9 @@ package io.airbyte.db; import com.google.common.collect.Maps; import io.airbyte.commons.lang.Exceptions; import io.airbyte.db.bigquery.BigQueryDatabase; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcSourceOperations; @@ -18,8 +21,8 @@ import java.io.IOException; import java.util.Map; import java.util.function.Function; import java.util.function.Supplier; +import javax.sql.DataSource; import lombok.val; -import org.apache.commons.dbcp2.BasicDataSource; import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,7 +52,7 @@ public class Databases { } public static Database createPostgresDatabase(final String username, final String password, final String jdbcConnectionString) { - return createDatabase(username, password, jdbcConnectionString, "org.postgresql.Driver", SQLDialect.POSTGRES); + return createDatabase(username, password, jdbcConnectionString, DatabaseDriver.POSTGRESQL.getDriverClassName(), SQLDialect.POSTGRES); } public static Database createPostgresDatabaseWithRetry(final String username, @@ -93,8 +96,6 @@ public class Databases { database = createPostgresDatabase(username, password, jdbcConnectionString); if (!isDbReady.apply(database)) { LOGGER.info("Database is not ready yet. Please wait a moment, it might still be initializing..."); - database.close(); - database = null; Exceptions.toRuntime(() -> Thread.sleep(DEFAULT_WAIT_MS)); totalTime += DEFAULT_WAIT_MS; @@ -141,9 +142,7 @@ public class Databases { final String jdbcConnectionString, final String driverClassName, final SQLDialect dialect) { - final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName); - - return new Database(connectionPool, dialect); + return new Database(DSLContextFactory.create(username, password, driverClassName, jdbcConnectionString, dialect)); } public static Database createDatabase(final String username, @@ -152,10 +151,7 @@ public class Databases { final String driverClassName, final SQLDialect dialect, final Map connectionProperties) { - final BasicDataSource connectionPool = - createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); - - return new Database(connectionPool, dialect); + return new Database(DSLContextFactory.create(username, password, driverClassName, jdbcConnectionString, dialect, connectionProperties)); } public static JdbcDatabase createJdbcDatabase(final String username, @@ -170,7 +166,7 @@ public class Databases { final String jdbcConnectionString, final String driverClassName, final JdbcSourceOperations sourceOperations) { - final BasicDataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName); + final DataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName); return new DefaultJdbcDatabase(connectionPool, sourceOperations); } @@ -190,7 +186,7 @@ public class Databases { final String driverClassName, final Map connectionProperties, final JdbcCompatibleSourceOperations sourceOperations) { - final BasicDataSource connectionPool = + final DataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new DefaultJdbcDatabase(connectionPool, sourceOperations); @@ -203,34 +199,26 @@ public class Databases { final Supplier streamingQueryConfigProvider, final Map connectionProperties, final JdbcCompatibleSourceOperations sourceOperations) { - final BasicDataSource connectionPool = + final DataSource connectionPool = createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties); return new StreamingJdbcDatabase(connectionPool, sourceOperations, streamingQueryConfigProvider); } - private static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName) { + private static DataSource createBasicDataSource(final String username, + final String password, + final String jdbcConnectionString, + final String driverClassName) { return createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Maps.newHashMap()); } - public static BasicDataSource createBasicDataSource(final String username, - final String password, - final String jdbcConnectionString, - final String driverClassName, - final Map connectionProperties) { - final BasicDataSource connectionPool = new BasicDataSource(); - connectionPool.setDriverClassName(driverClassName); - connectionPool.setUsername(username); - connectionPool.setPassword(password); - connectionPool.setInitialSize(0); - connectionPool.setMaxTotal(5); - connectionPool.setUrl(jdbcConnectionString); - connectionProperties.forEach(connectionPool::addConnectionProperty); - return connectionPool; + public static DataSource createBasicDataSource(final String username, + final String password, + final String jdbcConnectionString, + final String driverClassName, + final Map connectionProperties) { + return DataSourceFactory.create(username, password, driverClassName, jdbcConnectionString, connectionProperties); } public static BigQueryDatabase createBigQueryDatabase(final String projectId, final String jsonCreds) { diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/ExceptionWrappingDatabase.java b/airbyte-db/lib/src/main/java/io/airbyte/db/ExceptionWrappingDatabase.java index 2b3466073f0..02f6188a552 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/ExceptionWrappingDatabase.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/ExceptionWrappingDatabase.java @@ -10,7 +10,7 @@ import java.sql.SQLException; /** * Wraps a {@link Database} object and throwing IOExceptions instead of SQLExceptions. */ -public class ExceptionWrappingDatabase implements AutoCloseable { +public class ExceptionWrappingDatabase { private final Database database; @@ -34,9 +34,4 @@ public class ExceptionWrappingDatabase implements AutoCloseable { } } - @Override - public void close() throws Exception { - database.close(); - } - } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/TempBigQueryJoolDatabaseImpl.java b/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/TempBigQueryJoolDatabaseImpl.java index ec010064107..93d5472172c 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/TempBigQueryJoolDatabaseImpl.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/TempBigQueryJoolDatabaseImpl.java @@ -6,8 +6,8 @@ package io.airbyte.db.bigquery; import io.airbyte.db.ContextQueryFunction; import io.airbyte.db.Database; -import io.airbyte.db.Databases; import java.sql.SQLException; +import javax.annotation.Nullable; import org.jooq.Record; import org.jooq.Result; import org.jooq.SQLDialect; @@ -22,8 +22,8 @@ public class TempBigQueryJoolDatabaseImpl extends Database { private final BigQueryDatabase realDatabase; public TempBigQueryJoolDatabaseImpl(final String projectId, final String jsonCreds) { - super(null, null); - realDatabase = Databases.createBigQueryDatabase(projectId, jsonCreds); + super(null); + realDatabase = createBigQueryDatabase(projectId, jsonCreds); } @Override @@ -36,11 +36,6 @@ public class TempBigQueryJoolDatabaseImpl extends Database { return transform.query(new FakeDefaultDSLContext(realDatabase)); } - @Override - public void close() throws Exception { - realDatabase.close(); - } - public BigQueryDatabase getRealDatabase() { return realDatabase; } @@ -55,6 +50,7 @@ public class TempBigQueryJoolDatabaseImpl extends Database { } @Override + @Nullable public Result fetch(final String sql) throws DataAccessException { try { database.execute(sql); @@ -66,4 +62,8 @@ public class TempBigQueryJoolDatabaseImpl extends Database { } + public static BigQueryDatabase createBigQueryDatabase(final String projectId, final String jsonCreds) { + return new BigQueryDatabase(projectId, jsonCreds); + } + } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java index eba32e7cb62..929607cd4b9 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java @@ -4,6 +4,7 @@ package io.airbyte.db.factory; +import java.util.Map; import javax.sql.DataSource; import org.jooq.DSLContext; import org.jooq.SQLDialect; @@ -29,4 +30,42 @@ public class DSLContextFactory { return DSL.using(dataSource, dialect); } + /** + * Constructs a configured {@link DSLContext} instance using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @param dialect The SQL dialect to use with objects created from this context. + * @return The configured {@link DSLContext}. + */ + public static DSLContext create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString, + final SQLDialect dialect) { + return DSL.using(DataSourceFactory.create(username, password, driverClassName, jdbcConnectionString), dialect); + } + + /** + * Constructs a configured {@link DSLContext} instance using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @param dialect The SQL dialect to use with objects created from this context. + * @param connectionProperties Additional configuration properties for the underlying driver. + * @return The configured {@link DSLContext}. + */ + public static DSLContext create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString, + final SQLDialect dialect, + final Map connectionProperties) { + return DSL.using(DataSourceFactory.create(username, password, driverClassName, jdbcConnectionString, connectionProperties), dialect); + } + } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java index 31c659a548c..1a861fef8d1 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java @@ -4,6 +4,7 @@ package io.airbyte.db.factory; +import com.google.common.base.Preconditions; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.util.Map; @@ -151,17 +152,9 @@ public class DataSourceFactory { */ private static class DataSourceBuilder { - private static final Map JDBC_URL_FORMATS = Map.of("org.postgresql.Driver", "jdbc:postgresql://%s:%d/%s", - "com.amazon.redshift.jdbc.Driver", "jdbc:redshift://%s:%d/%s", - "com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%d/%s", - "com.microsoft.sqlserver.jdbc.SQLServerDriver", "jdbc:sqlserver://%s:%d/%s", - "oracle.jdbc.OracleDriver", "jdbc:oracle:thin:@%s:%d:%s", - "ru.yandex.clickhouse.ClickHouseDriver", "jdbc:ch://%s:%d/%s", - "org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"); - private Map connectionProperties = Map.of(); private String database; - private String driverClassName = "org.postgresql.Driver"; + private String driverClassName; private String host; private String jdbcUrl; private Integer maximumPoolSize = 5; @@ -231,19 +224,29 @@ public class DataSourceFactory { } public DataSource build() { + final DatabaseDriver databaseDriver = DatabaseDriver.findByDriverClassName(driverClassName); + + Preconditions.checkNotNull(databaseDriver, "Unknown or blank driver class name: '" + driverClassName + "'."); + final HikariConfig config = new HikariConfig(); - config.setDriverClassName(driverClassName); - config.setJdbcUrl(jdbcUrl != null ? jdbcUrl : String.format(JDBC_URL_FORMATS.getOrDefault(driverClassName, ""), host, port, database)); + + config.setDriverClassName(databaseDriver.getDriverClassName()); + config.setJdbcUrl(jdbcUrl != null ? jdbcUrl : String.format(databaseDriver.getUrlFormatString(), host, port, database)); config.setMaximumPoolSize(maximumPoolSize); config.setMinimumIdle(minimumPoolSize); config.setPassword(password); config.setUsername(username); + /* + * Disable to prevent failing on startup. Applications may start prior to the database container + * being available. To avoid failing to create the connection pool, disable the fail check. This + * will preserve existing behavior that tests for the connection on first use, not on creation. + */ + config.setInitializationFailTimeout(Integer.MIN_VALUE); + connectionProperties.forEach(config::addDataSourceProperty); - final HikariDataSource dataSource = new HikariDataSource(config); - dataSource.validate(); - return dataSource; + return new HikariDataSource(config); } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java new file mode 100644 index 00000000000..93c953cf62b --- /dev/null +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +/** + * Collection of JDBC driver class names and the associated JDBC URL format string. + */ +public enum DatabaseDriver { + + CLICKHOUSE("ru.yandex.clickhouse.ClickHouseDriver", "jdbc:clickhouse://%s:%d/%s"), + DB2("com.ibm.db2.jcc.DB2Driver", "jdbc:db2://%s:%d/%s"), + MARIADB("org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"), + MSSQLSERVER("com.microsoft.sqlserver.jdbc.SQLServerDriver", "jdbc:sqlserver://%s:%d/%s"), + MYSQL("com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%d/%s"), + ORACLE("oracle.jdbc.OracleDriver", "jdbc:oracle:thin:@%s:%d/%s"), + POSTGRESQL("org.postgresql.Driver", "jdbc:postgresql://%s:%d/%s"), + REDSHIFT("com.amazon.redshift.jdbc.Driver", "jdbc:redshift://%s:%d/%s"), + SNOWFLAKE("net.snowflake.client.jdbc.SnowflakeDriver", "jdbc:snowflake://%s/"); + + private final String driverClassName; + private final String urlFormatString; + + DatabaseDriver(final String driverClassName, final String urlFormatString) { + this.driverClassName = driverClassName; + this.urlFormatString = urlFormatString; + } + + public String getDriverClassName() { + return driverClassName; + } + + public String getUrlFormatString() { + return urlFormatString; + } + + /** + * Finds the {@link DatabaseDriver} enumerated value that matches the provided driver class name. + * + * @param driverClassName The driver class name. + * @return The matching {@link DatabaseDriver} enumerated value or {@code null} if no match is + * found. + */ + public static DatabaseDriver findByDriverClassName(final String driverClassName) { + DatabaseDriver selected = null; + + for (final DatabaseDriver candidate : values()) { + if (candidate.getDriverClassName().equalsIgnoreCase(driverClassName)) { + selected = candidate; + break; + } + } + + return selected; + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java index 0e5526745fd..2e8bbcfdd12 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/factory/FlywayFactory.java @@ -38,16 +38,46 @@ public class FlywayFactory { final String installedBy, final String dbIdentifier, final String... migrationFileLocations) { + return create(dataSource, + installedBy, + dbIdentifier, + BASELINE_VERSION, + BASELINE_DESCRIPTION, + BASELINE_ON_MIGRATION, + migrationFileLocations); + } + + /** + * Constructs a configured {@link Flyway} instance using the provided configuration. + * + * @param dataSource The {@link DataSource} used to connect to the database. + * @param installedBy The name of the module performing the migration. + * @param dbIdentifier The name of the database to be migrated. This is used to name the table to + * hold the migration history for the database. + * @param baselineVersion The version to tag an existing schema with when executing baseline. + * @param baselineDescription The description to tag an existing schema with when executing + * baseline. + * @param baselineOnMigrate Whether to automatically call baseline when migrate is executed against + * a non-empty schema with no schema history table. + * @param migrationFileLocations The array of migration files to be used. + * @return The configured {@link Flyway} instance. + */ + public static Flyway create(final DataSource dataSource, + final String installedBy, + final String dbIdentifier, + final String baselineVersion, + final String baselineDescription, + final boolean baselineOnMigrate, + final String... migrationFileLocations) { return Flyway.configure() .dataSource(dataSource) - .baselineVersion(BASELINE_VERSION) - .baselineDescription(BASELINE_DESCRIPTION) - .baselineOnMigrate(BASELINE_ON_MIGRATION) + .baselineVersion(baselineVersion) + .baselineDescription(baselineDescription) + .baselineOnMigrate(baselineOnMigrate) .installedBy(installedBy) .table(String.format(MIGRATION_TABLE_FORMAT, dbIdentifier)) .locations(migrationFileLocations) .load(); - } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/BaseDatabaseInstance.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/BaseDatabaseInstance.java index 3cc2c21333a..b7ba44fbe5a 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/BaseDatabaseInstance.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/BaseDatabaseInstance.java @@ -7,7 +7,6 @@ package io.airbyte.db.instance; import static org.jooq.impl.DSL.select; import io.airbyte.db.Database; -import io.airbyte.db.Databases; import io.airbyte.db.ExceptionWrappingDatabase; import java.io.IOException; import java.util.Set; @@ -24,45 +23,37 @@ public abstract class BaseDatabaseInstance implements DatabaseInstance { private static final Logger LOGGER = LoggerFactory.getLogger(BaseDatabaseInstance.class); - protected final String username; - protected final String password; - protected final String connectionString; - protected final String initialSchema; + protected final DSLContext dslContext; protected final String databaseName; protected final Set initialExpectedTables; + protected final String initialSchema; protected final Function isDatabaseReady; /** - * @param connectionString in the format of - * jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} + * @param dslContext The configured {@link DSLContext}. * @param databaseName this name is only for logging purpose; it may not be the actual database name * in the server * @param initialSchema the initial database structure. + * @param initialExpectedTables The set of tables that should be present in order to consider the + * database ready for use. * @param isDatabaseReady a function to check if the database has been initialized and ready for * consumption */ - protected BaseDatabaseInstance(final String username, - final String password, - final String connectionString, - final String initialSchema, + protected BaseDatabaseInstance(final DSLContext dslContext, final String databaseName, + final String initialSchema, final Set initialExpectedTables, final Function isDatabaseReady) { - this.username = username; - this.password = password; - this.connectionString = connectionString; - this.initialSchema = initialSchema; + this.dslContext = dslContext; this.databaseName = databaseName; + this.initialSchema = initialSchema; this.initialExpectedTables = initialExpectedTables; this.isDatabaseReady = isDatabaseReady; } @Override public boolean isInitialized() throws IOException { - final Database database = Databases.createPostgresDatabaseWithRetryTimeout( - username, - password, - connectionString, + final Database database = Database.createWithRetryTimeout(dslContext, isDatabaseConnected(databaseName), DEFAULT_CONNECTION_TIMEOUT_MS); return new ExceptionWrappingDatabase(database).transaction(ctx -> initialExpectedTables.stream().allMatch(tableName -> hasTable(ctx, tableName))); @@ -72,11 +63,7 @@ public abstract class BaseDatabaseInstance implements DatabaseInstance { public Database getInitialized() { // When we don't need to setup the database, it means the database is initialized // somewhere else, and it is considered ready only when data has been loaded into it. - return Databases.createPostgresDatabaseWithRetry( - username, - password, - connectionString, - isDatabaseReady); + return Database.createWithRetry(dslContext, isDatabaseReady); } @Override @@ -84,11 +71,7 @@ public abstract class BaseDatabaseInstance implements DatabaseInstance { // When we need to setup the database, it means the database will be initialized after // we connect to the database. So the database itself is considered ready as long as // the connection is alive. - final Database database = Databases.createPostgresDatabaseWithRetry( - username, - password, - connectionString, - isDatabaseConnected(databaseName)); + final Database database = Database.createWithRetry(dslContext, isDatabaseConnected(databaseName)); new ExceptionWrappingDatabase(database).transaction(ctx -> { final boolean hasTables = initialExpectedTables.stream().allMatch(tableName -> hasTable(ctx, tableName)); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java index 3ee251f2bd4..3890a414246 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayDatabaseMigrator.java @@ -14,7 +14,6 @@ import java.util.stream.Collectors; import org.flywaydb.core.Flyway; import org.flywaydb.core.api.MigrationInfo; import org.flywaydb.core.api.MigrationInfoService; -import org.flywaydb.core.api.configuration.FluentConfiguration; import org.flywaydb.core.api.output.BaselineResult; import org.flywaydb.core.api.output.MigrateResult; import org.slf4j.Logger; @@ -23,49 +22,15 @@ import org.slf4j.LoggerFactory; public class FlywayDatabaseMigrator implements DatabaseMigrator { private static final Logger LOGGER = LoggerFactory.getLogger(FlywayDatabaseMigrator.class); - // Constants for Flyway baseline. See here for details: - // https://flywaydb.org/documentation/command/baseline - private static final String BASELINE_VERSION = "0.29.0.001"; - private static final String BASELINE_DESCRIPTION = "Baseline from file-based migration v1"; - private static final boolean BASELINE_ON_MIGRATION = true; private final Database database; private final Flyway flyway; - /** - * @param dbIdentifier A name to identify the database. Preferably one word. This identifier will be - * used to construct the migration history table name. For example, if the identifier is - * "imports", the history table name will be "airbyte_imports_migrations". - * @param migrationFileLocations Example: "classpath:db/migration". See: - * https://flywaydb.org/documentation/concepts/migrations#discovery-1 - */ - protected FlywayDatabaseMigrator(final Database database, - final String dbIdentifier, - final String migrationRunner, - final String migrationFileLocations) { - this(database, getConfiguration(database, dbIdentifier, migrationRunner, migrationFileLocations).load()); - } - - @VisibleForTesting public FlywayDatabaseMigrator(final Database database, final Flyway flyway) { this.database = database; this.flyway = flyway; } - private static FluentConfiguration getConfiguration(final Database database, - final String dbIdentifier, - final String migrationRunner, - final String migrationFileLocations) { - return Flyway.configure() - .dataSource(database.getDataSource()) - .baselineVersion(BASELINE_VERSION) - .baselineDescription(BASELINE_DESCRIPTION) - .baselineOnMigrate(BASELINE_ON_MIGRATION) - .installedBy(migrationRunner) - .table(String.format("airbyte_%s_migrations", dbIdentifier)) - .locations(migrationFileLocations); - } - @Override public MigrateResult migrate() { final MigrateResult result = flyway.migrate(); diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java index 445d9fcbb14..11d529eff25 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/FlywayMigrationDatabase.java @@ -5,8 +5,13 @@ package io.airbyte.db.instance; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.FlywayFactory; import java.io.IOException; import java.sql.Connection; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.jooq.impl.DSL; @@ -32,9 +37,15 @@ public abstract class FlywayMigrationDatabase extends PostgresDatabase { private Connection connection; - protected abstract Database getAndInitializeDatabase(String username, String password, String connectionString) throws IOException; + protected abstract Database getAndInitializeDatabase(DSLContext dslContext) throws IOException; - protected abstract DatabaseMigrator getDatabaseMigrator(Database database); + protected abstract DatabaseMigrator getDatabaseMigrator(Database database, Flyway flyway); + + protected abstract String getInstalledBy(); + + protected abstract String getDbIdentifier(); + + protected abstract String[] getMigrationFileLocations(); @Override protected DSLContext create0() { @@ -64,11 +75,15 @@ public abstract class FlywayMigrationDatabase extends PostgresDatabase { .withPassword("jooq_generator"); container.start(); - final Database database = getAndInitializeDatabase(container.getUsername(), container.getPassword(), container.getJdbcUrl()); - final DatabaseMigrator migrator = getDatabaseMigrator(database); + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), container.getDriverClassName(), container.getJdbcUrl()); + final DSLContext dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + final Flyway flyway = FlywayFactory.create(dataSource, getInstalledBy(), getDbIdentifier(), getMigrationFileLocations()); + final Database database = getAndInitializeDatabase(dslContext); + final DatabaseMigrator migrator = getDatabaseMigrator(database, flyway); migrator.migrate(); - connection = database.getDataSource().getConnection(); + connection = dataSource.getConnection(); setConnection(connection); } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java index 7ecee82492b..e1ff55f2935 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseInstance.java @@ -6,7 +6,6 @@ package io.airbyte.db.instance.configs; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; -import io.airbyte.db.Databases; import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.BaseDatabaseInstance; import io.airbyte.db.instance.DatabaseInstance; @@ -14,6 +13,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Set; import java.util.function.Function; +import org.jooq.DSLContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,19 +38,15 @@ public class ConfigsDatabaseInstance extends BaseDatabaseInstance implements Dat private Database database; - public ConfigsDatabaseInstance(final String username, final String password, final String connectionString) throws IOException { - super(username, password, connectionString, MoreResources.readResource(SCHEMA_PATH), DATABASE_LOGGING_NAME, INITIAL_EXPECTED_TABLES, + public ConfigsDatabaseInstance(final DSLContext dslContext) throws IOException { + super(dslContext, DATABASE_LOGGING_NAME, MoreResources.readResource(SCHEMA_PATH), INITIAL_EXPECTED_TABLES, IS_CONFIGS_DATABASE_READY); } @Override public boolean isInitialized() throws IOException { if (database == null) { - database = Databases.createPostgresDatabaseWithRetry( - username, - password, - connectionString, - isDatabaseConnected(databaseName)); + database = Database.createWithRetry(dslContext, isDatabaseConnected(databaseName)); } return new ExceptionWrappingDatabase(database).transaction(ctx -> { @@ -73,11 +69,7 @@ public class ConfigsDatabaseInstance extends BaseDatabaseInstance implements Dat // we connect to the database. So the database itself is considered ready as long as // the connection is alive. if (database == null) { - database = Databases.createPostgresDatabaseWithRetry( - username, - password, - connectionString, - isDatabaseConnected(databaseName)); + database = Database.createWithRetry(dslContext, isDatabaseConnected(databaseName)); } new ExceptionWrappingDatabase(database).transaction(ctx -> { diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrationDevCenter.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrationDevCenter.java index 2c52bf7dcea..8fc570edbdd 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrationDevCenter.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrationDevCenter.java @@ -5,10 +5,13 @@ package io.airbyte.db.instance.configs; import io.airbyte.db.Database; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.FlywayDatabaseMigrator; import io.airbyte.db.instance.development.MigrationDevCenter; import java.io.IOException; -import org.testcontainers.containers.PostgreSQLContainer; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; /** * Helper class for migration development. See README for details. @@ -20,13 +23,19 @@ public class ConfigsDatabaseMigrationDevCenter extends MigrationDevCenter { } @Override - protected FlywayDatabaseMigrator getMigrator(final Database database) { - return new ConfigsDatabaseMigrator(database, ConfigsDatabaseMigrationDevCenter.class.getSimpleName()); + protected FlywayDatabaseMigrator getMigrator(final Database database, final Flyway flyway) { + return new ConfigsDatabaseMigrator(database, flyway); } @Override - protected Database getDatabase(final PostgreSQLContainer container) throws IOException { - return new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + protected Database getDatabase(final DSLContext dslContext) throws IOException { + return new ConfigsDatabaseInstance(dslContext).getAndInitialize(); + } + + @Override + protected Flyway getFlyway(final DataSource dataSource) { + return FlywayFactory.create(dataSource, getClass().getSimpleName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrator.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrator.java index f08199ec3d1..213758270ca 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrator.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigrator.java @@ -6,14 +6,15 @@ package io.airbyte.db.instance.configs; import io.airbyte.db.Database; import io.airbyte.db.instance.FlywayDatabaseMigrator; +import org.flywaydb.core.Flyway; public class ConfigsDatabaseMigrator extends FlywayDatabaseMigrator { public static final String DB_IDENTIFIER = "configs"; public static final String MIGRATION_FILE_LOCATION = "classpath:io/airbyte/db/instance/configs/migrations"; - public ConfigsDatabaseMigrator(final Database database, final String migrationRunner) { - super(database, DB_IDENTIFIER, migrationRunner, MIGRATION_FILE_LOCATION); + public ConfigsDatabaseMigrator(final Database database, final Flyway flyway) { + super(database, flyway); } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java index df2b145ff35..4293e815cb2 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java @@ -15,29 +15,28 @@ import io.airbyte.db.instance.test.TestDatabaseProvider; import java.io.IOException; import java.time.OffsetDateTime; import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; import org.jooq.JSONB; public class ConfigsDatabaseTestProvider implements TestDatabaseProvider { - private final String user; - private final String password; - private final String jdbcUrl; + private final DSLContext dslContext; + private final Flyway flyway; - public ConfigsDatabaseTestProvider(final String user, final String password, final String jdbcUrl) { - this.user = user; - this.password = password; - this.jdbcUrl = jdbcUrl; + public ConfigsDatabaseTestProvider(final DSLContext dslContext, final Flyway flyway) { + this.dslContext = dslContext; + this.flyway = flyway; } @Override public Database create(final boolean runMigration) throws IOException { - final Database database = new ConfigsDatabaseInstance(user, password, jdbcUrl) + final Database database = new ConfigsDatabaseInstance(dslContext) .getAndInitialize(); if (runMigration) { final DatabaseMigrator migrator = new ConfigsDatabaseMigrator( - database, - ConfigsDatabaseTestProvider.class.getSimpleName()); + database, flyway); migrator.createBaseline(); migrator.migrate(); } else { diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java index e9662e75748..2550e1a2ec7 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/ConfigsFlywayMigrationDatabase.java @@ -8,6 +8,8 @@ import io.airbyte.db.Database; import io.airbyte.db.instance.DatabaseMigrator; import io.airbyte.db.instance.FlywayMigrationDatabase; import java.io.IOException; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; /** * Configs database for jOOQ code generation. @@ -15,13 +17,28 @@ import java.io.IOException; public class ConfigsFlywayMigrationDatabase extends FlywayMigrationDatabase { @Override - protected Database getAndInitializeDatabase(final String username, final String password, final String connectionString) throws IOException { - return new ConfigsDatabaseInstance(username, password, connectionString).getAndInitialize(); + protected Database getAndInitializeDatabase(final DSLContext dslContext) throws IOException { + return new ConfigsDatabaseInstance(dslContext).getAndInitialize(); } @Override - protected DatabaseMigrator getDatabaseMigrator(final Database database) { - return new ConfigsDatabaseMigrator(database, ConfigsFlywayMigrationDatabase.class.getSimpleName()); + protected DatabaseMigrator getDatabaseMigrator(final Database database, final Flyway flyway) { + return new ConfigsDatabaseMigrator(database, flyway); + } + + @Override + protected String getInstalledBy() { + return ConfigsFlywayMigrationDatabase.class.getSimpleName(); + } + + @Override + protected String getDbIdentifier() { + return ConfigsDatabaseMigrator.DB_IDENTIFIER; + } + + @Override + protected String[] getMigrationFileLocations() { + return new String[] {ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION}; } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java index ce1e8f1debe..2844b4b0c26 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java @@ -13,6 +13,8 @@ import io.airbyte.config.EnvConfigs; import io.airbyte.config.StandardSyncState; import io.airbyte.config.State; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.instance.jobs.JobsDatabaseInstance; import java.io.IOException; import java.sql.SQLException; @@ -26,6 +28,7 @@ import org.flywaydb.core.api.migration.Context; import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.JSONB; +import org.jooq.SQLDialect; import org.jooq.Table; import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; @@ -115,7 +118,9 @@ public class V0_30_22_001__Store_last_sync_state extends BaseJavaMigration { } // If the environment variables exist, it means the migration is run in production. // Connect to the official job database. - final Database jobsDatabase = new JobsDatabaseInstance(databaseUser, databasePassword, databaseUrl).getInitialized(); + final DSLContext dslContext = + DSLContextFactory.create(databaseUser, databasePassword, DatabaseDriver.POSTGRESQL.getDriverClassName(), databaseUrl, SQLDialect.POSTGRES); + final Database jobsDatabase = new JobsDatabaseInstance(dslContext).getInitialized(); LOGGER.info("[{}] Connected to jobs database: {}", MIGRATION_NAME, databaseUrl); return Optional.of(jobsDatabase); } catch (final IllegalArgumentException e) { diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java index 58f26c184d3..379ab7a56f0 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java @@ -5,10 +5,16 @@ package io.airbyte.db.instance.development; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.instance.FlywayDatabaseMigrator; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrationDevCenter; import io.airbyte.db.instance.jobs.JobsDatabaseMigrationDevCenter; import java.io.IOException; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; /** @@ -44,37 +50,57 @@ public abstract class MigrationDevCenter { return container; } - protected abstract FlywayDatabaseMigrator getMigrator(Database database); + protected abstract FlywayDatabaseMigrator getMigrator(Database database, Flyway flyway); - protected abstract Database getDatabase(PostgreSQLContainer container) throws IOException; + protected abstract Database getDatabase(DSLContext dslContext) throws IOException; + + protected abstract Flyway getFlyway(DataSource dataSource); private void createMigration() { - try (final PostgreSQLContainer container = createContainer(); final Database database = getDatabase(container)) { - final FlywayDatabaseMigrator migrator = getMigrator(database); - MigrationDevHelper.createNextMigrationFile(dbIdentifier, migrator); + try (final PostgreSQLContainer container = createContainer()) { + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), container.getDriverClassName(), container.getJdbcUrl()); + try (final DSLContext dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES)) { + final Flyway flyway = getFlyway(dataSource); + final Database database = getDatabase(dslContext); + final FlywayDatabaseMigrator migrator = getMigrator(database, flyway); + MigrationDevHelper.createNextMigrationFile(dbIdentifier, migrator); + } } catch (final Exception e) { throw new RuntimeException(e); } } private void runLastMigration() { - try (final PostgreSQLContainer container = createContainer(); final Database database = getDatabase(container)) { - final FlywayDatabaseMigrator fullMigrator = getMigrator(database); - final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(fullMigrator); - MigrationDevHelper.runLastMigration(devDatabaseMigrator); - final String schema = fullMigrator.dumpSchema(); - MigrationDevHelper.dumpSchema(schema, schemaDumpFile, false); + try (final PostgreSQLContainer container = createContainer()) { + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), container.getDriverClassName(), container.getJdbcUrl()); + try (final DSLContext dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES)) { + final Flyway flyway = getFlyway(dataSource); + final Database database = getDatabase(dslContext); + final FlywayDatabaseMigrator fullMigrator = getMigrator(database, flyway); + final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(fullMigrator); + MigrationDevHelper.runLastMigration(devDatabaseMigrator); + final String schema = fullMigrator.dumpSchema(); + MigrationDevHelper.dumpSchema(schema, schemaDumpFile, false); + } } catch (final Exception e) { throw new RuntimeException(e); } } private void dumpSchema() { - try (final PostgreSQLContainer container = createContainer(); final Database database = getDatabase(container)) { - final FlywayDatabaseMigrator migrator = getMigrator(database); - migrator.migrate(); - final String schema = migrator.dumpSchema(); - MigrationDevHelper.dumpSchema(schema, schemaDumpFile, true); + try (final PostgreSQLContainer container = createContainer()) { + final DataSource dataSource = + DataSourceFactory.create(container.getUsername(), container.getPassword(), container.getDriverClassName(), container.getJdbcUrl()); + try (final DSLContext dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES)) { + final Flyway flyway = getFlyway(dataSource); + final Database database = getDatabase(dslContext); + final FlywayDatabaseMigrator migrator = getMigrator(database, flyway); + migrator.migrate(); + final String schema = migrator.dumpSchema(); + MigrationDevHelper.dumpSchema(schema, schemaDumpFile, true); + } } catch (final Exception e) { throw new RuntimeException(e); } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseInstance.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseInstance.java index 6464b18b7e0..8d378e07f84 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseInstance.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseInstance.java @@ -11,6 +11,7 @@ import io.airbyte.db.instance.BaseDatabaseInstance; import io.airbyte.db.instance.DatabaseInstance; import java.io.IOException; import java.util.function.Function; +import org.jooq.DSLContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,12 +31,12 @@ public class JobsDatabaseInstance extends BaseDatabaseInstance implements Databa }; @VisibleForTesting - public JobsDatabaseInstance(final String username, final String password, final String connectionString, final String schema) { - super(username, password, connectionString, schema, DATABASE_LOGGING_NAME, JobsDatabaseSchema.getTableNames(), IS_JOBS_DATABASE_READY); + public JobsDatabaseInstance(final DSLContext dslContext, final String schema) { + super(dslContext, DATABASE_LOGGING_NAME, schema, JobsDatabaseSchema.getTableNames(), IS_JOBS_DATABASE_READY); } - public JobsDatabaseInstance(final String username, final String password, final String connectionString) throws IOException { - this(username, password, connectionString, MoreResources.readResource(SCHEMA_PATH)); + public JobsDatabaseInstance(final DSLContext dslContext) throws IOException { + this(dslContext, MoreResources.readResource(SCHEMA_PATH)); } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrationDevCenter.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrationDevCenter.java index 733385f6b97..e73e0890685 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrationDevCenter.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrationDevCenter.java @@ -5,10 +5,14 @@ package io.airbyte.db.instance.jobs; import io.airbyte.db.Database; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.FlywayDatabaseMigrator; +import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.development.MigrationDevCenter; import java.io.IOException; -import org.testcontainers.containers.PostgreSQLContainer; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; /** * Helper class for migration development. See README for details. @@ -20,13 +24,19 @@ public class JobsDatabaseMigrationDevCenter extends MigrationDevCenter { } @Override - protected FlywayDatabaseMigrator getMigrator(final Database database) { - return new JobsDatabaseMigrator(database, JobsDatabaseMigrationDevCenter.class.getSimpleName()); + protected FlywayDatabaseMigrator getMigrator(final Database database, final Flyway flyway) { + return new JobsDatabaseMigrator(database, flyway); } @Override - protected Database getDatabase(final PostgreSQLContainer container) throws IOException { - return new JobsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + protected Database getDatabase(final DSLContext dslContext) throws IOException { + return new ConfigsDatabaseInstance(dslContext).getAndInitialize(); + } + + @Override + protected Flyway getFlyway(final DataSource dataSource) { + return FlywayFactory.create(dataSource, getClass().getSimpleName(), JobsDatabaseMigrator.DB_IDENTIFIER, + JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrator.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrator.java index 5bfb5aca524..3d3f9dc010f 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrator.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseMigrator.java @@ -6,14 +6,15 @@ package io.airbyte.db.instance.jobs; import io.airbyte.db.Database; import io.airbyte.db.instance.FlywayDatabaseMigrator; +import org.flywaydb.core.Flyway; public class JobsDatabaseMigrator extends FlywayDatabaseMigrator { public static final String DB_IDENTIFIER = "jobs"; public static final String MIGRATION_FILE_LOCATION = "classpath:io/airbyte/db/instance/jobs/migrations"; - public JobsDatabaseMigrator(final Database database, final String migrationRunner) { - super(database, DB_IDENTIFIER, migrationRunner, MIGRATION_FILE_LOCATION); + public JobsDatabaseMigrator(final Database database, final Flyway flyway) { + super(database, flyway); } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseTestProvider.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseTestProvider.java index 133b9c29080..850a020309a 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseTestProvider.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsDatabaseTestProvider.java @@ -8,28 +8,27 @@ import io.airbyte.db.Database; import io.airbyte.db.instance.DatabaseMigrator; import io.airbyte.db.instance.test.TestDatabaseProvider; import java.io.IOException; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; public class JobsDatabaseTestProvider implements TestDatabaseProvider { - private final String user; - private final String password; - private final String jdbcUrl; + private final DSLContext dslContext; + private final Flyway flyway; - public JobsDatabaseTestProvider(String user, String password, String jdbcUrl) { - this.user = user; - this.password = password; - this.jdbcUrl = jdbcUrl; + public JobsDatabaseTestProvider(final DSLContext dslContext, final Flyway flyway) { + this.dslContext = dslContext; + this.flyway = flyway; } @Override public Database create(final boolean runMigration) throws IOException { - final Database jobsDatabase = new JobsDatabaseInstance(user, password, jdbcUrl) + final Database jobsDatabase = new JobsDatabaseInstance(dslContext) .getAndInitialize(); if (runMigration) { final DatabaseMigrator migrator = new JobsDatabaseMigrator( - jobsDatabase, - JobsDatabaseTestProvider.class.getSimpleName()); + jobsDatabase, flyway); migrator.createBaseline(); migrator.migrate(); } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java index 6bcb85e0528..491fa3dc476 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/JobsFlywayMigrationDatabase.java @@ -8,6 +8,8 @@ import io.airbyte.db.Database; import io.airbyte.db.instance.DatabaseMigrator; import io.airbyte.db.instance.FlywayMigrationDatabase; import java.io.IOException; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; /** * Jobs database for jOOQ code generation. @@ -15,13 +17,28 @@ import java.io.IOException; public class JobsFlywayMigrationDatabase extends FlywayMigrationDatabase { @Override - protected Database getAndInitializeDatabase(final String username, final String password, final String connectionString) throws IOException { - return new JobsDatabaseInstance(username, password, connectionString).getAndInitialize(); + protected Database getAndInitializeDatabase(final DSLContext dslContext) throws IOException { + return new JobsDatabaseInstance(dslContext).getAndInitialize(); } @Override - protected DatabaseMigrator getDatabaseMigrator(final Database database) { - return new JobsDatabaseMigrator(database, JobsFlywayMigrationDatabase.class.getSimpleName()); + protected DatabaseMigrator getDatabaseMigrator(final Database database, final Flyway flyway) { + return new JobsDatabaseMigrator(database, flyway); + } + + @Override + protected String getInstalledBy() { + return JobsFlywayMigrationDatabase.class.getSimpleName(); + } + + @Override + protected String getDbIdentifier() { + return JobsDatabaseMigrator.DB_IDENTIFIER; + } + + @Override + protected String[] getMigrationFileLocations() { + return new String[] {JobsDatabaseMigrator.MIGRATION_FILE_LOCATION}; } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java index b15cc81db76..04681147bda 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/test/TestDatabaseProviders.java @@ -4,14 +4,16 @@ package io.airbyte.db.instance.test; -import com.google.api.client.util.Preconditions; -import io.airbyte.config.Configs; import io.airbyte.db.Database; +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; +import io.airbyte.db.instance.jobs.JobsDatabaseMigrator; import io.airbyte.db.instance.jobs.JobsDatabaseTestProvider; import java.io.IOException; -import java.util.Optional; -import org.testcontainers.containers.PostgreSQLContainer; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; /** * Use this class to create mock databases in unit tests. This class takes care of database @@ -20,13 +22,13 @@ import org.testcontainers.containers.PostgreSQLContainer; @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class TestDatabaseProviders { - private final Optional configs; - private final Optional> container; + private final DataSource dataSource; + private final DSLContext dslContext; private boolean runMigration = true; - public TestDatabaseProviders(final PostgreSQLContainer container) { - this.configs = Optional.empty(); - this.container = Optional.of(container); + public TestDatabaseProviders(final DataSource dataSource, final DSLContext dslContext) { + this.dataSource = dataSource; + this.dslContext = dslContext; } /** @@ -39,41 +41,17 @@ public class TestDatabaseProviders { } public Database createNewConfigsDatabase() throws IOException { - Preconditions.checkArgument(configs.isPresent() || container.isPresent()); - if (configs.isPresent()) { - final Configs c = configs.get(); - return new ConfigsDatabaseTestProvider( - c.getConfigDatabaseUser(), - c.getConfigDatabasePassword(), - c.getConfigDatabaseUrl()) - .create(runMigration); - } else { - final PostgreSQLContainer c = container.get(); - return new ConfigsDatabaseTestProvider( - c.getUsername(), - c.getPassword(), - c.getJdbcUrl()) - .create(runMigration); - } + final Flyway flyway = FlywayFactory.create(dataSource, ConfigsDatabaseTestProvider.class.getSimpleName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + return new ConfigsDatabaseTestProvider(dslContext, flyway) + .create(runMigration); } public Database createNewJobsDatabase() throws IOException { - Preconditions.checkArgument(configs.isPresent() || container.isPresent()); - if (configs.isPresent()) { - final Configs c = configs.get(); - return new JobsDatabaseTestProvider( - c.getDatabaseUser(), - c.getDatabasePassword(), - c.getDatabaseUrl()) - .create(runMigration); - } else { - final PostgreSQLContainer c = container.get(); - return new JobsDatabaseTestProvider( - c.getUsername(), - c.getPassword(), - c.getJdbcUrl()) - .create(runMigration); - } + final Flyway flyway = FlywayFactory.create(dataSource, JobsDatabaseTestProvider.class.getSimpleName(), JobsDatabaseMigrator.DB_IDENTIFIER, + JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); + return new JobsDatabaseTestProvider(dslContext, flyway) + .create(runMigration); } } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/PostgresUtilsTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/PostgresUtilsTest.java index a32262bd0d2..e5d4c1dac7f 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/PostgresUtilsTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/PostgresUtilsTest.java @@ -12,11 +12,13 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; -import org.apache.commons.dbcp2.BasicDataSource; +import javax.sql.DataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -27,7 +29,7 @@ class PostgresUtilsTest { private static PostgreSQLContainer PSQL_DB; - private BasicDataSource dataSource; + private DataSource dataSource; @BeforeAll static void init() { @@ -46,14 +48,14 @@ class PostgresUtilsTest { final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); - dataSource = new BasicDataSource(); - dataSource.setDriverClassName("org.postgresql.Driver"); - dataSource.setUsername(config.get("username").asText()); - dataSource.setPassword(config.get("password").asText()); - dataSource.setUrl(String.format("jdbc:postgresql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText())); + dataSource = DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText())); final JdbcDatabase defaultJdbcDatabase = new DefaultJdbcDatabase(dataSource); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java index b4bae85c24f..c447c2b3ca3 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java @@ -7,11 +7,11 @@ package io.airbyte.db.factory; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import java.util.Map; import javax.sql.DataSource; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; -import org.postgresql.Driver; /** * Test suite for the {@link DSLContextFactory} class. @@ -21,11 +21,39 @@ public class DSLContextFactoryTest extends AbstractFactoryTest { @Test void testCreatingADslContext() { final DataSource dataSource = - DataSourceFactory.create(container.getUsername(), container.getPassword(), Driver.class.getName(), container.getJdbcUrl()); + DataSourceFactory.create(container.getUsername(), container.getPassword(), container.getDriverClassName(), container.getJdbcUrl()); final SQLDialect dialect = SQLDialect.POSTGRES; final DSLContext dslContext = DSLContextFactory.create(dataSource, dialect); assertNotNull(dslContext); assertEquals(dialect, dslContext.configuration().dialect()); } + @Test + void testCreatingADslContextWithIndividualConfiguration() { + final SQLDialect dialect = SQLDialect.POSTGRES; + final DSLContext dslContext = DSLContextFactory.create( + container.getUsername(), + container.getPassword(), + container.getDriverClassName(), + container.getJdbcUrl(), + dialect); + assertNotNull(dslContext); + assertEquals(dialect, dslContext.configuration().dialect()); + } + + @Test + void testCreatingADslContextWithIndividualConfigurationAndConnectionProperties() { + final Map connectionProperties = Map.of("foo", "bar"); + final SQLDialect dialect = SQLDialect.POSTGRES; + final DSLContext dslContext = DSLContextFactory.create( + container.getUsername(), + container.getPassword(), + container.getDriverClassName(), + container.getJdbcUrl(), + dialect, + connectionProperties); + assertNotNull(dslContext); + assertEquals(dialect, dslContext.configuration().dialect()); + } + } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java index 4cfe7cc1412..15fd845934e 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java @@ -12,7 +12,6 @@ import com.zaxxer.hikari.HikariDataSource; import java.util.Map; import javax.sql.DataSource; import org.junit.jupiter.api.Test; -import org.postgresql.Driver; /** * Test suite for the {@link DataSourceFactory} class. @@ -23,32 +22,34 @@ public class DataSourceFactoryTest extends AbstractFactoryTest { void testCreatingADataSourceWithJdbcUrl() { final String username = container.getUsername(); final String password = container.getPassword(); - final String driverClassName = Driver.class.getName(); + final String driverClassName = container.getDriverClassName(); final String jdbcUrl = container.getJdbcUrl(); final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); } @Test void testCreatingADataSourceWithJdbcUrlAndConnectionProperties() { final String username = container.getUsername(); final String password = container.getPassword(); - final String driverClassName = Driver.class.getName(); + final String driverClassName = container.getDriverClassName(); final String jdbcUrl = container.getJdbcUrl(); final Map connectionProperties = Map.of("foo", "bar"); final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl, connectionProperties); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); } @Test void testCreatingADataSourceWithHostAndPort() { final String username = container.getUsername(); final String password = container.getPassword(); - final String driverClassName = Driver.class.getName(); + final String driverClassName = container.getDriverClassName(); final String host = container.getHost(); final Integer port = container.getFirstMappedPort(); final String database = container.getDatabaseName(); @@ -56,13 +57,14 @@ public class DataSourceFactoryTest extends AbstractFactoryTest { final DataSource dataSource = DataSourceFactory.create(username, password, host, port, database, driverClassName); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); } @Test void testCreatingADataSourceWithHostPortAndConnectionProperties() { final String username = container.getUsername(); final String password = container.getPassword(); - final String driverClassName = Driver.class.getName(); + final String driverClassName = container.getDriverClassName(); final String host = container.getHost(); final Integer port = container.getFirstMappedPort(); final String database = container.getDatabaseName(); @@ -71,6 +73,7 @@ public class DataSourceFactoryTest extends AbstractFactoryTest { final DataSource dataSource = DataSourceFactory.create(username, password, host, port, database, driverClassName, connectionProperties); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); } @Test @@ -98,6 +101,7 @@ public class DataSourceFactoryTest extends AbstractFactoryTest { final DataSource dataSource = DataSourceFactory.createPostgres(username, password, host, port, database); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); } } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java index 2c2913261b2..8782605dbb0 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java @@ -8,10 +8,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.airbyte.test.utils.DatabaseConnectionHelper; import javax.sql.DataSource; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.Test; -import org.postgresql.Driver; /** * Test suite for the {@link FlywayFactory} class. @@ -22,15 +22,35 @@ public class FlywayFactoryTest extends AbstractFactoryTest { void testCreatingAFlywayInstance() { final String installedBy = "test"; final String dbIdentifier = "test"; + final String baselineVersion = "1.2.3"; + final String baselineDescription = "A test baseline description"; + final boolean baselineOnMigrate = true; final String migrationFileLocation = "classpath:io/airbyte/db/instance/toys/migrations"; - final DataSource dataSource = - DataSourceFactory.create(container.getUsername(), container.getPassword(), Driver.class.getName(), container.getJdbcUrl()); + final DataSource dataSource = DatabaseConnectionHelper.createDataSource(container); + final Flyway flyway = + FlywayFactory.create(dataSource, installedBy, dbIdentifier, baselineVersion, baselineDescription, baselineOnMigrate, migrationFileLocation); + assertNotNull(flyway); + assertTrue(flyway.getConfiguration().isBaselineOnMigrate()); + assertEquals(baselineDescription, flyway.getConfiguration().getBaselineDescription()); + assertEquals(baselineVersion, flyway.getConfiguration().getBaselineVersion().getVersion()); + assertEquals(baselineOnMigrate, flyway.getConfiguration().isBaselineOnMigrate()); + assertEquals(installedBy, flyway.getConfiguration().getInstalledBy()); + assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, dbIdentifier), flyway.getConfiguration().getTable()); + assertEquals(migrationFileLocation, flyway.getConfiguration().getLocations()[0].getDescriptor()); + } + @Test + void testCreatingAFlywayInstanceWithDefaults() { + final String installedBy = "test"; + final String dbIdentifier = "test"; + final String migrationFileLocation = "classpath:io/airbyte/db/instance/toys/migrations"; + final DataSource dataSource = DatabaseConnectionHelper.createDataSource(container); final Flyway flyway = FlywayFactory.create(dataSource, installedBy, dbIdentifier, migrationFileLocation); assertNotNull(flyway); assertTrue(flyway.getConfiguration().isBaselineOnMigrate()); assertEquals(FlywayFactory.BASELINE_DESCRIPTION, flyway.getConfiguration().getBaselineDescription()); assertEquals(FlywayFactory.BASELINE_VERSION, flyway.getConfiguration().getBaselineVersion().getVersion()); + assertEquals(FlywayFactory.BASELINE_ON_MIGRATION, flyway.getConfiguration().isBaselineOnMigrate()); assertEquals(installedBy, flyway.getConfiguration().getInstalledBy()); assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, dbIdentifier), flyway.getConfiguration().getTable()); assertEquals(migrationFileLocation, flyway.getConfiguration().getLocations()[0].getDescriptor()); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/AbstractDatabaseTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/AbstractDatabaseTest.java index de040b91c34..5fbb5723b89 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/AbstractDatabaseTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/AbstractDatabaseTest.java @@ -5,7 +5,13 @@ package io.airbyte.db.instance; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.test.utils.DatabaseConnectionHelper; +import java.io.Closeable; import java.io.IOException; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -31,22 +37,41 @@ public abstract class AbstractDatabaseTest { } protected Database database; + protected DataSource dataSource; + protected DSLContext dslContext; @BeforeEach - public void setup() throws Exception { - database = getDatabase(); + public void setup() throws IOException { + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + database = getDatabase(dataSource, dslContext); } @AfterEach - void tearDown() throws Exception { - database.close(); + void tearDown() throws IOException { + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } /** - * Create an initialized database. The downstream implementation should do it by calling + * Create an initialized {@link Database}. The downstream implementation should do it by calling * {@link DatabaseInstance#getAndInitialize} or {@link DatabaseInstance#getInitialized}, and * {@link DatabaseMigrator#migrate} if necessary. + * + * @param dataSource The {@link DataSource} used to access the database. + * @param dslContext The {@link DSLContext} used to execute queries. + * @return an initialized {@link Database} instance. */ - public abstract Database getDatabase() throws IOException; + public abstract Database getDatabase(DataSource dataSource, DSLContext dslContext) throws IOException; + + public DataSource getDataSource() { + return dataSource; + } + + public DSLContext getDslContext() { + return dslContext; + } } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/BaseDatabaseInstanceTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/BaseDatabaseInstanceTest.java index 58984362acf..ba7e82378a5 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/BaseDatabaseInstanceTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/BaseDatabaseInstanceTest.java @@ -8,7 +8,12 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.test.utils.DatabaseConnectionHelper; +import java.io.Closeable; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -41,18 +46,23 @@ class BaseDatabaseInstanceTest { } private Database database; + private DataSource dataSource; + private DSLContext dslContext; @BeforeEach void createDatabase() { - database = Databases.createPostgresDatabaseWithRetry( - container.getUsername(), container.getPassword(), container.getJdbcUrl(), - BaseDatabaseInstance.isDatabaseConnected(DATABASE_NAME)); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + database = Database.createWithRetry(dslContext, BaseDatabaseInstance.isDatabaseConnected(DATABASE_NAME)); } @AfterEach void tearDown() throws Exception { database.transaction(ctx -> ctx.execute(String.format("DROP TABLE IF EXISTS %s;", TABLE_NAME))); - database.close(); + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } @Test diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/AbstractConfigsDatabaseTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/AbstractConfigsDatabaseTest.java index c786918ae91..bcbd5c0c744 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/AbstractConfigsDatabaseTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/AbstractConfigsDatabaseTest.java @@ -8,12 +8,14 @@ import io.airbyte.db.Database; import io.airbyte.db.instance.AbstractDatabaseTest; import io.airbyte.db.instance.test.TestDatabaseProviders; import java.io.IOException; +import javax.sql.DataSource; +import org.jooq.DSLContext; public abstract class AbstractConfigsDatabaseTest extends AbstractDatabaseTest { @Override - public Database getDatabase() throws IOException { - return new TestDatabaseProviders(container).turnOffMigration().createNewConfigsDatabase(); + public Database getDatabase(final DataSource dataSource, final DSLContext dslContext) throws IOException { + return new TestDatabaseProviders(dataSource, dslContext).turnOffMigration().createNewConfigsDatabase(); } } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigratorTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigratorTest.java index aa7ab377eec..219bbba1830 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigratorTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/ConfigsDatabaseMigratorTest.java @@ -4,9 +4,11 @@ package io.airbyte.db.instance.configs; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.DatabaseMigrator; import io.airbyte.db.instance.development.MigrationDevHelper; import java.io.IOException; +import org.flywaydb.core.Flyway; import org.junit.jupiter.api.Test; public class ConfigsDatabaseMigratorTest extends AbstractConfigsDatabaseTest { @@ -15,7 +17,9 @@ public class ConfigsDatabaseMigratorTest extends AbstractConfigsDatabaseTest { @Test public void dumpSchema() throws IOException { - final DatabaseMigrator migrator = new ConfigsDatabaseMigrator(database, ConfigsDatabaseMigratorTest.class.getSimpleName()); + final Flyway flyway = FlywayFactory.create(getDataSource(), getClass().getSimpleName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final DatabaseMigrator migrator = new ConfigsDatabaseMigrator(database, flyway); migrator.migrate(); final String schema = migrator.dumpSchema(); MigrationDevHelper.dumpSchema(schema, SCHEMA_DUMP_FILE, false); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state_test.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state_test.java index c864462a7fe..b958954d833 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state_test.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state_test.java @@ -47,7 +47,7 @@ import org.jooq.Field; import org.jooq.JSONB; import org.jooq.Table; import org.jooq.impl.SQLDataType; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -84,17 +84,13 @@ class V0_30_22_001__Store_last_sync_state_test extends AbstractConfigsDatabaseTe private static final StandardSyncState STD_CONNECTION_STATE_3 = getStandardSyncState(CONNECTION_3_ID, CONNECTION_3_STATE); private static final Set STD_CONNECTION_STATES = Set.of(STD_CONNECTION_STATE_2, STD_CONNECTION_STATE_3); - private static Database jobDatabase; + private Database jobDatabase; - @BeforeAll + @BeforeEach @Timeout(value = 2, unit = TimeUnit.MINUTES) - public static void setupJobDatabase() throws Exception { - jobDatabase = new JobsDatabaseInstance( - container.getUsername(), - container.getPassword(), - container.getJdbcUrl()) - .getAndInitialize(); + public void setupJobDatabase() throws Exception { + jobDatabase = new JobsDatabaseInstance(dslContext).getAndInitialize(); } @Test @@ -166,7 +162,7 @@ class V0_30_22_001__Store_last_sync_state_test extends AbstractConfigsDatabaseTe */ final OffsetDateTime timestamp = timestampWithFullPrecision.withNano(1000 * (timestampWithFullPrecision.getNano() / 1000)); - database.query(ctx -> { + jobDatabase.query(ctx -> { V0_30_22_001__Store_last_sync_state.copyData(ctx, STD_CONNECTION_STATES, timestamp); checkSyncStates(ctx, STD_CONNECTION_STATES, timestamp); @@ -185,7 +181,7 @@ class V0_30_22_001__Store_last_sync_state_test extends AbstractConfigsDatabaseTe @Test @Order(40) public void testMigration() throws Exception { - database.query(ctx -> ctx.deleteFrom(TABLE_AIRBYTE_CONFIGS) + jobDatabase.query(ctx -> ctx.deleteFrom(TABLE_AIRBYTE_CONFIGS) .where(COLUMN_CONFIG_TYPE.eq(ConfigSchema.STANDARD_SYNC_STATE.name())) .execute()); @@ -201,7 +197,7 @@ class V0_30_22_001__Store_last_sync_state_test extends AbstractConfigsDatabaseTe @Override public Connection getConnection() { try { - return database.getDataSource().getConnection(); + return dataSource.getConnection(); } catch (final SQLException e) { throw new RuntimeException(e); } @@ -209,7 +205,7 @@ class V0_30_22_001__Store_last_sync_state_test extends AbstractConfigsDatabaseTe }; migration.migrate(context); - database.query(ctx -> { + jobDatabase.query(ctx -> { checkSyncStates(ctx, STD_CONNECTION_STATES, null); return null; }); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java index d2020d279b1..289085ed9c5 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java @@ -4,8 +4,13 @@ package io.airbyte.db.instance.configs.migrations; -import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.*; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.destinationConnections; import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.destinationOauthParameters; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.now; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.sourceConnections; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.sourceOauthParameters; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardDestinationDefinitions; +import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardSourceDefinitions; import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardSyncOperations; import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardSyncStates; import static io.airbyte.db.instance.configs.migrations.SetupForNormalizedTablesTest.standardSyncs; @@ -31,7 +36,6 @@ import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardSyncState; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.State; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.NamespaceDefinitionType; @@ -60,8 +64,7 @@ public class V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test extends Abst @Test public void testCompleteMigration() throws IOException, SQLException { - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); SetupForNormalizedTablesTest.setup(context); V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); @@ -79,7 +82,7 @@ public class V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test extends Abst } private void assertDataForWorkspace(final DSLContext context) { - Result workspaces = context.select(asterisk()) + final Result workspaces = context.select(asterisk()) .from(table("workspace")) .fetch(); Assertions.assertEquals(1, workspaces.size()); @@ -105,7 +108,7 @@ public class V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test extends Abst final List notificationList = new ArrayList<>(); final List fetchedNotifications = Jsons.deserialize(workspace.get(notifications).data(), List.class); - for (Object notification : fetchedNotifications) { + for (final Object notification : fetchedNotifications) { notificationList.add(Jsons.convertValue(notification, Notification.class)); } final StandardWorkspace workspaceFromNewTable = new StandardWorkspace() @@ -428,7 +431,7 @@ public class V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test extends Abst final List ids = new ArrayList<>(); - for (Record record : connectionOperations) { + for (final Record record : connectionOperations) { ids.add(record.get(operationId)); Assertions.assertNotNull(record.get(id)); Assertions.assertEquals(now(), record.get(createdAt).toInstant()); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_14_001__AddTombstoneToActorDefinitionTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_14_001__AddTombstoneToActorDefinitionTest.java index cd86e051c42..fc715959b6d 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_14_001__AddTombstoneToActorDefinitionTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_14_001__AddTombstoneToActorDefinitionTest.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.configs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import java.io.IOException; @@ -21,9 +20,7 @@ public class V0_35_14_001__AddTombstoneToActorDefinitionTest extends AbstractCon @Test public void test() throws SQLException, IOException { - - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); // necessary to add actor_definition table V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_15_001__AddReleaseStageAndReleaseDateToActorDefinition_Test.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_15_001__AddReleaseStageAndReleaseDateToActorDefinition_Test.java index ad9ee714234..98b18daad06 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_15_001__AddReleaseStageAndReleaseDateToActorDefinition_Test.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_15_001__AddReleaseStageAndReleaseDateToActorDefinition_Test.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.configs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import io.airbyte.db.instance.configs.migrations.V0_35_15_001__AddReleaseStageAndReleaseDateToActorDefinition.ReleaseStage; @@ -21,9 +20,7 @@ public class V0_35_15_001__AddReleaseStageAndReleaseDateToActorDefinition_Test e @Test public void test() throws SQLException, IOException { - - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); // necessary to add actor_definition table V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth_Test.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth_Test.java index 99fa6944a95..6da56e5f428 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth_Test.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth_Test.java @@ -13,7 +13,6 @@ import static org.jooq.impl.DSL.table; import io.airbyte.commons.json.Jsons; import io.airbyte.config.DestinationOAuthParameter; import io.airbyte.config.SourceOAuthParameter; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import java.io.IOException; @@ -35,8 +34,7 @@ public class V0_35_1_001__RemoveForeignKeyFromActorOauth_Test extends AbstractCo @Test public void testCompleteMigration() throws IOException, SQLException { - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); SetupForNormalizedTablesTest.setup(context); V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java index d7d0b5c2ba1..2ac67c52df6 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.configs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import java.io.IOException; @@ -21,9 +20,7 @@ class V0_35_26_001__PersistDiscoveredCatalogTest extends AbstractConfigsDatabase @Test public void test() throws SQLException, IOException { - - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); V0_35_26_001__PersistDiscoveredCatalog.migrate(context); assertCanInsertData(context); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_28_001__AddActorCatalogMetadataColumnsTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_28_001__AddActorCatalogMetadataColumnsTest.java index 7038326b98d..46ed1d9c97a 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_28_001__AddActorCatalogMetadataColumnsTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_28_001__AddActorCatalogMetadataColumnsTest.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.configs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import java.io.IOException; @@ -21,9 +20,7 @@ public class V0_35_28_001__AddActorCatalogMetadataColumnsTest extends AbstractCo @Test public void test() throws SQLException, IOException { - - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); V0_35_26_001__PersistDiscoveredCatalog.migrate(context); V0_35_28_001__AddActorCatalogMetadataColumns.migrate(context); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_3_001__DropAirbyteConfigsTableTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_3_001__DropAirbyteConfigsTableTest.java index c2087655df6..62db5e8f26e 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_3_001__DropAirbyteConfigsTableTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_3_001__DropAirbyteConfigsTableTest.java @@ -8,7 +8,6 @@ import static org.jooq.impl.DSL.select; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import java.io.IOException; import java.sql.SQLException; @@ -20,8 +19,7 @@ public class V0_35_3_001__DropAirbyteConfigsTableTest extends AbstractConfigsDat @Test public void test() throws IOException, SQLException { - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); assertTrue(airbyteConfigsExists(context)); V0_35_3_001__DropAirbyteConfigsTable.dropTable(context); assertFalse(airbyteConfigsExists(context)); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_001__AddPublicToActorDefinitionTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_001__AddPublicToActorDefinitionTest.java index d2e7f080d72..b70c2b5ea80 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_001__AddPublicToActorDefinitionTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_001__AddPublicToActorDefinitionTest.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.configs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import java.io.IOException; @@ -21,9 +20,7 @@ public class V0_35_59_001__AddPublicToActorDefinitionTest extends AbstractConfig @Test public void test() throws SQLException, IOException { - - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); // necessary to add actor_definition table V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_002__AddActorDefinitionWorkspaceGrantTableTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_002__AddActorDefinitionWorkspaceGrantTableTest.java index bce4bd541ff..b24cbc0a332 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_002__AddActorDefinitionWorkspaceGrantTableTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_002__AddActorDefinitionWorkspaceGrantTableTest.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.configs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import java.io.IOException; @@ -21,9 +20,7 @@ public class V0_35_59_002__AddActorDefinitionWorkspaceGrantTableTest extends Abs @Test public void test() throws SQLException, IOException { - - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); final UUID actorDefinitionId = new UUID(0L, 1L); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_003__AddCustomToActorDefinitionTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_003__AddCustomToActorDefinitionTest.java index 1c0a2862ca0..4c445e92d1a 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_003__AddCustomToActorDefinitionTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_59_003__AddCustomToActorDefinitionTest.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.configs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; import java.io.IOException; @@ -21,9 +20,7 @@ class V0_35_59_003__AddCustomToActorDefinitionTest extends AbstractConfigsDataba @Test public void test() throws SQLException, IOException { - - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); // necessary to add actor_definition table V0_32_8_001__AirbyteConfigDatabaseDenormalization.migrate(context); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/AbstractJobsDatabaseTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/AbstractJobsDatabaseTest.java index b14fc53b8a6..fae7759a3d7 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/AbstractJobsDatabaseTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/AbstractJobsDatabaseTest.java @@ -8,12 +8,14 @@ import io.airbyte.db.Database; import io.airbyte.db.instance.AbstractDatabaseTest; import io.airbyte.db.instance.test.TestDatabaseProviders; import java.io.IOException; +import javax.sql.DataSource; +import org.jooq.DSLContext; public abstract class AbstractJobsDatabaseTest extends AbstractDatabaseTest { @Override - public Database getDatabase() throws IOException { - return new TestDatabaseProviders(container).turnOffMigration().createNewJobsDatabase(); + public Database getDatabase(final DataSource dataSource, final DSLContext dslContext) throws IOException { + return new TestDatabaseProviders(dataSource, dslContext).turnOffMigration().createNewJobsDatabase(); } } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseInstanceTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseInstanceTest.java index a3ec8d804cf..aa1339876d8 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseInstanceTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseInstanceTest.java @@ -7,6 +7,7 @@ package io.airbyte.db.instance.jobs; import static org.jooq.impl.DSL.select; import static org.junit.jupiter.api.Assertions.assertThrows; +import io.airbyte.db.Database; import org.jooq.exception.DataAccessException; import org.junit.jupiter.api.Test; @@ -14,15 +15,15 @@ class JobsDatabaseInstanceTest extends AbstractJobsDatabaseTest { @Test public void testGet() throws Exception { - // when the database has been initialized and loaded with data (in setup method), the get method - // should return the database - database = new JobsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getInitialized(); + final Database database = new JobsDatabaseInstance(getDslContext()).getInitialized(); // check table database.query(ctx -> ctx.fetchExists(select().from("airbyte_metadata"))); } @Test public void testGetAndInitialize() throws Exception { + final Database database = new JobsDatabaseInstance(getDslContext()).getInitialized(); + // check table database.query(ctx -> ctx.fetchExists(select().from("jobs"))); database.query(ctx -> ctx.fetchExists(select().from("attempts"))); @@ -31,9 +32,9 @@ class JobsDatabaseInstanceTest extends AbstractJobsDatabaseTest { // when the jobs database has been initialized, calling getAndInitialize again will not change // anything final String testSchema = "CREATE TABLE IF NOT EXISTS airbyte_test_metadata(id BIGINT PRIMARY KEY);"; - database = new JobsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl(), testSchema).getAndInitialize(); + final Database database2 = new JobsDatabaseInstance(getDslContext(), testSchema).getAndInitialize(); // the airbyte_test_metadata table does not exist - assertThrows(DataAccessException.class, () -> database.query(ctx -> ctx.fetchExists(select().from("airbyte_test_metadata")))); + assertThrows(DataAccessException.class, () -> database2.query(ctx -> ctx.fetchExists(select().from("airbyte_test_metadata")))); } } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseMigratorTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseMigratorTest.java index 219fc8b0e99..8447a8c4c8e 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseMigratorTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/JobsDatabaseMigratorTest.java @@ -4,9 +4,11 @@ package io.airbyte.db.instance.jobs; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.DatabaseMigrator; import io.airbyte.db.instance.development.MigrationDevHelper; import java.io.IOException; +import org.flywaydb.core.Flyway; import org.junit.jupiter.api.Test; public class JobsDatabaseMigratorTest extends AbstractJobsDatabaseTest { @@ -15,7 +17,9 @@ public class JobsDatabaseMigratorTest extends AbstractJobsDatabaseTest { @Test public void dumpSchema() throws IOException { - final DatabaseMigrator migrator = new JobsDatabaseMigrator(database, JobsDatabaseMigratorTest.class.getSimpleName()); + final Flyway flyway = FlywayFactory.create(getDataSource(), getClass().getSimpleName(), JobsDatabaseMigrator.DB_IDENTIFIER, + JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final DatabaseMigrator migrator = new JobsDatabaseMigrator(database, flyway); migrator.migrate(); final String schema = migrator.dumpSchema(); MigrationDevHelper.dumpSchema(schema, SCHEMA_DUMP_FILE, false); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001_MigrateFailureReasonEnumValues_Test.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001_MigrateFailureReasonEnumValues_Test.java index b2d7e93f4db..4a1d0818b97 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001_MigrateFailureReasonEnumValues_Test.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001_MigrateFailureReasonEnumValues_Test.java @@ -18,7 +18,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import io.airbyte.commons.json.Jsons; import io.airbyte.config.Metadata; -import io.airbyte.db.Database; import io.airbyte.db.instance.jobs.AbstractJobsDatabaseTest; import io.airbyte.db.instance.jobs.migrations.V0_35_40_001__MigrateFailureReasonEnumValues.AttemptFailureSummaryForMigration; import io.airbyte.db.instance.jobs.migrations.V0_35_40_001__MigrateFailureReasonEnumValues.FailureReasonForMigration; @@ -100,8 +99,7 @@ public class V0_35_40_001_MigrateFailureReasonEnumValues_Test extends AbstractJo @Test public void test() throws Exception { - final Database database = getDatabase(); - final DSLContext ctx = DSL.using(database.getDataSource().getConnection()); + final DSLContext ctx = getDslContext(); V0_35_5_001__Add_failureSummary_col_to_Attempts.migrate(ctx); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_5_001__Add_failureSummary_col_to_AttemptsTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_5_001__Add_failureSummary_col_to_AttemptsTest.java index 94b79480346..eb3163d0e73 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_5_001__Add_failureSummary_col_to_AttemptsTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/jobs/migrations/V0_35_5_001__Add_failureSummary_col_to_AttemptsTest.java @@ -4,7 +4,6 @@ package io.airbyte.db.instance.jobs.migrations; -import io.airbyte.db.Database; import io.airbyte.db.instance.jobs.AbstractJobsDatabaseTest; import java.io.IOException; import java.sql.SQLException; @@ -17,8 +16,7 @@ public class V0_35_5_001__Add_failureSummary_col_to_AttemptsTest extends Abstrac @Test public void test() throws SQLException, IOException { - final Database database = getDatabase(); - final DSLContext context = DSL.using(database.getDataSource().getConnection()); + final DSLContext context = getDslContext(); Assertions.assertFalse(failureSummaryColumnExists(context)); V0_35_5_001__Add_failureSummary_col_to_Attempts.addFailureSummaryColumn(context); Assertions.assertTrue(failureSummaryColumnExists(context)); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseInstance.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseInstance.java index bd8813fdae7..78bd6a232ed 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseInstance.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseInstance.java @@ -10,6 +10,7 @@ import io.airbyte.db.instance.BaseDatabaseInstance; import java.io.IOException; import java.util.Collections; import java.util.function.Function; +import org.jooq.DSLContext; /** * A database instance for testing purposes only. @@ -27,8 +28,8 @@ public class ToysDatabaseInstance extends BaseDatabaseInstance { } }; - protected ToysDatabaseInstance(final String username, final String password, final String connectionString) throws IOException { - super(username, password, connectionString, MoreResources.readResource(SCHEMA_PATH), DATABASE_LOGGING_NAME, Collections.singleton(TABLE_NAME), + protected ToysDatabaseInstance(final DSLContext dslContext) throws IOException { + super(dslContext, DATABASE_LOGGING_NAME, MoreResources.readResource(SCHEMA_PATH), Collections.singleton(TABLE_NAME), IS_DATABASE_READY); } diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java index a77eed3e809..f2f275f1bd5 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigrator.java @@ -6,6 +6,7 @@ package io.airbyte.db.instance.toys; import io.airbyte.db.Database; import io.airbyte.db.instance.FlywayDatabaseMigrator; +import org.flywaydb.core.Flyway; /** * A database migrator for testing purposes only. @@ -15,8 +16,8 @@ public class ToysDatabaseMigrator extends FlywayDatabaseMigrator { public static final String DB_IDENTIFIER = "toy"; public static final String MIGRATION_FILE_LOCATION = "classpath:io/airbyte/db/instance/toys/migrations"; - public ToysDatabaseMigrator(final Database database, final String migrationRunner) { - super(database, DB_IDENTIFIER, migrationRunner, MIGRATION_FILE_LOCATION); + public ToysDatabaseMigrator(final Database database, final Flyway flyway) { + super(database, flyway); } @Override diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigratorTest.java b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigratorTest.java index 4c432e84426..67206e2f857 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigratorTest.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/instance/toys/ToysDatabaseMigratorTest.java @@ -8,9 +8,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.AbstractDatabaseTest; import io.airbyte.db.instance.DatabaseMigrator; import java.io.IOException; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; import org.junit.jupiter.api.Test; class ToysDatabaseMigratorTest extends AbstractDatabaseTest { @@ -19,13 +23,16 @@ class ToysDatabaseMigratorTest extends AbstractDatabaseTest { private static final String POST_MIGRATION_SCHEMA_DUMP = "toys_database/schema_dump.txt"; @Override - public Database getDatabase() throws IOException { - return new ToysDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + public Database getDatabase(final DataSource dataSource, final DSLContext dslContext) throws IOException { + return new ToysDatabaseInstance(dslContext).getAndInitialize(); } @Test public void testMigration() throws Exception { - final DatabaseMigrator migrator = new ToysDatabaseMigrator(database, ToysDatabaseMigratorTest.class.getSimpleName()); + final DataSource dataSource = getDataSource(); + final Flyway flyway = FlywayFactory.create(dataSource, getClass().getSimpleName(), ToysDatabaseMigrator.DB_IDENTIFIER, + ToysDatabaseMigrator.MIGRATION_FILE_LOCATION); + final DatabaseMigrator migrator = new ToysDatabaseMigrator(database, flyway); // Compare pre migration baseline schema migrator.createBaseline(); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestDefaultJdbcDatabase.java b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestDefaultJdbcDatabase.java index 0661c5b29af..2dbc7f70741 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestDefaultJdbcDatabase.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestDefaultJdbcDatabase.java @@ -12,11 +12,13 @@ import com.google.common.collect.Lists; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; import java.util.List; import java.util.stream.Stream; +import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -95,14 +97,15 @@ public class TestDefaultJdbcDatabase { } private JdbcDatabase getDatabaseFromConfig(final JsonNode config) { - return Databases.createJdbcDatabase( + final DataSource dataSource = DataSourceFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver"); + config.get("port").asInt(), + config.get("database").asText())); + return new DefaultJdbcDatabase(dataSource); } private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName) { diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index 001bfa1d3a9..0c86779cd05 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -17,6 +17,8 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.MoreStreams; import io.airbyte.commons.string.Strings; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; @@ -30,7 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.apache.commons.dbcp2.BasicDataSource; +import javax.sql.DataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -46,7 +48,7 @@ public class TestJdbcUtils { private static PostgreSQLContainer PSQL_DB; - private BasicDataSource dataSource; + private DataSource dataSource; private static final JdbcSourceOperations sourceOperations = JdbcUtils.getDefaultSourceOperations(); @BeforeAll @@ -66,14 +68,14 @@ public class TestJdbcUtils { final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); - dataSource = new BasicDataSource(); - dataSource.setDriverClassName("org.postgresql.Driver"); - dataSource.setUsername(config.get("username").asText()); - dataSource.setPassword(config.get("password").asText()); - dataSource.setUrl(String.format("jdbc:postgresql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText())); + dataSource = DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText())); final JdbcDatabase defaultJdbcDatabase = new DefaultJdbcDatabase(dataSource); diff --git a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestStreamingJdbcDatabase.java b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestStreamingJdbcDatabase.java index cac812dd71c..c7c08a33ec0 100644 --- a/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestStreamingJdbcDatabase.java +++ b/airbyte-db/lib/src/test/java/io/airbyte/db/jdbc/TestStreamingJdbcDatabase.java @@ -14,6 +14,8 @@ import com.google.common.collect.Lists; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.db.jdbc.streaming.FetchSizeConstants; import io.airbyte.test.utils.PostgreSQLContainerHelper; @@ -24,7 +26,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import org.apache.commons.dbcp2.BasicDataSource; +import javax.sql.DataSource; import org.elasticsearch.common.collect.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -65,14 +67,14 @@ public class TestStreamingJdbcDatabase { final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); - final BasicDataSource connectionPool = new BasicDataSource(); - connectionPool.setDriverClassName("org.postgresql.Driver"); - connectionPool.setUsername(config.get("username").asText()); - connectionPool.setPassword(config.get("password").asText()); - connectionPool.setUrl(String.format("jdbc:postgresql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText())); + final DataSource connectionPool = DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText())); defaultJdbcDatabase = spy(new DefaultJdbcDatabase(connectionPool)); streamingJdbcDatabase = new StreamingJdbcDatabase(connectionPool, JdbcUtils.getDefaultSourceOperations(), AdaptiveStreamingQueryConfig::new); diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index c77f75cb38a..42139f79dbe 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -18,8 +18,10 @@ dependencies { implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' implementation 'org.bouncycastle:bctls-jdk15on:1.66' - implementation "org.testcontainers:testcontainers:1.15.3" - implementation "org.testcontainers:jdbc:1.15.3" + implementation libs.testcontainers + implementation libs.testcontainers.jdbc implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) + + testImplementation 'commons-lang:commons-lang:2.6' } diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java index 1816b3abf1c..25e467bb5e8 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java @@ -35,9 +35,9 @@ import java.util.Collection; import java.util.List; import java.util.function.Consumer; import java.util.stream.Collectors; +import org.apache.commons.lang.RandomStringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils; public class BufferedStreamConsumerTest { diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java index 6ba055ea19c..8e637e654e4 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java @@ -172,8 +172,6 @@ public abstract class AbstractSourceDatabaseTypeTest extends AbstractSourceConne return null; }); } - - database.close(); } /** diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java index 9680761dd69..096bace51d1 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java @@ -68,9 +68,6 @@ public abstract class AbstractSourceFillDbWithTestData extends AbstractSourceBas } return null; }); - - database.close(); - } /** diff --git a/airbyte-integrations/connectors/destination-cassandra/build.gradle b/airbyte-integrations/connectors/destination-cassandra/build.gradle index c9560e86586..fb307b0ced8 100644 --- a/airbyte-integrations/connectors/destination-cassandra/build.gradle +++ b/airbyte-integrations/connectors/destination-cassandra/build.gradle @@ -10,7 +10,6 @@ application { } def cassandraDriver = '4.13.0' -def testContainersVersion = '1.16.0' def assertVersion = '3.21.0' dependencies { @@ -26,7 +25,7 @@ dependencies { // https://mvnrepository.com/artifact/org.assertj/assertj-core testImplementation "org.assertj:assertj-core:${assertVersion}" - testImplementation "org.testcontainers:cassandra:${testContainersVersion}" + testImplementation libs.testcontainers.cassandra integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index 5899daf2bc9..72efaa86a9f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -21,10 +21,10 @@ dependencies { implementation 'ru.yandex.clickhouse:clickhouse-jdbc:0.3.1-patch' // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - testImplementation 'org.testcontainers:clickhouse:1.16.2' + testImplementation libs.testcontainers.clickhouse integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-clickhouse') // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - integrationTestJavaImplementation "org.testcontainers:clickhouse:1.16.2" + integrationTestJavaImplementation libs.testcontainers.clickhouse } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index ee22f15849f..f9822772d1d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -132,11 +133,13 @@ public class ClickhouseDestinationStrictEncryptAcceptanceTest extends Destinatio config.get("host").asText(), config.get("port").asText(), config.get("database").asText()); - return Databases.createJdbcDatabase( + return new DefaultJdbcDatabase(DataSourceFactory.create( config.get("username").asText(), config.has("password") ? config.get("password").asText() : null, - jdbcStr, - ClickhouseDestination.DRIVER_CLASS); + ClickhouseDestination.DRIVER_CLASS, + jdbcStr + ) + ); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 496a0e7f09e..acf3a11a0d0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -21,11 +21,11 @@ dependencies { implementation 'ru.yandex.clickhouse:clickhouse-jdbc:0.3.1-patch' // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - testImplementation 'org.testcontainers:clickhouse:1.16.2' + testImplementation libs.testcontainers.clickhouse integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-clickhouse') integrationTestJavaImplementation project(':airbyte-workers') // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - integrationTestJavaImplementation "org.testcontainers:clickhouse:1.16.2" + integrationTestJavaImplementation libs.testcontainers.clickhouse } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index b77459a8ee6..a41b3f4f2a4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -7,6 +7,7 @@ package io.airbyte.integrations.destination.clickhouse; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; @@ -25,7 +26,7 @@ public class ClickhouseDestination extends AbstractJdbcDestination implements De private static final Logger LOGGER = LoggerFactory.getLogger(ClickhouseDestination.class); - public static final String DRIVER_CLASS = "ru.yandex.clickhouse.ClickHouseDriver"; + public static final String DRIVER_CLASS = DatabaseDriver.CLICKHOUSE.getDriverClassName(); public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 6dee8276d19..1166411612c 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -8,7 +8,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -126,14 +128,17 @@ public class ClickhouseDestinationAcceptanceTest extends DestinationAcceptanceTe } private static JdbcDatabase getDatabase(final JsonNode config) { - return Databases.createJdbcDatabase( - config.get("username").asText(), - config.has("password") ? config.get("password").asText() : null, - String.format("jdbc:clickhouse://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - ClickhouseDestination.DRIVER_CLASS); + return new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.has("password") ? config.get("password").asText() : null, + ClickhouseDestination.DRIVER_CLASS, + String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()) + ) + ); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index d5b0707116e..eeb4bd757ac 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -7,7 +7,9 @@ package io.airbyte.integrations.destination.clickhouse; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshBastionContainer; @@ -144,14 +146,17 @@ public abstract class SshClickhouseDestinationAcceptanceTest extends Destination } private static JdbcDatabase getDatabase(final JsonNode config) { - return Databases.createJdbcDatabase( - config.get("username").asText(), - config.has("password") ? config.get("password").asText() : null, - String.format("jdbc:clickhouse://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - ClickhouseDestination.DRIVER_CLASS); + return new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.has("password") ? config.get("password").asText() : null, + ClickhouseDestination.DRIVER_CLASS, + String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()) + ) + ); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 75e5d966374..765acaef02c 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -10,7 +10,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.AirbyteMessageConsumer; @@ -125,14 +126,17 @@ public class ClickhouseDestinationTest { .withData(Jsons.jsonNode(ImmutableMap.of(DB_NAME + "." + STREAM_NAME, 10))))); consumer.close(); - final JdbcDatabase database = Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:clickhouse://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - ClickhouseDestination.DRIVER_CLASS); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + ClickhouseDestination.DRIVER_CLASS, + String.format("jdbc:clickhouse://%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("database").asText()) + ) + ); final List actualRecords = database.bufferedResultSetQuery( connection -> connection.createStatement().executeQuery( diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java index 8441df92bbb..b5f0df7de87 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java @@ -5,7 +5,8 @@ package io.airbyte.integrations.destination.databricks; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.IntegrationRunner; @@ -76,11 +77,13 @@ public class DatabricksDestination extends CopyDestination { } static JdbcDatabase getDatabase(final DatabricksDestinationConfig databricksConfig) { - return Databases.createJdbcDatabase( + return new DefaultJdbcDatabase(DataSourceFactory.create( DatabricksConstants.DATABRICKS_USERNAME, databricksConfig.getDatabricksPersonalAccessToken(), - getDatabricksConnectionString(databricksConfig), - DatabricksConstants.DATABRICKS_DRIVER_CLASS); + DatabricksConstants.DATABRICKS_DRIVER_CLASS, + getDatabricksConnectionString(databricksConfig) + ) + ); } } diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java index 55f03862fa3..80a19ab3612 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -32,6 +32,7 @@ import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,12 +142,8 @@ public class DatabricksDestinationAcceptanceTest extends DestinationAcceptanceTe } private static Database getDatabase(final DatabricksDestinationConfig databricksConfig) { - return Databases.createDatabase( - DatabricksConstants.DATABRICKS_USERNAME, - databricksConfig.getDatabricksPersonalAccessToken(), - DatabricksDestination.getDatabricksConnectionString(databricksConfig), - DatabricksConstants.DATABRICKS_DRIVER_CLASS, - SQLDialect.DEFAULT); + final DSLContext dslContext = DSLContextFactory.create(DatabricksConstants.DATABRICKS_USERNAME, databricksConfig.getDatabricksPersonalAccessToken(), DatabricksConstants.DATABRICKS_DRIVER_CLASS, DatabricksDestination.getDatabricksConnectionString(databricksConfig), SQLDialect.DEFAULT); + return new Database(dslContext); } } diff --git a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle index c4de7d30318..d498126e8a3 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle @@ -29,9 +29,9 @@ dependencies { // MIT // https://www.testcontainers.org/ - //implementation "org.testcontainers:testcontainers:1.16.0" - testImplementation "org.testcontainers:elasticsearch:1.15.3" - integrationTestJavaImplementation "org.testcontainers:elasticsearch:1.15.3" + //implementation libs.testcontainers.elasticsearch + testImplementation libs.testcontainers.elasticsearch + integrationTestJavaImplementation libs.testcontainers.elasticsearch integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-elasticsearch') diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index 8b03c8e8d13..ebbad184ccb 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -33,6 +33,7 @@ dependencies { implementation group: 'com.github.airbytehq', name: 'json-avro-converter', version: '1.0.1' testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-gcs') diff --git a/airbyte-integrations/connectors/destination-jdbc/build.gradle b/airbyte-integrations/connectors/destination-jdbc/build.gradle index ea25dc88d16..a34f53785bc 100644 --- a/airbyte-integrations/connectors/destination-jdbc/build.gradle +++ b/airbyte-integrations/connectors/destination-jdbc/build.gradle @@ -23,11 +23,11 @@ dependencies { // https://github.com/aesy/datasize implementation "io.aesy:datasize:1.0.0" - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.testcontainers.postgresql testImplementation "org.mockito:mockito-inline:4.1.0" integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3" + integrationTestJavaImplementation libs.testcontainers.postgresql implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java index d8afb121b15..ff33348ed0e 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -6,7 +6,8 @@ package io.airbyte.integrations.destination.jdbc; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.BaseConnector; @@ -86,12 +87,15 @@ public abstract class AbstractJdbcDestination extends BaseConnector implements D protected JdbcDatabase getDatabase(final JsonNode config) { final JsonNode jdbcConfig = toJdbcConfig(config); - return Databases.createJdbcDatabase( - jdbcConfig.get("username").asText(), - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - driverClass, - getConnectionProperties(config)); + return new DefaultJdbcDatabase( + DataSourceFactory.create( + jdbcConfig.get("username").asText(), + jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, + driverClass, + jdbcConfig.get("jdbc_url").asText(), + getConnectionProperties(config) + ) + ); } protected Map getConnectionProperties(final JsonNode config) { diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java index 671bbda75df..a497ed32b57 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java @@ -14,7 +14,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; @@ -46,11 +48,14 @@ class SqlOperationsUtilsTest { final JsonNode config = createConfig(); - database = Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - config.get("jdbc_url").asText(), - "org.postgresql.Driver"); + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + config.get("jdbc_url").asText() + ) + ); uuidSupplier = mock(Supplier.class); } diff --git a/airbyte-integrations/connectors/destination-kafka/build.gradle b/airbyte-integrations/connectors/destination-kafka/build.gradle index 159c3bb5de4..24657316da4 100644 --- a/airbyte-integrations/connectors/destination-kafka/build.gradle +++ b/airbyte-integrations/connectors/destination-kafka/build.gradle @@ -19,7 +19,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-kafka') - integrationTestJavaImplementation "org.testcontainers:kafka:1.15.3" + integrationTestJavaImplementation libs.testcontainers.kafka implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle index 39274993ff4..174572671b7 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle @@ -22,5 +22,5 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mariadb-columnstore') - integrationTestJavaImplementation "org.testcontainers:mariadb:1.16.2" + integrationTestJavaImplementation libs.testcontainers.mariadb } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index f8d6ed49025..cd4e6219e9e 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -7,6 +7,7 @@ package io.airbyte.integrations.destination.mariadb_columnstore; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; @@ -23,7 +24,7 @@ import org.slf4j.LoggerFactory; public class MariadbColumnstoreDestination extends AbstractJdbcDestination implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(MariadbColumnstoreDestination.class); - public static final String DRIVER_CLASS = "org.mariadb.jdbc.Driver"; + public static final String DRIVER_CLASS = DatabaseDriver.MARIADB.getDriverClassName(); public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); @@ -75,9 +76,9 @@ public class MariadbColumnstoreDestination extends AbstractJdbcDestination imple @Override public JsonNode toJdbcConfig(final JsonNode config) { - final String jdbcUrl = String.format("jdbc:mariadb://%s:%s/%s", + final String jdbcUrl = String.format(DatabaseDriver.MARIADB.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("database").asText()); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java index c2b193d2280..d755925e4cc 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java @@ -8,7 +8,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -101,14 +103,17 @@ public class MariadbColumnstoreDestinationAcceptanceTest extends DestinationAcce } private static JdbcDatabase getDatabase(final JsonNode config) { - return Databases.createJdbcDatabase( - config.get("username").asText(), - config.has("password") ? config.get("password").asText() : null, - String.format("jdbc:mariadb://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - MariadbColumnstoreDestination.DRIVER_CLASS); + return new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.has("password") ? config.get("password").asText() : null, + MariadbColumnstoreDestination.DRIVER_CLASS, + String.format(DatabaseDriver.MARIADB.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()) + ) + ); } @Override diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java index 907397a1efa..3b450643c70 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java @@ -9,7 +9,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshBastionContainer; @@ -19,6 +20,8 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTes import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.MariaDBContainer; @@ -73,10 +76,10 @@ public abstract class SshMariadbColumnstoreDestinationAcceptanceTest extends Des } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws Exception { return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) .stream() @@ -101,13 +104,15 @@ public abstract class SshMariadbColumnstoreDestinationAcceptanceTest extends Des } private static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createMariaDbDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:mariadb://%s:%s/%s", + DatabaseDriver.MARIADB.getDriverClassName(), + String.format(DatabaseDriver.MARIADB.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText())); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.MARIADB); + return new Database(dslContext); } @Override @@ -121,25 +126,25 @@ public abstract class SshMariadbColumnstoreDestinationAcceptanceTest extends Des } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv) throws Exception { bastion.initAndStartBastion(); startAndInitJdbcContainer(); } private void startAndInitJdbcContainer() throws Exception { - DockerImageName mcsImage = DockerImageName.parse("fengdi/columnstore:1.5.2").asCompatibleSubstituteFor("mariadb"); + final DockerImageName mcsImage = DockerImageName.parse("fengdi/columnstore:1.5.2").asCompatibleSubstituteFor("mariadb"); db = new MariaDBContainer<>(mcsImage) .withNetwork(bastion.getNetWork()); db.start(); - String createUser = String.format("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", db.getUsername(), db.getPassword()); - String grantAll = String.format("GRANT ALL PRIVILEGES ON *.* TO '%s'@'%%' IDENTIFIED BY '%s';", db.getUsername(), db.getPassword()); - String createDb = String.format("CREATE DATABASE %s DEFAULT CHARSET = utf8;", db.getDatabaseName()); + final String createUser = String.format("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", db.getUsername(), db.getPassword()); + final String grantAll = String.format("GRANT ALL PRIVILEGES ON *.* TO '%s'@'%%' IDENTIFIED BY '%s';", db.getUsername(), db.getPassword()); + final String createDb = String.format("CREATE DATABASE %s DEFAULT CHARSET = utf8;", db.getDatabaseName()); db.execInContainer("mariadb", "-e", createUser + grantAll + createDb); } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { bastion.stopAndCloseContainers(db); } diff --git a/airbyte-integrations/connectors/destination-meilisearch/build.gradle b/airbyte-integrations/connectors/destination-meilisearch/build.gradle index 6ea15c9d089..c1a10c4f776 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/build.gradle +++ b/airbyte-integrations/connectors/destination-meilisearch/build.gradle @@ -20,7 +20,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-meilisearch') - integrationTestJavaImplementation "org.testcontainers:testcontainers:1.15.3" + integrationTestJavaImplementation libs.testcontainers implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle index 21ebce1af5f..81ee2032de4 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation project(':airbyte-integrations:connectors:destination-mongodb') implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - testImplementation 'org.testcontainers:mongodb:1.15.3' + testImplementation libs.testcontainers.mongodb integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb-strict-encrypt') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') diff --git a/airbyte-integrations/connectors/destination-mongodb/build.gradle b/airbyte-integrations/connectors/destination-mongodb/build.gradle index 0a5f8f840e6..253bd086c9c 100644 --- a/airbyte-integrations/connectors/destination-mongodb/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - testImplementation 'org.testcontainers:mongodb:1.15.3' + testImplementation libs.testcontainers.mongodb integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java index 8cce8c2cc8a..4c0d64b963c 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -116,7 +117,8 @@ public class MongodbRecordConsumer extends FailureTrackingAirbyteMessageConsumer try { final AirbyteRecordMessage recordMessage = message.getRecord(); final Map result = objectMapper.convertValue(recordMessage.getData(), new TypeReference<>() {}); - final var newDocumentDataHashCode = UUID.nameUUIDFromBytes(DigestUtils.md5Hex(Jsons.toBytes(recordMessage.getData())).getBytes()).toString(); + final var newDocumentDataHashCode = UUID.nameUUIDFromBytes(DigestUtils.md5Hex(Jsons.toBytes(recordMessage.getData())).getBytes( + Charset.defaultCharset())).toString(); final var newDocument = new Document(); newDocument.put(AIRBYTE_DATA, new Document(result)); newDocument.put(AIRBYTE_DATA_HASH, newDocumentDataHashCode); diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle index c641cf2b3a7..69932bb3d6d 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle @@ -16,11 +16,12 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-mssql') + implementation project(':airbyte-test-utils') implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation "org.testcontainers:mssqlserver:1.15.3" + testImplementation libs.testcontainers.mssqlserver integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mssql-strict-encrypt') diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java index a84c56bcd3c..76ca12d6cc0 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java @@ -13,17 +13,21 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -121,8 +125,8 @@ public class MssqlStrictEncryptDestinationAcceptanceTest extends DestinationAcce } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createSqlServerDatabase(db.getUsername(), db.getPassword(), - db.getJdbcUrl()).query( + final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, null); + return new Database(dslContext).query( ctx -> { ctx.fetch(String.format("USE %s;", config.get("database"))); return ctx @@ -135,12 +139,14 @@ public class MssqlStrictEncryptDestinationAcceptanceTest extends DestinationAcce } private static Database getDatabase(final JsonNode config) { - return Databases.createSqlServerDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s", + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format(DatabaseDriver.MSSQLSERVER.getUrlFormatString(), config.get("host").asText(), - config.get("port").asInt())); + config.get("port").asInt()), SQLDialect.DEFAULT); + return new Database(dslContext); } @Override diff --git a/airbyte-integrations/connectors/destination-mssql/build.gradle b/airbyte-integrations/connectors/destination-mssql/build.gradle index 394142734d1..0950b705f4a 100644 --- a/airbyte-integrations/connectors/destination-mssql/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql/build.gradle @@ -14,11 +14,12 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-protocol:models') implementation project(':airbyte-integrations:connectors:destination-jdbc') + implementation project(':airbyte-test-utils') implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation "org.testcontainers:mssqlserver:1.15.3" + testImplementation libs.testcontainers.mssqlserver integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mssql') diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java index 7acba580fe2..bd4f667a2e3 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java +++ b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java @@ -7,6 +7,7 @@ package io.airbyte.integrations.destination.mssql; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.ssh.SshWrappedDestination; @@ -23,7 +24,7 @@ public class MSSQLDestination extends AbstractJdbcDestination implements Destina private static final Logger LOGGER = LoggerFactory.getLogger(MSSQLDestination.class); - public static final String DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + public static final String DRIVER_CLASS = DatabaseDriver.MSSQLSERVER.getDriverClassName(); public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java index d0a2dc25eb7..b52d9220626 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java @@ -5,20 +5,22 @@ package io.airbyte.integrations.destination.mssql; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import org.jooq.DSLContext; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.MSSQLServerContainer; @@ -27,8 +29,6 @@ public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes private static MSSQLServerContainer db; private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); - private final ObjectMapper mapper = new ObjectMapper(); - private JsonNode configWithoutDbName; private JsonNode config; @Override @@ -98,8 +98,8 @@ public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createSqlServerDatabase(db.getUsername(), db.getPassword(), - db.getJdbcUrl()).query( + final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, null); + return new Database(dslContext).query( ctx -> { ctx.fetch(String.format("USE %s;", config.get("database"))); return ctx @@ -117,16 +117,14 @@ public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes } private static Database getDatabase(final JsonNode config) { - // todo (cgardens) - rework this abstraction so that we do not have to pass a null into the - // constructor. at least explicitly handle it, even if the impl doesn't change. - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), String.format("jdbc:sqlserver://%s:%s", config.get("host").asText(), - config.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + config.get("port").asInt()), null); + return new Database(dslContext); } // how to interact with the mssql test container manaully. @@ -134,7 +132,7 @@ public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes // 2. /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "A_Str0ng_Required_Password" @Override protected void setup(final TestDestinationEnv testEnv) throws SQLException { - configWithoutDbName = getConfig(db); + final JsonNode configWithoutDbName = getConfig(db); final String dbName = Strings.addRandomSuffix("db", "_", 10); final Database database = getDatabase(configWithoutDbName); diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java index 8717a8c7f6c..cf182f0e8c3 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java @@ -10,15 +10,19 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.MSSQLServerContainer; @@ -119,8 +123,8 @@ public class MSSQLDestinationAcceptanceTestSSL extends DestinationAcceptanceTest } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createSqlServerDatabase(db.getUsername(), db.getPassword(), - db.getJdbcUrl()).query( + final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, SQLDialect.DEFAULT); + return new Database(dslContext).query( ctx -> { ctx.fetch(String.format("USE %s;", config.get("database"))); return ctx @@ -140,19 +144,17 @@ public class MSSQLDestinationAcceptanceTestSSL extends DestinationAcceptanceTest } private static Database getDatabase(final JsonNode config) { - // todo (cgardens) - rework this abstraction so that we do not have to pass a null into the - // constructor. at least explicitly handle it, even if the impl doesn't change. - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), String.format("jdbc:sqlserver://%s:%s", config.get("host").asText(), - config.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + config.get("port").asInt()), null); + return new Database(dslContext); } - // how to interact with the mssql test container manaully. + // how to interact with the mssql test container manually. // 1. exec into mssql container (not the test container container) // 2. /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "A_Str0ng_Required_Password" @Override diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java index e0a1cba4bb3..1780fed4798 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java @@ -10,7 +10,9 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshBastionContainer; @@ -21,7 +23,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import javax.sql.DataSource; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.containers.Network; @@ -120,14 +124,14 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAccep } private static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), String.format("jdbc:sqlserver://%s:%s", config.get("host").asText(), - config.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + config.get("port").asInt()), null); + return new Database(dslContext); } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle index 6083f79111c..a058837e5a6 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle @@ -20,7 +20,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mysql') - integrationTestJavaImplementation "org.testcontainers:mysql:1.15.3" + integrationTestJavaImplementation libs.testcontainers.mysql implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java index 0c29d20a296..885a56a22ad 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java @@ -10,7 +10,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -27,6 +29,8 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -99,15 +103,15 @@ public class MySQLStrictEncryptDestinationAcceptanceTest extends DestinationAcce } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( db.getUsername(), db.getPassword(), + db.getDriverClassName(), String.format("jdbc:mysql://%s:%s/%s?useSSL=true&requireSSL=true&verifyServerCertificate=false", db.getHost(), db.getFirstMappedPort(), - db.getDatabaseName()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL).query( + db.getDatabaseName()), SQLDialect.MYSQL); + return new Database(dslContext).query( ctx -> ctx .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) @@ -160,15 +164,15 @@ public class MySQLStrictEncryptDestinationAcceptanceTest extends DestinationAcce private void executeQuery(final String query) { try { - Databases.createDatabase( - "root", - "test", + final DSLContext dslContext = DSLContextFactory.create( + db.getUsername(), + db.getPassword(), + db.getDriverClassName(), String.format("jdbc:mysql://%s:%s/%s?useSSL=true&requireSSL=true&verifyServerCertificate=false", db.getHost(), db.getFirstMappedPort(), - db.getDatabaseName()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL).query( + db.getDatabaseName()), SQLDialect.MYSQL); + new Database(dslContext).query( ctx -> ctx .execute(query)); } catch (final SQLException e) { diff --git a/airbyte-integrations/connectors/destination-mysql/build.gradle b/airbyte-integrations/connectors/destination-mysql/build.gradle index ad7e68f4c40..94ca3433798 100644 --- a/airbyte-integrations/connectors/destination-mysql/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql/build.gradle @@ -19,7 +19,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mysql') - integrationTestJavaImplementation "org.testcontainers:mysql:1.15.3" + integrationTestJavaImplementation libs.testcontainers.mysql implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index da86b862edb..4675fd75c53 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; @@ -34,7 +35,7 @@ public class MySQLDestination extends AbstractJdbcDestination implements Destina public static final String SSL_KEY = "ssl"; public static final String USERNAME_KEY = "username"; - public static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + public static final String DRIVER_CLASS = DatabaseDriver.MYSQL.getDriverClassName(); static final Map DEFAULT_JDBC_PARAMETERS = ImmutableMap.of( // zero dates by default cannot be parsed into java date objects (they will throw an error) diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index 6016ca02c3b..9dcada9cbde 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -10,7 +10,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -27,6 +30,8 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -101,15 +106,15 @@ public class MySQLDestinationAcceptanceTest extends DestinationAcceptanceTest { } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( db.getUsername(), db.getPassword(), - String.format("jdbc:mysql://%s:%s/%s", + db.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), db.getHost(), db.getFirstMappedPort(), - db.getDatabaseName()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL).query( + db.getDatabaseName()), SQLDialect.MYSQL); + return new Database(dslContext).query( ctx -> ctx .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) @@ -162,15 +167,15 @@ public class MySQLDestinationAcceptanceTest extends DestinationAcceptanceTest { private void executeQuery(final String query) { try { - Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( "root", "test", - String.format("jdbc:mysql://%s:%s/%s", + db.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), db.getHost(), db.getFirstMappedPort(), - db.getDatabaseName()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL).query( + db.getDatabaseName()), SQLDialect.MYSQL); + new Database(dslContext).query( ctx -> ctx .execute(query)); } catch (final SQLException e) { diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java index 168e017eac9..b75aa323756 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java @@ -12,7 +12,8 @@ import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshTunnel; @@ -22,7 +23,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import javax.sql.DataSource; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; /** * Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file @@ -111,12 +115,14 @@ public abstract class SshMySQLDestinationAcceptanceTest extends DestinationAccep } private static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createMySqlDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + "com.mysql.cj.jdbc.Driver", String.format("jdbc:mysql://%s:%s", config.get("host").asText(), - config.get("port").asText())); + config.get("port").asText()), SQLDialect.MYSQL); + return new Database(dslContext); } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java index b60ac9b2950..8aa616a2b1e 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java @@ -7,13 +7,17 @@ package io.airbyte.integrations.destination.mysql; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -92,15 +96,15 @@ public class SslMySQLDestinationAcceptanceTest extends MySQLDestinationAcceptanc } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( db.getUsername(), db.getPassword(), + db.getDriverClassName(), String.format("jdbc:mysql://%s:%s/%s?useSSL=true&requireSSL=true&verifyServerCertificate=false", db.getHost(), db.getFirstMappedPort(), - db.getDatabaseName()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL).query( + db.getDatabaseName()), SQLDialect.DEFAULT); + return new Database(dslContext).query( ctx -> ctx .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) @@ -124,15 +128,15 @@ public class SslMySQLDestinationAcceptanceTest extends MySQLDestinationAcceptanc private void executeQuery(final String query) { try { - Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( "root", "test", + db.getDriverClassName(), String.format("jdbc:mysql://%s:%s/%s?useSSL=true&requireSSL=true&verifyServerCertificate=false", db.getHost(), db.getFirstMappedPort(), - db.getDatabaseName()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL).query( + db.getDatabaseName()), SQLDialect.DEFAULT); + new Database(dslContext).query( ctx -> ctx .execute(query)); } catch (final SQLException e) { diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle index 4d155c66429..f7a4beb847f 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle @@ -25,7 +25,7 @@ dependencies { testImplementation project(':airbyte-test-utils') testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.testcontainers:oracle-xe:1.16.0' + testImplementation libs.testcontainers.oracle.xe integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-oracle') diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index be74f4c06cf..a9bcfcd40a0 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -13,7 +13,10 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -24,6 +27,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.JSONFormat; import org.junit.Test; @@ -113,24 +118,30 @@ public class OracleStrictEncryptDestinationAcceptanceTest extends DestinationAcc private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT); - final List result = getDatabase(config).query(ctx -> ctx.fetch(query).stream().toList()); - return result - .stream() - .map(r -> r.formatJSON(JSON_FORMAT)) - .map(Jsons::deserialize) - .collect(Collectors.toList()); + + try (final DSLContext dslContext = getDslContext(config)) { + final List result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList()); + return result + .stream() + .map(r -> r.formatJSON(JSON_FORMAT)) + .map(Jsons::deserialize) + .collect(Collectors.toList()); + } } - private static Database getDatabase(final JsonNode config) { - return Databases.createDatabase( + private static Database getDatabase(final DSLContext dslContext) { + return new Database(dslContext); + } + + private static DSLContext getDslContext(final JsonNode config) { + return DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - null); + config.get("port").asInt(), + config.get("sid").asText()), null); } @Override @@ -144,14 +155,14 @@ public class OracleStrictEncryptDestinationAcceptanceTest extends DestinationAcc config = getConfig(db); - final Database database = getDatabase(config); - database.query( - ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); - database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = getDatabase(dslContext); + database.query( + ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); + database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); - database.close(); - - ((ObjectNode) config).put("schema", dbName); + ((ObjectNode) config).put("schema", dbName); + } } @Override @@ -163,18 +174,19 @@ public class OracleStrictEncryptDestinationAcceptanceTest extends DestinationAcc @Test public void testEncryption() throws SQLException { final String algorithm = "AES256"; - final JsonNode config = getConfig(); - - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); + final DataSource dataSource = + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()), + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); + final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -191,15 +203,16 @@ public class OracleStrictEncryptDestinationAcceptanceTest extends DestinationAcc final String algorithm = clone.get("encryption") .get("encryption_algorithm").asText(); - final JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); + final DataSource dataSource = + DataSourceFactory.create(config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()), + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); + final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index dd031f075ec..007d9c9e69c 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.testcontainers:oracle-xe:1.16.0' + testImplementation libs.testcontainers.oracle.xe integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-oracle') diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 8259258686d..ce3a1c6b0a4 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -7,6 +7,7 @@ package io.airbyte.integrations.destination.oracle; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.JavaBaseConstants; @@ -29,7 +30,7 @@ public class OracleDestination extends AbstractJdbcDestination implements Destin public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); - public static final String DRIVER_CLASS = "oracle.jdbc.OracleDriver"; + public static final String DRIVER_CLASS = DatabaseDriver.ORACLE.getDriverClassName(); public static final String COLUMN_NAME_AB_ID = "\"" + JavaBaseConstants.COLUMN_NAME_AB_ID.toUpperCase() + "\""; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java index 69794489206..cb8c9200873 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java @@ -11,7 +11,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import java.sql.SQLException; import java.util.List; @@ -30,14 +32,18 @@ public class NneOracleDestinationAcceptanceTest extends UnencryptedOracleDestina .put("encryption_algorithm", algorithm) .build())); - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - getAdditionalProperties(algorithm)); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()), + getAdditionalProperties(algorithm) + ) + ); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -63,14 +69,18 @@ public class NneOracleDestinationAcceptanceTest extends UnencryptedOracleDestina final String algorithm = clone.get("encryption") .get("encryption_algorithm").asText(); - final JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - getAdditionalProperties(algorithm)); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + clone.get("username").asText(), + clone.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + clone.get("host").asText(), + clone.get("port").asInt(), + clone.get("sid").asText()), + getAdditionalProperties(algorithm) + ) + ); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java index 133a44263c2..048f1d4773f 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -10,7 +10,8 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshBastionContainer; @@ -22,6 +23,7 @@ import java.io.IOException; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import org.jooq.DSLContext; import org.testcontainers.containers.Network; public abstract class SshOracleDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -155,15 +157,13 @@ public abstract class SshOracleDestinationAcceptanceTest extends DestinationAcce } private Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", + final DSLContext dslContext = DSLContextFactory.create( + config.get("username").asText(), config.get("password").asText(), DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - null); + config.get("port").asInt(), + config.get("sid").asText()), null); + return new Database(dslContext); } @Override diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index 1ec98f5aab5..268320d64ab 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -12,7 +12,10 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -21,6 +24,8 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataCompa import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.junit.Test; public class UnencryptedOracleDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -130,15 +135,13 @@ public class UnencryptedOracleDestinationAcceptanceTest extends DestinationAccep } private static Database getDatabase(final JsonNode config) { - return Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", + final DSLContext dslContext = DSLContextFactory.create( + config.get("username").asText(), config.get("password").asText(), DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - null); + config.get("port").asInt(), + config.get("sid").asText()), null); + return new Database(dslContext); } @Override @@ -157,8 +160,6 @@ public class UnencryptedOracleDestinationAcceptanceTest extends DestinationAccep ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName))); database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName))); - database.close(); - ((ObjectNode) config).put("schema", dbName); } @@ -172,13 +173,13 @@ public class UnencryptedOracleDestinationAcceptanceTest extends DestinationAccep public void testNoneEncryption() throws SQLException { final JsonNode config = getConfig(); - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); + final DataSource dataSource = + DataSourceFactory.create(config.get("username").asText(), config.get("password").asText(), DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText())); + final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle index ba26bb6486f..cc1e21d55da 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle @@ -18,7 +18,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3" + integrationTestJavaImplementation libs.testcontainers.postgresql implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java index ce71fbf7ec9..7861214ca8b 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java @@ -7,7 +7,9 @@ package io.airbyte.integrations.destination.postgres; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; @@ -16,6 +18,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; @@ -109,8 +112,15 @@ public class PostgresDestinationStrictEncryptAcceptanceTest extends DestinationA } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createPostgresDatabase(db.getUsername(), db.getPassword(), - db.getJdbcUrl()).query( + return new Database( + DSLContextFactory.create( + db.getUsername(), + db.getPassword(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + db.getJdbcUrl(), + SQLDialect.POSTGRES + ) + ).query( ctx -> ctx .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) .stream() diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index b15cb0f9fe9..487e065d86c 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -17,12 +17,12 @@ dependencies { testImplementation project(':airbyte-test-utils') - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.testcontainers.postgresql integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-postgres') - integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3" + integrationTestJavaImplementation libs.testcontainers.postgresql implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index 5d1daf58e72..0f90fad6250 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -7,6 +7,7 @@ package io.airbyte.integrations.destination.postgres; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.ssh.SshWrappedDestination; @@ -22,7 +23,7 @@ public class PostgresDestination extends AbstractJdbcDestination implements Dest private static final Logger LOGGER = LoggerFactory.getLogger(PostgresDestination.class); - public static final String DRIVER_CLASS = "org.postgresql.Driver"; + public static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); public static final String DATABASE_KEY = "database"; diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java index 00a8767eee0..643b8bdeac2 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java @@ -7,7 +7,9 @@ package io.airbyte.integrations.destination.postgres; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; @@ -15,6 +17,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataCompa import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; public class PostgresDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { @@ -108,8 +111,15 @@ public class PostgresDestinationAcceptanceTest extends JdbcDestinationAcceptance } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return Databases.createPostgresDatabase(db.getUsername(), db.getPassword(), - db.getJdbcUrl()).query(ctx -> { + return new Database( + DSLContextFactory.create( + db.getUsername(), + db.getPassword(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + db.getJdbcUrl(), + SQLDialect.POSTGRES + ) + ).query(ctx -> { ctx.execute("set time zone 'UTC';"); return ctx.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) .stream() diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java index 1f15f7b1575..e938456b230 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java @@ -9,7 +9,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshTunnel; @@ -19,6 +20,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataCompa import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; // todo (cgardens) - likely some of this could be further de-duplicated with @@ -109,11 +111,18 @@ public abstract class SshPostgresDestinationAcceptanceTest extends JdbcDestinati } private static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createPostgresDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", config.get("host").asText(), config.get("port").asText(), - config.get("database").asText())); + return new Database( + DSLContextFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()), + SQLDialect.POSTGRES + ) + ); } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index db9082f1371..2bcece2a15e 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -148,7 +148,7 @@ public class PostgresDestinationTest { .withState(new AirbyteStateMessage().withData(Jsons.jsonNode(ImmutableMap.of(SCHEMA_NAME + "." + STREAM_NAME, 10))))); consumer.close(); - final JdbcDatabase database = PostgreSQLContainerHelper.getJdbcDatabaseFromConfig(config); + final JdbcDatabase database = PostgreSQLContainerHelper.getJdbcDatabaseFromConfig(PostgreSQLContainerHelper.getDataSourceFromConfig(config)); final List actualRecords = database.bufferedResultSetQuery( connection -> connection.createStatement().executeQuery("SELECT * FROM public._airbyte_raw_id_and_name;"), diff --git a/airbyte-integrations/connectors/destination-pulsar/build.gradle b/airbyte-integrations/connectors/destination-pulsar/build.gradle index bb261f20318..6b5c89fdf9a 100644 --- a/airbyte-integrations/connectors/destination-pulsar/build.gradle +++ b/airbyte-integrations/connectors/destination-pulsar/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation 'org.apache.pulsar:pulsar-client:2.8.1' - testImplementation "org.testcontainers:pulsar:1.16.2" + testImplementation libs.testcontainers.pulsar integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-pulsar') diff --git a/airbyte-integrations/connectors/destination-redis/build.gradle b/airbyte-integrations/connectors/destination-redis/build.gradle index 664bf8c4611..d06899cc8af 100644 --- a/airbyte-integrations/connectors/destination-redis/build.gradle +++ b/airbyte-integrations/connectors/destination-redis/build.gradle @@ -11,7 +11,6 @@ application { def redisDriver = '3.7.0' def assertVersion = '3.21.0' -def testContainersVersion = '1.16.2' dependencies { implementation project(':airbyte-config:models') @@ -29,7 +28,7 @@ dependencies { // https://mvnrepository.com/artifact/org.assertj/assertj-core testImplementation "org.assertj:assertj-core:${assertVersion}" // https://mvnrepository.com/artifact/org.testcontainers/testcontainers - testImplementation "org.testcontainers:testcontainers:${testContainersVersion}" + testImplementation libs.testcontainers integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-redis') diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 7dee925f655..576323918e6 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -7,7 +7,9 @@ package io.airbyte.integrations.destination.redshift; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.integrations.destination.redshift.enums.RedshiftDataTmpTableMode; @@ -16,7 +18,7 @@ import java.util.Optional; public class RedshiftInsertDestination extends AbstractJdbcDestination { - private static final String DRIVER_CLASS = "com.amazon.redshift.jdbc.Driver"; + private static final String DRIVER_CLASS = DatabaseDriver.REDSHIFT.getDriverClassName(); private static final String USERNAME = "username"; private static final String PASSWORD = "password"; private static final String SCHEMA = "schema"; @@ -47,12 +49,14 @@ public class RedshiftInsertDestination extends AbstractJdbcDestination { public static JdbcDatabase getJdbcDatabase(final JsonNode config) { final var jdbcConfig = RedshiftInsertDestination.getJdbcConfig(config); - return Databases.createJdbcDatabase( + return new DefaultJdbcDatabase( + DataSourceFactory.create( jdbcConfig.get(USERNAME).asText(), jdbcConfig.has(PASSWORD) ? jdbcConfig.get(PASSWORD).asText() : null, - jdbcConfig.get(JDBC_URL).asText(), RedshiftInsertDestination.DRIVER_CLASS, - SSL_JDBC_PARAMETERS); + jdbcConfig.get(JDBC_URL).asText(), + SSL_JDBC_PARAMETERS) + ); } public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) { diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index c0187c793fe..bd5ea0c3d25 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -11,7 +11,8 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; @@ -148,15 +149,19 @@ public class RedshiftCopyDestinationAcceptanceTest extends JdbcDestinationAccept } protected Database getDatabase() { - return Databases.createDatabase( - baseConfig.get("username").asText(), - baseConfig.get("password").asText(), - String.format("jdbc:redshift://%s:%s/%s", - baseConfig.get("host").asText(), - baseConfig.get("port").asText(), - baseConfig.get("database").asText()), - "com.amazon.redshift.jdbc.Driver", null, - RedshiftInsertDestination.SSL_JDBC_PARAMETERS); + return new Database( + DSLContextFactory.create( + baseConfig.get("username").asText(), + baseConfig.get("password").asText(), + DatabaseDriver.REDSHIFT.getDriverClassName(), + String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + baseConfig.get("host").asText(), + baseConfig.get("port").asInt(), + baseConfig.get("database").asText()), + null, + RedshiftInsertDestination.SSL_JDBC_PARAMETERS + ) + ); } public RedshiftSQLNameTransformer getNamingResolver() { diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index 9882a51d6cd..0ffaf055f27 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -28,6 +28,7 @@ dependencies { implementation group: 'com.github.airbytehq', name: 'json-avro-converter', version: '1.0.1' testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation "org.mockito:mockito-inline:4.1.0" integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') diff --git a/airbyte-integrations/connectors/destination-scylla/build.gradle b/airbyte-integrations/connectors/destination-scylla/build.gradle index e1a9d6aab8d..0fceee39162 100644 --- a/airbyte-integrations/connectors/destination-scylla/build.gradle +++ b/airbyte-integrations/connectors/destination-scylla/build.gradle @@ -11,7 +11,6 @@ application { def scyllaDriver = '3.10.2-scylla-1' def assertVersion = '3.21.0' -def testContainersVersion = '1.16.2' dependencies { implementation project(':airbyte-config:models') @@ -24,7 +23,7 @@ dependencies { // https://mvnrepository.com/artifact/org.assertj/assertj-core testImplementation "org.assertj:assertj-core:${assertVersion}" // https://mvnrepository.com/artifact/org.testcontainers/testcontainers - testImplementation "org.testcontainers:testcontainers:${testContainersVersion}" + testImplementation libs.testcontainers diff --git a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java index c1687551b92..9507b5ae3f5 100644 --- a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java +++ b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java @@ -13,7 +13,6 @@ import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.Databases; import io.airbyte.db.SqlDatabase; import io.airbyte.db.bigquery.BigQueryDatabase; import io.airbyte.db.bigquery.BigQuerySourceOperations; @@ -60,7 +59,7 @@ public class BigQuerySource extends AbstractRelationalDbSource { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } - database.close(); } @Override diff --git a/airbyte-integrations/connectors/source-cockroachdb/build.gradle b/airbyte-integrations/connectors/source-cockroachdb/build.gradle index bcb93db191e..93e7d47c3bc 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/build.gradle +++ b/airbyte-integrations/connectors/source-cockroachdb/build.gradle @@ -17,11 +17,11 @@ dependencies { implementation project(':airbyte-integrations:connectors:source-relational-db') implementation 'org.apache.commons:commons-lang3:3.11' - implementation "org.postgresql:postgresql:42.3.1" + implementation libs.postgresql testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation "org.testcontainers:cockroachdb:1.15.3" + testImplementation libs.testcontainers.cockroachdb testImplementation 'org.apache.commons:commons-lang3:3.11' integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-cockroachdb') diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java index 571091d4046..620d2dca347 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java @@ -9,7 +9,9 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; @@ -37,7 +39,7 @@ public class CockroachDbSource extends AbstractJdbcSource { private static final Logger LOGGER = LoggerFactory.getLogger(CockroachDbSource.class); - static final String DRIVER_CLASS = "org.postgresql.Driver"; + static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); public static final List HOST_KEY = List.of("host"); public static final List PORT_KEY = List.of("port"); @@ -116,13 +118,16 @@ public class CockroachDbSource extends AbstractJdbcSource { public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(config); - final JdbcDatabase database = Databases.createJdbcDatabase( - jdbcConfig.get("username").asText(), - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - driverClass, - JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties"), - sourceOperations); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + jdbcConfig.get("username").asText(), + jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, + driverClass, + jdbcConfig.get("jdbc_url").asText(), + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties") + ), + sourceOperations + ); quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java index 472fae1798f..9f230810d16 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java @@ -10,7 +10,9 @@ import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; @@ -22,6 +24,8 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.CockroachContainer; @@ -48,27 +52,26 @@ public class CockroachDbSourceAcceptanceTest extends SourceAcceptanceTest { .put("ssl", false) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } @Override diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index 57abee37cb8..497444a640d 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -8,12 +8,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,15 +46,15 @@ public class CockroachDbSourceDatatypeTest extends AbstractSourceDatabaseTypeTes .build()); LOGGER.warn("PPP:config:" + config); - final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); + final Database database = new Database(dslContext); database.query(ctx -> ctx.fetch("CREATE SCHEMA TEST;")); database.query(ctx -> ctx.fetch("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');")); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java index 5cc4d38901d..a787f284aca 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java @@ -15,7 +15,8 @@ import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; @@ -81,13 +82,15 @@ class CockroachDbJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { .build()); final JsonNode jdbcConfig = getToDatabaseConfigFunction().apply(config); - database = Databases.createJdbcDatabase( - jdbcConfig.get("username").asText(), - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - getDriverClass(), - JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties")); - + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + jdbcConfig.get("username").asText(), + jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, + getDriverClass(), + jdbcConfig.get("jdbc_url").asText(), + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties") + ) + ); database.execute(connection -> connection.createStatement().execute("CREATE DATABASE " + config.get("database") + ";")); super.setup(); } diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java index 2099b5c4d27..e6290696e03 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java @@ -15,7 +15,9 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; @@ -33,6 +35,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -102,39 +106,43 @@ class CockroachDbSourceTest { dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); final JsonNode config = getConfig(PSQL_DB, dbName); - final Database database = getDatabaseFromConfig(config); - database.query(ctx -> { - ctx.fetch("CREATE DATABASE " + dbName + ";"); - ctx.fetch( - "CREATE TABLE id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE DATABASE " + dbName + ";"); + ctx.fetch( + "CREATE TABLE id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); + ctx.fetch( + "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - ctx.fetch( - "CREATE TABLE id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); - ctx.fetch( - "INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + ctx.fetch( + "CREATE TABLE id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); + ctx.fetch( + "INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - ctx.fetch( - "CREATE TABLE names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); - ctx.fetch( - "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); - return null; - }); - database.close(); + ctx.fetch( + "CREATE TABLE names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); + ctx.fetch( + "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); + return null; + }); + } } - private static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + private static Database getDatabase(final DSLContext dslContext) { + return new Database(dslContext); + } + + private static DSLContext getDslContext(final JsonNode config) { + return DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); } private JsonNode getConfig(final CockroachContainer psqlDb, final String dbName) { @@ -169,7 +177,8 @@ class CockroachDbSourceTest { // .withCommand("postgres -c client_encoding=sql_ascii") db.start(); final JsonNode config = getConfig(db); - try (final Database database = getDatabaseFromConfig(config)) { + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = getDatabase(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); ctx.fetch( @@ -208,26 +217,28 @@ class CockroachDbSourceTest { @Test void testDiscoverWithPermissions() throws Exception { - JsonNode config = getConfig(PSQL_DB, dbName); - final Database database = getDatabaseFromConfig(config); - database.query(ctx -> { - ctx.fetch( - "CREATE USER cock;"); - ctx.fetch( - "CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch( - "CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch( - "CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch("grant all on database " + dbName + " to cock;"); - ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;"); - ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;"); - return null; - }); + final JsonNode config = getConfig(PSQL_DB, dbName); + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch( + "CREATE USER cock;"); + ctx.fetch( + "CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch( + "CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch( + "CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch("grant all on database " + dbName + " to cock;"); + ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;"); + ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;"); + return null; + }); + } - List expected = List.of("id_and_name_perm1", "id_and_name_perm2"); + final List expected = List.of("id_and_name_perm1", "id_and_name_perm2"); - AirbyteCatalog airbyteCatalog = new CockroachDbSource().discover(getConfig(PSQL_DB, dbName, "cock")); + final AirbyteCatalog airbyteCatalog = new CockroachDbSource().discover(getConfig(PSQL_DB, dbName, "cock")); final List actualNamesWithPermission = airbyteCatalog .getStreams() diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle index a1fbedab5eb..883d5acaff7 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle @@ -22,7 +22,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(':airbyte-test-utils') - testImplementation "org.testcontainers:db2:1.15.3" + testImplementation libs.testcontainers.db2 integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-db2') diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java index e90d751cf02..c50258a6f38 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java @@ -6,6 +6,7 @@ package io.airbyte.integrations.source.db2_strict_encrypt; import com.fasterxml.jackson.databind.node.ArrayNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; @@ -17,7 +18,7 @@ import org.slf4j.LoggerFactory; public class Db2StrictEncryptSource extends SpecModifyingSource implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(Db2StrictEncryptSource.class); - public static final String DRIVER_CLASS = "com.ibm.db2.jcc.DB2Driver"; + public static final String DRIVER_CLASS = Db2Source.DRIVER_CLASS; public Db2StrictEncryptSource() { super(new Db2Source()); diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java index a21c00c2f01..50b56d3e53d 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java @@ -9,7 +9,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.db2.Db2Source; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -120,11 +121,14 @@ public class Db2StrictEncryptSourceCertificateAcceptanceTest extends SourceAccep db.getMappedPort(50000), config.get("db").asText()) + SSL_CONFIG; - database = Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - jdbcUrl, - Db2Source.DRIVER_CLASS); + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + Db2Source.DRIVER_CLASS, + jdbcUrl + ) + ); final String createSchemaQuery = String.format("CREATE SCHEMA %s", SCHEMA_NAME); final String createTableQuery1 = String diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index d5e33af7ac7..aae32ccedf7 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -21,7 +21,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(':airbyte-test-utils') - testImplementation "org.testcontainers:db2:1.15.3" + testImplementation libs.testcontainers.db2 integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-db2') diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java index d817b10f6ac..9264b08b5b2 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -34,7 +35,7 @@ import org.slf4j.LoggerFactory; public class Db2Source extends AbstractJdbcSource implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(Db2Source.class); - public static final String DRIVER_CLASS = "com.ibm.db2.jcc.DB2Driver"; + public static final String DRIVER_CLASS = DatabaseDriver.DB2.getDriverClassName(); public static final String USERNAME = "username"; public static final String PASSWORD = "password"; @@ -54,9 +55,9 @@ public class Db2Source extends AbstractJdbcSource implements Source { @Override public JsonNode toDatabaseConfig(final JsonNode config) { - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:db2://%s:%s/%s", + final StringBuilder jdbcUrl = new StringBuilder(String.format(DatabaseDriver.DB2.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("db").asText())); var result = Jsons.jsonNode(ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java index d51fe0242c0..9ba59ff17fa 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java @@ -11,7 +11,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.db2.Db2Source; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -62,7 +64,7 @@ public class Db2SourceAcceptanceTest extends SourceAcceptanceTest { return config; } - private JsonNode getConfig(String userName, String password) { + private JsonNode getConfig(final String userName, final String password) { return Jsons.jsonNode(ImmutableMap.builder() .put("host", db.getHost()) .put("port", db.getFirstMappedPort()) @@ -111,14 +113,17 @@ public class Db2SourceAcceptanceTest extends SourceAcceptanceTest { config = getConfig(db.getUsername(), db.getPassword()); - database = Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:db2://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("db").asText()), - Db2Source.DRIVER_CLASS); + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + Db2Source.DRIVER_CLASS, + String.format(DatabaseDriver.DB2.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("db").asText()) + ) + ); final String createSchemaQuery = String.format("CREATE SCHEMA %s", SCHEMA_NAME); final String createTableQuery1 = String @@ -160,10 +165,10 @@ public class Db2SourceAcceptanceTest extends SourceAcceptanceTest { @Test public void testCheckPrivilegesForUserWithLessPerm() throws Exception { createUser(LESS_PERMITTED_USER); - JsonNode config = getConfig(LESS_PERMITTED_USER, PASSWORD); + final JsonNode config = getConfig(LESS_PERMITTED_USER, PASSWORD); final List actualNamesWithPermission = getActualNamesWithPermission(config); - List expected = List.of(STREAM_NAME3, STREAM_NAME1); + final List expected = List.of(STREAM_NAME3, STREAM_NAME1); assertEquals(expected.size(), actualNamesWithPermission.size()); assertEquals(expected, actualNamesWithPermission); } @@ -172,21 +177,21 @@ public class Db2SourceAcceptanceTest extends SourceAcceptanceTest { public void testCheckPrivilegesForUserWithoutPerm() throws Exception { createUser(USER_WITH_OUT_PERMISSIONS); - JsonNode config = getConfig(USER_WITH_OUT_PERMISSIONS, PASSWORD); + final JsonNode config = getConfig(USER_WITH_OUT_PERMISSIONS, PASSWORD); final List actualNamesWithPermission = getActualNamesWithPermission(config); - List expected = Collections.emptyList(); + final List expected = Collections.emptyList(); assertEquals(0, actualNamesWithPermission.size()); assertEquals(expected, actualNamesWithPermission); } - private void createUser(String lessPermittedUser) throws IOException, InterruptedException { - String encryptedPassword = db.execInContainer("openssl", "passwd", PASSWORD).getStdout().replaceAll("\n", ""); + private void createUser(final String lessPermittedUser) throws IOException, InterruptedException { + final String encryptedPassword = db.execInContainer("openssl", "passwd", PASSWORD).getStdout().replaceAll("\n", ""); db.execInContainer("useradd", lessPermittedUser, "-p", encryptedPassword); } - private List getActualNamesWithPermission(JsonNode config) throws Exception { - AirbyteCatalog airbyteCatalog = new Db2Source().discover(config); + private List getActualNamesWithPermission(final JsonNode config) throws Exception { + final AirbyteCatalog airbyteCatalog = new Db2Source().discover(config); return airbyteCatalog .getStreams() .stream() diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java index f35bbb52e20..0f11b1fa1fc 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java @@ -9,7 +9,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.db2.Db2Source; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -119,11 +120,14 @@ public class Db2SourceCertificateAcceptanceTest extends SourceAcceptanceTest { config.get("db").asText()) + ":sslConnection=true;sslTrustStoreLocation=" + KEY_STORE_FILE_PATH + ";sslTrustStorePassword=" + TEST_KEY_STORE_PASS + ";"; - database = Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - jdbcUrl, - Db2Source.DRIVER_CLASS); + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + Db2Source.DRIVER_CLASS, + jdbcUrl + ) + ); final String createSchemaQuery = String.format("CREATE SCHEMA %s", SCHEMA_NAME); final String createTableQuery1 = String diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 0efdb43f756..32f01908501 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -8,12 +8,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.source.db2.Db2Source; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.Db2Container; @@ -56,15 +60,15 @@ public class Db2SourceDatatypeTest extends AbstractSourceDatabaseTypeTest { .build())) .build()); - final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:db2://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("db").asText()), Db2Source.DRIVER_CLASS, - SQLDialect.DEFAULT); + String.format(DatabaseDriver.DB2.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("db").asText()), SQLDialect.DEFAULT); + final Database database = new Database(dslContext); database.query(ctx -> ctx.fetch("CREATE SCHEMA TEST")); diff --git a/airbyte-integrations/connectors/source-jdbc/build.gradle b/airbyte-integrations/connectors/source-jdbc/build.gradle index 44813fddb4d..b6aa9b2290c 100644 --- a/airbyte-integrations/connectors/source-jdbc/build.gradle +++ b/airbyte-integrations/connectors/source-jdbc/build.gradle @@ -27,11 +27,11 @@ dependencies { testImplementation project(':airbyte-test-utils') - testImplementation "org.postgresql:postgresql:42.2.18" - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.postgresql + testImplementation libs.testcontainers.postgresql integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3" + integrationTestJavaImplementation libs.testcontainers.postgresql testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" testFixturesImplementation project(':airbyte-protocol:models') diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 0f2aeac9531..a3f8381b7ce 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -25,11 +25,12 @@ import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.Databases; import io.airbyte.db.JdbcCompatibleSourceOperations; import io.airbyte.db.SqlDatabase; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.jdbc.streaming.JdbcStreamingQueryConfig; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; @@ -289,15 +290,17 @@ public abstract class AbstractJdbcSource extends AbstractRelationalDbS @Override public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(config); - - final JdbcDatabase database = Databases.createStreamingJdbcDatabase( - jdbcConfig.has("username") ? jdbcConfig.get("username").asText() : null, - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - driverClass, - streamingQueryConfigProvider, - JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties", getJdbcParameterDelimiter()), - sourceOperations); + final JdbcDatabase database = new StreamingJdbcDatabase( + DataSourceFactory.create( + jdbcConfig.has("username") ? jdbcConfig.get("username").asText() : null, + jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, + driverClass, + jdbcConfig.get("jdbc_url").asText(), + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties", getJdbcParameterDelimiter()) + ), + sourceOperations, + streamingQueryConfigProvider + ); quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); diff --git a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java index 1541ae6ce95..30e63c3797d 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java @@ -9,7 +9,8 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; @@ -19,6 +20,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; import java.util.HashMap; +import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; /** @@ -48,10 +50,15 @@ public class JdbcSourceSourceAcceptanceTest extends SourceAcceptanceTest { container.getDatabaseName())) .build()); - database = Databases.createPostgresDatabase( - config.get("username").asText(), - config.get("password").asText(), - config.get("jdbc_url").asText()); + database = new Database( + DSLContextFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + config.get("jdbc_url").asText(), + SQLDialect.POSTGRES + ) + ); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java index d81b0a3d466..1622cc4e79c 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -90,7 +91,7 @@ class AbstractJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresTestSource.class); - static final String DRIVER_CLASS = "org.postgresql.Driver"; + static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); public PostgresTestSource() { super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); @@ -100,9 +101,9 @@ class AbstractJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { public JsonNode toDatabaseConfig(final JsonNode config) { final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", String.format("jdbc:postgresql://%s:%s/%s", + .put("jdbc_url", String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("database").asText())); if (config.has("password")) { diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcStressTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcStressTest.java index 06ca30f2b3b..02cde9a867b 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcStressTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcStressTest.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -95,7 +96,7 @@ class DefaultJdbcStressTest extends JdbcStressTest { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresTestSource.class); - static final String DRIVER_CLASS = "org.postgresql.Driver"; + static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); public PostgresTestSource() { super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); @@ -105,9 +106,9 @@ class DefaultJdbcStressTest extends JdbcStressTest { public JsonNode toDatabaseConfig(final JsonNode config) { final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", String.format("jdbc:postgresql://%s:%s/%s", + .put("jdbc_url", String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("database").asText())); if (config.has("password")) { diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcSourceStressTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcSourceStressTest.java index 192fdb38a1e..7468a1d5544 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcSourceStressTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcSourceStressTest.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -93,7 +94,7 @@ class JdbcSourceStressTest extends JdbcStressTest { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresTestSource.class); - static final String DRIVER_CLASS = "org.postgresql.Driver"; + static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); public PostgresTestSource() { super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); @@ -103,9 +104,9 @@ class JdbcSourceStressTest extends JdbcStressTest { public JsonNode toDatabaseConfig(final JsonNode config) { final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", String.format("jdbc:postgresql://%s:%s/%s", + .put("jdbc_url", String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("database").asText())); if (config.has("password")) { diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index ed66b4f428d..467a26a43fd 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -22,10 +22,11 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; @@ -202,14 +203,17 @@ public abstract class JdbcSourceAcceptanceTest { streamName = TABLE_NAME; - database = Databases.createStreamingJdbcDatabase( - jdbcConfig.get("username").asText(), - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - getDriverClass(), - AdaptiveStreamingQueryConfig::new, - JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties", getJdbcParameterDelimiter()), - JdbcUtils.getDefaultSourceOperations()); + database = new StreamingJdbcDatabase( + DataSourceFactory.create( + jdbcConfig.get("username").asText(), + jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, + getDriverClass(), + jdbcConfig.get("jdbc_url").asText(), + JdbcUtils.parseJdbcParameters(jdbcConfig, "connection_properties", getJdbcParameterDelimiter()) + ), + JdbcUtils.getDefaultSourceOperations(), + AdaptiveStreamingQueryConfig::new + ); if (supportsSchemas()) { createSchemas(); diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java index 1870b64c87d..6ca768e5b1a 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java @@ -13,7 +13,8 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.MoreStreams; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.protocol.models.AirbyteCatalog; @@ -114,11 +115,14 @@ public abstract class JdbcStressTest { config = getConfig(); final JsonNode jdbcConfig = source.toDatabaseConfig(config); - final JdbcDatabase database = Databases.createJdbcDatabase( - jdbcConfig.get("username").asText(), - jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, - jdbcConfig.get("jdbc_url").asText(), - getDriverClass()); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + jdbcConfig.get("username").asText(), + jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, + getDriverClass(), + jdbcConfig.get("jdbc_url").asText() + ) + ); database.execute(connection -> connection.createStatement().execute( createTableQuery("id_and_name", String.format("id %s, name VARCHAR(200)", COL_ID_TYPE)))); diff --git a/airbyte-integrations/connectors/source-kafka/build.gradle b/airbyte-integrations/connectors/source-kafka/build.gradle index e42edca9723..78ef8ad884c 100644 --- a/airbyte-integrations/connectors/source-kafka/build.gradle +++ b/airbyte-integrations/connectors/source-kafka/build.gradle @@ -19,7 +19,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-kafka') - integrationTestJavaImplementation "org.testcontainers:kafka:1.15.3" + integrationTestJavaImplementation libs.testcontainers.kafka implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index 3f1e1c7b8a7..d503ce48b38 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation 'org.mongodb:mongodb-driver-sync:4.4.0' - testImplementation 'org.testcontainers:mongodb:1.15.3' + testImplementation libs.testcontainers.mongodb integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mongodb-v2') diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java index e4f4c73235d..73f34ce8485 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java @@ -13,7 +13,6 @@ import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.Databases; import io.airbyte.db.mongodb.MongoDatabase; import io.airbyte.db.mongodb.MongoUtils; import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType; @@ -81,7 +80,7 @@ public class MongoDbSource extends AbstractDbSource { @Override protected MongoDatabase createDatabase(final JsonNode config) throws Exception { final var dbConfig = toDatabaseConfig(config); - return Databases.createMongoDatabase(dbConfig.get("connectionString").asText(), + return new MongoDatabase(dbConfig.get("connectionString").asText(), dbConfig.get("database").asText()); } @@ -131,7 +130,7 @@ public class MongoDbSource extends AbstractDbSource { return tableInfos; } - private Set getAuthorizedCollections(MongoDatabase database) { + private Set getAuthorizedCollections(final MongoDatabase database) { /* * db.runCommand ({listCollections: 1.0, authorizedCollections: true, nameOnly: true }) the command * returns only those collections for which the user has privileges. For example, if a user has find @@ -139,7 +138,7 @@ public class MongoDbSource extends AbstractDbSource { * find or any other action, on the database resource, the command lists all collections in the * database. */ - Document document = database.getDatabase().runCommand(new Document("listCollections", 1) + final Document document = database.getDatabase().runCommand(new Document("listCollections", 1) .append("authorizedCollections", true) .append("nameOnly", true)) .append("filter", "{ 'type': 'collection' }"); diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle index 417845bb2ef..0a330893fe8 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle @@ -21,7 +21,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation "org.testcontainers:mssqlserver:1.15.3" + testImplementation libs.testcontainers.mssqlserver integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mssql-strict-encrypt') diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java index d853c616565..b344a5d2455 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java @@ -10,7 +10,9 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -21,7 +23,9 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; import java.util.HashMap; +import javax.sql.DataSource; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.utility.DockerImageName; @@ -66,14 +70,14 @@ public class MssqlStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest } private static Database getDatabase(final JsonNode baseConfig) { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( baseConfig.get("username").asText(), baseConfig.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), String.format("jdbc:sqlserver://%s:%s;encrypt=true;trustServerCertificate=true;", baseConfig.get("host").asText(), - baseConfig.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + baseConfig.get("port").asInt()), null); + return new Database(dslContext); } @Override diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptJdbcSourceAcceptanceTest.java index 5813742a207..02dcd381997 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -12,7 +12,9 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshHelpers; @@ -48,13 +50,16 @@ public class MssqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAccept .put("password", dbContainer.getPassword()) .build()); - database = Databases.createJdbcDatabase( - configWithoutDbName.get("username").asText(), - configWithoutDbName.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s", - configWithoutDbName.get("host").asText(), - configWithoutDbName.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver"); + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + configWithoutDbName.get("username").asText(), + configWithoutDbName.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d", + configWithoutDbName.get("host").asText(), + configWithoutDbName.get("port").asInt()) + ) + ); final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index 11bcc513555..5c6f970c0ea 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -11,7 +11,7 @@ application { } dependencies { - implementation "org.postgresql:postgresql:42.2.18" + implementation libs.postgresql implementation project(':airbyte-db:lib') implementation project(':airbyte-integrations:bases:base-java') @@ -27,7 +27,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation "org.testcontainers:mssqlserver:1.15.3" + testImplementation libs.testcontainers.mssqlserver integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') performanceTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index d8b97821ea4..5c1667c705a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -16,6 +16,7 @@ import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -50,7 +51,7 @@ public class MssqlSource extends AbstractJdbcSource implements Source private static final Logger LOGGER = LoggerFactory.getLogger(MssqlSource.class); - static final String DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + static final String DRIVER_CLASS = DatabaseDriver.MSSQLSERVER.getDriverClassName(); public static final String MSSQL_CDC_OFFSET = "mssql_cdc_offset"; public static final String MSSQL_DB_HISTORY = "mssql_db_history"; public static final String CDC_LSN = "_ab_cdc_lsn"; diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java index 1e869ebb777..b827e1e1f48 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java @@ -9,7 +9,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.base.ssh.SshTunnel; @@ -26,7 +28,9 @@ import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; import java.util.List; import java.util.Objects; +import javax.sql.DataSource; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.containers.Network; @@ -63,14 +67,14 @@ public abstract class AbstractSshMssqlSourceAcceptanceTest extends SourceAccepta } private static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s;", + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d;", config.get("host").asText(), - config.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + config.get("port").asInt()), null); + return new Database(dslContext); } private void startTestContainers() { diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java index 5b82dd8185a..c177f4db635 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java @@ -9,7 +9,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -22,6 +22,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.List; +import org.jooq.DSLContext; import org.testcontainers.containers.MSSQLServerContainer; public class CdcMssqlSourceAcceptanceTest extends SourceAcceptanceTest { @@ -91,14 +92,15 @@ public class CdcMssqlSourceAcceptanceTest extends SourceAcceptanceTest { container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); container.addEnv("MSSQL_AGENT_ENABLED", "True"); // need this running for cdc to work container.start(); - database = Databases.createDatabase( + + final DSLContext dslContext = DSLContextFactory.create( container.getUsername(), container.getPassword(), - String.format("jdbc:sqlserver://%s:%s", - container.getHost(), - container.getFirstMappedPort()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + container.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d;", + config.get("host").asText(), + config.get("port").asInt()), null); + database = new Database(dslContext); config = Jsons.jsonNode(ImmutableMap.builder() .put("host", container.getHost()) diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java index 5aec0eed0be..b625658b973 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java @@ -8,11 +8,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; +import org.jooq.DSLContext; import org.testcontainers.containers.MSSQLServerContainer; public class CdcMssqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { @@ -47,14 +49,14 @@ public class CdcMssqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { container.addEnv("MSSQL_AGENT_ENABLED", "True"); // need this running for cdc to work container.start(); - final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( container.getUsername(), container.getPassword(), - String.format("jdbc:sqlserver://%s:%s", - container.getHost(), - container.getFirstMappedPort()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + container.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%s;", + config.get("host").asText(), + config.get("port").asInt()), null); + final Database database = new Database(dslContext); config = Jsons.jsonNode(ImmutableMap.builder() .put("host", container.getHost()) @@ -78,14 +80,16 @@ public class CdcMssqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { } private void executeQuery(final String query) { - try (final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( + DataSourceFactory.create( container.getUsername(), container.getPassword(), - String.format("jdbc:sqlserver://%s:%s", - container.getHost(), - container.getFirstMappedPort()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null)) { + container.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d;", + config.get("host").asText(), + config.get("port").asInt())), null); + + try (final Database database = new Database(dslContext)) { database.query( ctx -> ctx .execute(query)); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlRdsSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlRdsSourceAcceptanceTest.java index c70ab223f90..001440f7392 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlRdsSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlRdsSourceAcceptanceTest.java @@ -9,11 +9,15 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import java.nio.file.Path; import java.sql.SQLException; +import javax.sql.DataSource; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; public class MssqlRdsSourceAcceptanceTest extends MssqlSourceAcceptanceTest { @@ -50,15 +54,15 @@ public class MssqlRdsSourceAcceptanceTest extends MssqlSourceAcceptanceTest { case "unencrypted" -> additionalParameter = "encrypt=false;"; case "encrypted_trust_server_certificate" -> additionalParameter = "encrypt=true;trustServerCertificate=true;"; } - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( baseConfig.get("username").asText(), baseConfig.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s;%s", + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d;%s", baseConfig.get("host").asText(), baseConfig.get("port").asInt(), - additionalParameter), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + additionalParameter), null); + return new Database(dslContext); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java index fdcaaf5215c..408581234b7 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java @@ -10,7 +10,9 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -21,6 +23,8 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; import java.util.HashMap; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.testcontainers.containers.MSSQLServerContainer; public class MssqlSourceAcceptanceTest extends SourceAcceptanceTest { @@ -96,14 +100,14 @@ public class MssqlSourceAcceptanceTest extends SourceAcceptanceTest { } private static Database getDatabase(final JsonNode config) { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s", + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d;", config.get("host").asText(), - config.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + config.get("port").asInt()), null); + return new Database(dslContext); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java index a55bba73c8b..4508c4a3645 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java @@ -10,11 +10,15 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.testcontainers.containers.MSSQLServerContainer; public class MssqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { @@ -50,14 +54,14 @@ public class MssqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { } private static Database getDatabase(final JsonNode config) { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s", + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d;", config.get("host").asText(), - config.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + config.get("port").asInt()), null); + return new Database(dslContext); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java index b1c8581e4e3..a2cb9a2bc11 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java @@ -9,10 +9,12 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import java.sql.SQLException; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.utility.DockerImageName; @@ -52,14 +54,14 @@ public class SslEnabledMssqlSourceAcceptanceTest extends MssqlSourceAcceptanceTe } private static Database getDatabase(final JsonNode baseConfig) { - return Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( baseConfig.get("username").asText(), baseConfig.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), String.format("jdbc:sqlserver://%s:%s;encrypt=true;trustServerCertificate=true;", baseConfig.get("host").asText(), - baseConfig.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + baseConfig.get("port").asInt()), null); + return new Database(dslContext); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java index ccc2996916e..0a7c760c5ce 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import java.util.stream.Stream; @@ -46,17 +47,15 @@ public class FillMsSqlTestDbScriptTest extends AbstractSourceFillDbWithTestData .put("replication_method", replicationMethod) .build()); - final Database database = Databases.createDatabase( + return new Database(DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), String.format("jdbc:sqlserver://%s:%s;databaseName=%s;", config.get("host").asText(), config.get("port").asInt(), dbName), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); - - return database; + null)); } /** diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java index b2b063aa69f..c526bfccdeb 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java @@ -21,12 +21,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.debezium.CdcSourceTest; @@ -39,6 +41,7 @@ import java.sql.SQLException; import java.util.List; import java.util.Map; import java.util.Optional; +import org.jooq.DSLContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -57,6 +60,7 @@ public class CdcMssqlSourceTest extends CdcSourceTest { private JdbcDatabase testJdbcDatabase; private MssqlSource source; private JsonNode config; + private DSLContext dslContext; @BeforeEach public void setup() throws SQLException { @@ -85,22 +89,24 @@ public class CdcMssqlSourceTest extends CdcSourceTest { .put("replication_method", "CDC") .build()); - database = Databases.createDatabase( + dslContext = DSLContextFactory.create( container.getUsername(), container.getPassword(), + DRIVER_CLASS, String.format("jdbc:sqlserver://%s:%s", container.getHost(), container.getFirstMappedPort()), - DRIVER_CLASS, null); - testJdbcDatabase = Databases.createJdbcDatabase( + database = new Database(dslContext); + + testJdbcDatabase = new DefaultJdbcDatabase(DataSourceFactory.create( TEST_USER_NAME, TEST_USER_PASSWORD, + DRIVER_CLASS, String.format("jdbc:sqlserver://%s:%s", container.getHost(), - container.getFirstMappedPort()), - DRIVER_CLASS); + container.getFirstMappedPort()))); executeQuery("CREATE DATABASE " + dbName + ";"); switchSnapshotIsolation(true, dbName); @@ -203,7 +209,8 @@ public class CdcMssqlSourceTest extends CdcSourceTest { @AfterEach public void tearDown() { try { - database.close(); + dslContext.close(); + testJdbcDatabase.close(); container.close(); } catch (final Exception e) { throw new RuntimeException(e); @@ -309,17 +316,17 @@ public class CdcMssqlSourceTest extends CdcSourceTest { } catch (final InterruptedException e) { throw new RuntimeException(e); } - final JdbcDatabase jdbcDatabase = Databases.createStreamingJdbcDatabase( - config.get("username").asText(), + final JdbcDatabase jdbcDatabase = new StreamingJdbcDatabase( + DataSourceFactory.create(config.get("username").asText(), config.get("password").asText(), + DRIVER_CLASS, String.format("jdbc:sqlserver://%s:%s;databaseName=%s;", config.get("host").asText(), config.get("port").asInt(), - dbName), - DRIVER_CLASS, - AdaptiveStreamingQueryConfig::new, - Maps.newHashMap(), - new MssqlSourceOperations()); + dbName)), + new MssqlSourceOperations(), + AdaptiveStreamingQueryConfig::new + ); return MssqlCdcTargetPosition.getTargetPosition(jdbcDatabase, dbName); } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java index b9023997932..72937e76fef 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java @@ -9,7 +9,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; @@ -40,13 +42,16 @@ public class MssqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { .put("password", dbContainer.getPassword()) .build()); - database = Databases.createJdbcDatabase( - configWithoutDbName.get("username").asText(), - configWithoutDbName.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s", - configWithoutDbName.get("host").asText(), - configWithoutDbName.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver"); + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + configWithoutDbName.get("username").asText(), + configWithoutDbName.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d", + configWithoutDbName.get("host").asText(), + configWithoutDbName.get("port").asInt()) + ) + ); final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java index f94365f2662..0485ca54c7d 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java @@ -13,7 +13,8 @@ import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; @@ -108,14 +109,16 @@ class MssqlSourceTest { public static Database getDatabase(final JsonNode config) { // todo (cgardens) - rework this abstraction so that we do not have to pass a null into the // constructor. at least explicitly handle it, even if the impl doesn't change. - return Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s", - config.get("host").asText(), - config.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver", - null); + return new Database( + DSLContextFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%s", + config.get("host").asText(), + config.get("port").asInt()), + null) + ); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java index f8c1b5d8954..3f6e0a7b079 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java @@ -9,7 +9,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; @@ -43,13 +45,16 @@ public class MssqlStressTest extends JdbcStressTest { .put("password", dbContainer.getPassword()) .build()); - database = Databases.createJdbcDatabase( - configWithoutDbName.get("username").asText(), - configWithoutDbName.get("password").asText(), - String.format("jdbc:sqlserver://%s:%s", - configWithoutDbName.get("host").asText(), - configWithoutDbName.get("port").asInt()), - "com.microsoft.sqlserver.jdbc.SQLServerDriver"); + database = new DefaultJdbcDatabase( + DataSourceFactory.create( + configWithoutDbName.get("username").asText(), + configWithoutDbName.get("password").asText(), + DatabaseDriver.MSSQLSERVER.getDriverClassName(), + String.format("jdbc:sqlserver://%s:%d", + configWithoutDbName.get("host").asText(), + configWithoutDbName.get("port").asInt()) + ) + ); final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle index a1590a593fc..6696af405ae 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle @@ -21,7 +21,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(':airbyte-test-utils') - testImplementation 'org.testcontainers:mysql:1.15.3' + testImplementation libs.testcontainers.mysql integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java index f46ce730bbf..80f3e8adec3 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java @@ -12,7 +12,9 @@ import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.mysql.MySqlSource; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -26,6 +28,8 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -51,26 +55,25 @@ public class MySqlStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest .put("replication_method", MySqlSource.ReplicationMethod.STANDARD) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), - config.get("password").asText(), + "", + DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s/%s?%s", config.get("host").asText(), config.get("port").asText(), config.get("database").asText(), - String.join("&", SSL_PARAMETERS)), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL); + String.join("&", SSL_PARAMETERS)), SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } @Override diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index d6d38b2419c..48235e1395e 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -13,7 +13,8 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; @@ -22,6 +23,7 @@ import io.airbyte.protocol.models.ConnectorSpecification; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -37,6 +39,7 @@ class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTes protected static MySQLContainer container; protected Database database; + protected DSLContext dslContext; @BeforeAll static void init() throws SQLException { @@ -60,29 +63,27 @@ class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTes .put("password", TEST_PASSWORD) .build()); - database = Databases.createDatabase( + dslContext = DSLContextFactory.create( config.get("username").asText(), - config.get("password").asText(), + "", + DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s?%s", config.get("host").asText(), config.get("port").asText(), - String.join("&", SSL_PARAMETERS)), - MySqlSource.DRIVER_CLASS, - - SQLDialect.MYSQL); + String.join("&", SSL_PARAMETERS)), SQLDialect.MYSQL); + database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE DATABASE " + config.get("database").asText()); return null; }); - database.close(); super.setup(); } @AfterEach void tearDownMySql() throws Exception { - database.close(); + dslContext.close(); super.tearDown(); } diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 296f77ab2d2..01bd7d91a5c 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -24,7 +24,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:bases:debezium')) testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.testcontainers:mysql:1.15.3' + testImplementation libs.testcontainers.mysql integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mysql') diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 9afbfc53926..bc34c4270d1 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -16,6 +16,7 @@ import com.mysql.cj.MysqlType; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -47,7 +48,7 @@ public class MySqlSource extends AbstractJdbcSource implements Source private static final Logger LOGGER = LoggerFactory.getLogger(MySqlSource.class); - public static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + public static final String DRIVER_CLASS = DatabaseDriver.MYSQL.getDriverClassName(); public static final String MYSQL_CDC_OFFSET = "mysql_cdc_offset"; public static final String MYSQL_DB_HISTORY = "mysql_db_history"; public static final String CDC_LOG_FILE = "_ab_cdc_log_file"; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java index 04084d6a621..b50b8140187 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java @@ -13,7 +13,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -31,6 +32,7 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.List; import java.util.stream.Collectors; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -130,15 +132,15 @@ public class CdcMySqlSourceAcceptanceTest extends SourceAcceptanceTest { } private void executeQuery(final String query) { - try (final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( "root", "test", - String.format("jdbc:mysql://%s:%s/%s", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), container.getHost(), container.getFirstMappedPort(), - container.getDatabaseName()), - MySqlSource.DRIVER_CLASS, - SQLDialect.MYSQL)) { + container.getDatabaseName()), SQLDialect.MYSQL); + try (final Database database = new Database(dslContext)) { database.query( ctx -> ctx .execute(query)); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java index 40453e22801..d78864d7aed 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceDatatypeTest.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -18,6 +19,7 @@ import java.io.IOException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,15 +61,15 @@ public class CdcMySqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { .put("replication_method", MySqlSource.ReplicationMethod.CDC) .build()); - final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:mysql://%s:%s/%s", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.MYSQL); + final Database database = new Database(dslContext); // It disable strict mode in the DB and allows to insert specific values. // For example, it's possible to insert date with zero values "2021-00-00" @@ -95,15 +97,16 @@ public class CdcMySqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { } private void executeQuery(final String query) { - try (final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( "root", "test", - String.format("jdbc:mysql://%s:%s/%s", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), container.getHost(), container.getFirstMappedPort(), - container.getDatabaseName()), - MySqlSource.DRIVER_CLASS, - SQLDialect.MYSQL)) { + container.getDatabaseName()), SQLDialect.MYSQL); + + try (final Database database = new Database(dslContext)) { database.query( ctx -> ctx .execute(query)); @@ -409,10 +412,10 @@ public class CdcMySqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { } private String getFileDataInBase64() { - File file = new File(getClass().getClassLoader().getResource("test.png").getFile()); + final File file = new File(getClass().getClassLoader().getResource("test.png").getFile()); try { return Base64.encodeBase64String(FileUtils.readFileToByteArray(file)); - } catch (IOException e) { + } catch (final IOException e) { LOGGER.error(String.format("Fail to read the file: %s. Error: %s", file.getAbsoluteFile(), e.getMessage())); } return null; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java index d3966a335a7..6c6b7708aa7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java @@ -9,7 +9,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -23,6 +24,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -48,25 +50,24 @@ public class MySqlSourceAcceptanceTest extends SourceAcceptanceTest { .put("replication_method", ReplicationMethod.STANDARD) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:mysql://%s:%s/%s", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } @Override diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java index dbbbc529b76..8867e857e83 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java @@ -9,7 +9,8 @@ import com.google.common.collect.ImmutableMap; import com.mysql.cj.MysqlType; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; @@ -66,16 +67,18 @@ public class MySqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { .put("replication_method", ReplicationMethod.STANDARD) .build()); - final Database database = Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:mysql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull")); + final Database database = new Database( + DSLContextFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()), + SQLDialect.MYSQL, + Map.of("zeroDateTimeBehavior", "convertToNull")) + ); // It disable strict mode in the DB and allows to insert specific values. // For example, it's possible to insert date with zero values "2021-00-00" diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java index e33e39d269d..48e693bda70 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java @@ -9,9 +9,11 @@ import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -32,28 +34,27 @@ public class MySqlSslSourceAcceptanceTest extends MySqlSourceAcceptanceTest { .put("replication_method", ReplicationMethod.STANDARD) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s/%s?%s", config.get("host").asText(), config.get("port").asText(), config.get("database").asText(), - String.join("&", SSL_PARAMETERS)), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL); + String.join("&", SSL_PARAMETERS)), SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java index f2a3ca89086..b7d1d8a94c5 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; @@ -45,16 +46,18 @@ public class FillMySqlTestDbScriptTest extends AbstractSourceFillDbWithTestData .put("replication_method", ReplicationMethod.STANDARD) .build()); - final Database database = Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:mysql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - dbName), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull")); + final Database database = new Database( + DSLContextFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()), + SQLDialect.MYSQL, + Map.of("zeroDateTimeBehavior", "convertToNull")) + ); // It disable strict mode in the DB and allows to insert specific values. // For example, it's possible to insert date with zero values "2021-00-00" diff --git a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java index 2d95fe80bd5..163924e7586 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java @@ -9,11 +9,13 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; import java.nio.file.Path; import java.util.Map; import java.util.stream.Stream; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.params.provider.Arguments; @@ -39,21 +41,23 @@ public class MySqlRdsSourcePerformanceSecretTest extends AbstractSourcePerforman .put("replication_method", plainConfig.get("replication_method")) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:mysql://%s:%s/%s", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - dbName), - "com.mysql.cj.jdbc.Driver", + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull")); + Map.of("zeroDateTimeBehavior", "convertToNull"))) { - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.execute("SET @@sql_mode=''")); - database.close(); + final Database database = new Database(dslContext); + + // It disable strict mode in the DB and allows to insert specific values. + // For example, it's possible to insert date with zero values "2021-00-00" + database.query(ctx -> ctx.execute("SET @@sql_mode=''")); + } } /** diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index 0b4525659f3..7cf4d9b0e32 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -13,13 +13,15 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.protocol.models.ConnectorSpecification; import java.sql.Connection; import java.sql.DriverManager; import java.util.concurrent.Callable; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -35,6 +37,7 @@ class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected static MySQLContainer container; protected Database database; + protected DSLContext dslContext; @BeforeAll static void init() throws Exception { @@ -58,28 +61,26 @@ class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { .put("password", TEST_PASSWORD.call()) .build()); - database = Databases.createDatabase( + dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", config.get("host").asText(), - config.get("port").asText()), - MySqlSource.DRIVER_CLASS, - - SQLDialect.MYSQL); + config.get("port").asText()), SQLDialect.MYSQL); + database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE DATABASE " + config.get("database").asText()); return null; }); - database.close(); super.setup(); } @AfterEach void tearDownMySql() throws Exception { - database.close(); + dslContext.close(); super.tearDown(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java index d6780431f48..c64bdee25ab 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java @@ -9,7 +9,10 @@ import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Databases; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.BeforeEach; @@ -26,23 +29,23 @@ class MySqlSslJdbcSourceAcceptanceTest extends MySqlJdbcSourceAcceptanceTest { .put("ssl", true) .build()); - database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:mysql://%s:%s?%s", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s?%s", config.get("host").asText(), config.get("port").asText(), - String.join("&", SSL_PARAMETERS)), - MySqlSource.DRIVER_CLASS, + config.get("database").asText(), + String.join("&", SSL_PARAMETERS)), SQLDialect.MYSQL)) { + database = new Database(dslContext); - SQLDialect.MYSQL); - - database.query(ctx -> { - ctx.fetch("CREATE DATABASE " + config.get("database").asText()); - ctx.fetch("SHOW STATUS LIKE 'Ssl_cipher'"); - return null; - }); - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE DATABASE " + config.get("database").asText()); + ctx.fetch("SHOW STATUS LIKE 'Ssl_cipher'"); + return null; + }); + } super.setup(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java index a33e9db6a72..645c0f57ada 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java @@ -10,13 +10,15 @@ import com.mysql.cj.MysqlType; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; import java.sql.Connection; import java.sql.DriverManager; import java.util.Optional; import java.util.concurrent.Callable; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -34,6 +36,7 @@ class MySqlStressTest extends JdbcStressTest { private JsonNode config; private Database database; + private DSLContext dslContext; @BeforeAll static void init() throws Exception { @@ -57,27 +60,26 @@ class MySqlStressTest extends JdbcStressTest { .put("password", TEST_PASSWORD.call()) .build()); - database = Databases.createDatabase( + dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), + DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", config.get("host").asText(), - config.get("port").asText()), - MySqlSource.DRIVER_CLASS, - SQLDialect.MYSQL); + config.get("port").asText()), SQLDialect.MYSQL); + database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE DATABASE " + config.get("database").asText()); return null; }); - database.close(); super.setup(); } @AfterEach - void tearDown() throws Exception { - database.close(); + void tearDown() { + dslContext.close(); } @AfterAll diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle index dc40076a4e4..7544d22f5da 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle @@ -27,7 +27,7 @@ dependencies { testImplementation project(':airbyte-test-utils') testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.testcontainers:oracle-xe:1.15.3' + testImplementation libs.testcontainers.oracle.xe integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java index 720226fffbc..895068e01ef 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java @@ -11,7 +11,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import java.sql.SQLException; @@ -31,16 +33,20 @@ public class OracleSourceNneAcceptanceTest extends OracleStrictEncryptSourceAcce final String algorithm = clone.get("encryption") .get("encryption_algorithm").asText(); - final JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + - "oracle.net.encryption_types_client=( " - + algorithm + " )")); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + clone.get("username").asText(), + clone.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + clone.get("host").asText(), + clone.get("port").asInt(), + clone.get("sid").asText()), + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + + "oracle.net.encryption_types_client=( " + + algorithm + " )") + ) + ); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -61,15 +67,19 @@ public class OracleSourceNneAcceptanceTest extends OracleStrictEncryptSourceAcce final String algorithm = clone.get("encryption") .get("encryption_algorithm").asText(); - final JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";")); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + clone.get("username").asText(), + clone.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + clone.get("host").asText(), + clone.get("port").asInt(), + clone.get("sid").asText()), + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + + "oracle.net.encryption_types_client=( " + algorithm + " )", ";") + ) + ); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java index 0106bd75a51..b0b640a5dde 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java @@ -9,7 +9,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.ssh.SshHelpers; @@ -22,8 +24,10 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; +import java.io.Closeable; import java.util.HashMap; import java.util.List; +import javax.sql.DataSource; public class OracleStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest { @@ -54,15 +58,19 @@ public class OracleStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTes .build())) .build()); - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), + final DataSource dataSource = DataSourceFactory.create( + config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( 3DES168 )", ";")); + "oracle.net.encryption_types_client=( 3DES168 )", ";") + ); + + final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); database.execute(connection -> { connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); @@ -76,7 +84,9 @@ public class OracleStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTes connection.createStatement().execute("INSERT INTO jdbc_space.starships (id, name) VALUES (3, 'yamato')"); }); - database.close(); + if(dataSource instanceof Closeable closeable) { + closeable.close(); + } } @Override diff --git a/airbyte-integrations/connectors/source-oracle/build.gradle b/airbyte-integrations/connectors/source-oracle/build.gradle index 7f67eb97119..593ede089ed 100644 --- a/airbyte-integrations/connectors/source-oracle/build.gradle +++ b/airbyte-integrations/connectors/source-oracle/build.gradle @@ -26,7 +26,7 @@ dependencies { testImplementation project(':airbyte-test-utils') testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.testcontainers:oracle-xe:1.15.3' + testImplementation libs.testcontainers.oracle.xe integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index acd8e04c6a5..ad6a0f3cc4a 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -7,6 +7,7 @@ package io.airbyte.integrations.source.oracle; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; @@ -33,7 +34,7 @@ public class OracleSource extends AbstractJdbcSource implements Source private static final Logger LOGGER = LoggerFactory.getLogger(OracleSource.class); - public static final String DRIVER_CLASS = "oracle.jdbc.OracleDriver"; + public static final String DRIVER_CLASS = DatabaseDriver.ORACLE.getDriverClassName(); private List schemas; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java index cb95a60ac93..a9a193fac7c 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java @@ -8,7 +8,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshHelpers; @@ -46,13 +48,17 @@ public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAccept } private void populateDatabaseTestData() throws Exception { - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()) + ) + ); database.execute(connection -> { connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java index 332b712840b..bc686fdc7ac 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java @@ -8,7 +8,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -51,13 +53,17 @@ public class OracleSourceAcceptanceTest extends SourceAcceptanceTest { .build())) .build()); - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()) + ) + ); database.execute(connection -> { connection.createStatement().execute("CREATE USER JDBC_SPACE IDENTIFIED BY JDBC_SPACE DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index b655321dc73..a3aa297d436 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -21,6 +22,7 @@ import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.TimeZone; +import org.jooq.DSLContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.OracleContainer; @@ -46,12 +48,15 @@ public class OracleSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { .put("schemas", List.of("TEST")) .build()); - final Database database = Databases.createOracleDatabase(config.get("username").asText(), + final DSLContext dslContext = DSLContextFactory.create( + config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText())); + config.get("port").asInt(), + config.get("sid").asText()), null); + final Database database = new Database(dslContext); LOGGER.warn("config: " + config); database.query(ctx -> ctx.fetch("CREATE USER test IDENTIFIED BY test DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS")); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java index aaabf381fcf..cc0e0842f5a 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java @@ -11,7 +11,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import java.sql.SQLException; @@ -31,16 +33,20 @@ public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest { final String algorithm = clone.get("encryption") .get("encryption_algorithm").asText(); - final JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + - "oracle.net.encryption_types_client=( " - + algorithm + " )")); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format("jdbc:oracle:thin:@//%s:%d/%s", + clone.get("host").asText(), + clone.get("port").asInt(), + clone.get("sid").asText()), + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + + "oracle.net.encryption_types_client=( " + + algorithm + " )") + ) + ); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -52,13 +58,17 @@ public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest { @Test public void testNoneEncrytion() throws SQLException { - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()) + ) + ); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -78,16 +88,19 @@ public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest { final String algorithm = clone.get("encryption") .get("encryption_algorithm").asText(); - final JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(), - clone.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - clone.get("host").asText(), - clone.get("port").asText(), - clone.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver", - JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + - "oracle.net.encryption_types_client=( " - + algorithm + " )")); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()), + JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + + "oracle.net.encryption_types_client=( " + + algorithm + " )") + )); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java index ff86b31c011..0b6e9194997 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java @@ -12,7 +12,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; @@ -70,13 +72,17 @@ class OracleSourceTest { .put("schemas", List.of("TEST")) .build()); - final JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:oracle:thin:@//%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("sid").asText()), - "oracle.jdbc.driver.OracleDriver"); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.ORACLE.getDriverClassName(), + String.format(DatabaseDriver.ORACLE.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("sid").asText()) + ) + ); database.execute(connection -> { connection.createStatement().execute("CREATE USER TEST IDENTIFIED BY TEST DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS"); diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle index 6a76c5a7f42..f75bae6518f 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle @@ -23,7 +23,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(':airbyte-test-utils') - testImplementation 'org.testcontainers:postgresql:1.15.3' + testImplementation libs.testcontainers.postgresql integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java index 262f870a2b6..47ae9869028 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java @@ -10,7 +10,8 @@ import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -23,6 +24,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; @@ -52,25 +54,24 @@ public class PostgresSourceStrictEncryptAcceptanceTest extends SourceAcceptanceT .put("replication_method", replicationMethod) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } @Override diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 9b70dbf458f..659190e8fcb 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -19,14 +19,14 @@ dependencies { implementation project(':airbyte-integrations:connectors:source-relational-db') implementation 'org.apache.commons:commons-lang3:3.11' - implementation "org.postgresql:postgresql:42.2.18" + implementation libs.postgresql testImplementation testFixtures(project(':airbyte-integrations:bases:debezium')) testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(":airbyte-json-validation") testImplementation project(':airbyte-test-utils') - testImplementation 'org.testcontainers:postgresql:1.15.3' + testImplementation libs.testcontainers.postgresql integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') performanceTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 87f6f3bc2b3..0a8ee9ff0b4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -17,6 +17,7 @@ import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -51,7 +52,7 @@ public class PostgresSource extends AbstractJdbcSource implements Sour private static final Logger LOGGER = LoggerFactory.getLogger(PostgresSource.class); public static final String CDC_LSN = "_ab_cdc_lsn"; - static final String DRIVER_CLASS = "org.postgresql.Driver"; + static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); private List schemas; public static Source sshWrappedSource() { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java index 4f0c3279308..583b9e9354c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.base.ssh.SshTunnel; @@ -24,6 +25,7 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; import java.util.List; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; @@ -58,25 +60,24 @@ public abstract class AbstractSshPostgresSourceAcceptanceTest extends SourceAcce } private static void populateDatabaseTestData() throws Exception { - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } @Override diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index 065fe9e587f..071327e2f41 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -9,7 +9,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -23,6 +24,7 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; import java.util.List; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; @@ -73,30 +75,30 @@ public class CdcPostgresSourceAcceptanceTest extends SourceAcceptanceTest { .put("ssl", false) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); - /** - * cdc expects the INCREMENTAL tables to contain primary key checkout - * {@link io.airbyte.integrations.source.postgres.PostgresSource#removeIncrementalWithoutPk(AirbyteStream)} - */ - database.query(ctx -> { - ctx.execute("SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); - ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); - ctx.execute("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.execute("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.execute("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.execute("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES)) { + final Database database = new Database(dslContext); - database.close(); + /** + * cdc expects the INCREMENTAL tables to contain primary key checkout + * {@link io.airbyte.integrations.source.postgres.PostgresSource#removeIncrementalWithoutPk(AirbyteStream)} + */ + database.query(ctx -> { + ctx.execute("SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); + ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); + ctx.execute("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.execute("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.execute("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.execute("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } @Override diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java index 8b7b239d288..616eeac5149 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java @@ -8,12 +8,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; import java.util.List; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; @@ -56,15 +58,15 @@ public class CdcPostgresSourceDatatypeTest extends AbstractSourceDatabaseTypeTes .put("ssl", false) .build()); - final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); + final Database database = new Database(dslContext); database.query(ctx -> { ctx.execute("SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index 01d1bafbaf1..268d8198ded 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -9,7 +9,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -23,6 +24,7 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; import java.util.List; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; @@ -53,26 +55,25 @@ public class PostgresSourceAcceptanceTest extends SourceAcceptanceTest { .put("replication_method", replicationMethod) .build()); - final Database database = Databases.createDatabase( + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - ctx.fetch("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + ctx.fetch("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); + return null; + }); + } } @Override diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java index 25ebd308cbb..1a1df3e7f85 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java @@ -8,13 +8,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; import java.util.Set; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; @@ -41,15 +43,15 @@ public class PostgresSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { .put("replication_method", replicationMethod) .build()); - final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); + final Database database = new Database(dslContext); database.query(ctx -> { ctx.execute(String.format("CREATE SCHEMA %S;", SCHEMA_NAME)); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java index 19caac05a62..ab16afce18e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java @@ -8,10 +8,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import java.util.stream.Stream; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.params.provider.Arguments; @@ -47,15 +49,15 @@ public class FillPostgresTestDbScriptTest extends AbstractSourceFillDbWithTestDa .put("replication_method", replicationMethod) .build()); - final Database database = Databases.createDatabase( + final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - dbName), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); + final Database database = new Database(dslContext); return database; } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index ac4c0c98b9a..1d5079a54fa 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -21,8 +21,11 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; import io.airbyte.db.PgLsn; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.debezium.CdcSourceTest; @@ -33,6 +36,7 @@ import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; import java.util.List; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -49,6 +53,7 @@ abstract class CdcPostgresSourceTest extends CdcSourceTest { private String dbName; private Database database; + private DSLContext dslContext; private PostgresSource source; private JsonNode config; @@ -56,7 +61,7 @@ abstract class CdcPostgresSourceTest extends CdcSourceTest { @AfterEach void tearDown() throws Exception { - database.close(); + dslContext.close(); container.close(); } @@ -76,7 +81,8 @@ abstract class CdcPostgresSourceTest extends CdcSourceTest { config = getConfig(dbName); final String fullReplicationSlot = SLOT_NAME_BASE + "_" + dbName; - database = getDatabaseFromConfig(config); + dslContext = getDslContext(config); + database = getDatabase(dslContext); database.query(ctx -> { ctx.execute("SELECT pg_create_logical_replication_slot('" + fullReplicationSlot + "', '" + getPluginName() + "');"); ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); @@ -106,16 +112,19 @@ abstract class CdcPostgresSourceTest extends CdcSourceTest { .build()); } - private Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + private static Database getDatabase(final DSLContext dslContext) { + return new Database(dslContext); + } + + private static DSLContext getDslContext(final JsonNode config) { + return DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); } @Test @@ -161,14 +170,18 @@ abstract class CdcPostgresSourceTest extends CdcSourceTest { @Override protected CdcTargetPosition cdcLatestTargetPosition() { - final JdbcDatabase database = Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver"); + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()) + ) + ); + return PostgresCdcTargetPosition.targetPosition(database); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java index 54021a83f23..d93ab136979 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java @@ -21,7 +21,9 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; @@ -36,6 +38,8 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -99,33 +103,40 @@ class PostgresSourceSSLTest { PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); final JsonNode config = getConfig(PSQL_DB, dbName); - final Database database = getDatabaseFromConfig(config); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); - ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); + ctx.fetch( + "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); - ctx.fetch("INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); + ctx.fetch( + "INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - ctx.fetch("CREATE TABLE names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); - ctx.fetch( - "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); - return null; - }); - database.close(); + ctx.fetch( + "CREATE TABLE names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); + ctx.fetch( + "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); + return null; + }); + } } - private Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + private static Database getDatabase(final DSLContext dslContext) { + return new Database(dslContext); + } + + private static DSLContext getDslContext(final JsonNode config) { + return DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s?sslmode=require", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); } private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName) { diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index b8b46b1b37a..8cf536879ed 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -21,7 +21,8 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; @@ -36,7 +37,10 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import javax.xml.crypto.Data; +import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.jooq.impl.DSL; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -125,45 +129,52 @@ class PostgresSourceTest { PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); final JsonNode config = getConfig(PSQL_DB, dbName); - final Database database = getDatabaseFromConfig(config); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); - ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); - ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); - ctx.fetch("INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));"); + ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); + ctx.fetch( + "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - ctx.fetch("CREATE TABLE names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); - ctx.fetch( - "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); - return null; - }); - database.close(); + ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);"); + ctx.fetch( + "INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); + + ctx.fetch( + "CREATE TABLE names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));"); + ctx.fetch( + "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); + return null; + }); + } } - private static Database getDatabaseWithSpecifiedUser(final JsonNode config, final String username, final String password) { - return Databases.createDatabase( + private static DSLContext getDslContextWithSpecifiedUser(final JsonNode config, final String username, final String password) { + return DSLContextFactory.create( username, password, - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); } - private static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + private static Database getDatabase(final DSLContext dslContext) { + return new Database(dslContext); + } + + private static DSLContext getDslContext(final JsonNode config) { + return DSLContextFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.POSTGRES); } private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName) { @@ -210,7 +221,8 @@ class PostgresSourceTest { try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withCommand("postgres -c client_encoding=sql_ascii")) { db.start(); final JsonNode config = getConfig(db); - try (final Database database = getDatabaseFromConfig(config)) { + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = getDatabase(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');"); @@ -231,7 +243,8 @@ class PostgresSourceTest { try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { db.start(); final JsonNode config = getConfig(db); - try (final Database database = getDatabaseFromConfig(config)) { + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'John'), (2, 'Alfred'), (3, 'Alex');"); @@ -241,7 +254,8 @@ class PostgresSourceTest { return null; }); } - try (final Database database = getDatabaseWithSpecifiedUser(config, "test_user_3", "132")) { + try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_3", "132")) { + final Database database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); ctx.fetch("CREATE VIEW id_and_name_3_view(id, name) as\n" @@ -279,7 +293,8 @@ class PostgresSourceTest { try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { db.start(); final JsonNode config = getConfig(db); - try (final Database database = getDatabaseFromConfig(config)) { + try (final DSLContext dslContext = getDslContext(config)) { + final Database database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name_7(id INTEGER, name VARCHAR(200));"); ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); @@ -303,7 +318,8 @@ class PostgresSourceTest { return null; }); } - try (final Database database = getDatabaseWithSpecifiedUser(config, "test_user_4", "132")) { + try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_4", "132")) { + final Database database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); return null; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java index 75773468b57..f7e60e15559 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; @@ -95,7 +96,7 @@ class PostgresStressTest extends JdbcStressTest { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresTestSource.class); - static final String DRIVER_CLASS = "org.postgresql.Driver"; + static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); public PostgresTestSource() { super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); @@ -105,9 +106,9 @@ class PostgresStressTest extends JdbcStressTest { public JsonNode toDatabaseConfig(final JsonNode config) { final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) - .put("jdbc_url", String.format("jdbc:postgresql://%s:%s/%s", + .put("jdbc_url", String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("database").asText())); if (config.has("password")) { diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java index b81e77dd71a..e5340d130f6 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java @@ -7,6 +7,7 @@ package io.airbyte.integrations.source.redshift; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; @@ -29,7 +30,7 @@ import org.slf4j.LoggerFactory; public class RedshiftSource extends AbstractJdbcSource implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSource.class); - public static final String DRIVER_CLASS = "com.amazon.redshift.jdbc.Driver"; + public static final String DRIVER_CLASS = DatabaseDriver.REDSHIFT.getDriverClassName(); private static final String SCHEMAS = "schemas"; private List schemas; @@ -45,9 +46,9 @@ public class RedshiftSource extends AbstractJdbcSource implements Sour final ImmutableMap.Builder builder = ImmutableMap.builder() .put("username", redshiftConfig.get("username").asText()) .put("password", redshiftConfig.get("password").asText()) - .put("jdbc_url", String.format("jdbc:redshift://%s:%s/%s", + .put("jdbc_url", String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), redshiftConfig.get("host").asText(), - redshiftConfig.get("port").asText(), + redshiftConfig.get("port").asInt(), redshiftConfig.get("database").asText())); if (redshiftConfig.has(SCHEMAS) && redshiftConfig.get(SCHEMAS).isArray()) { diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java index 59a7b01407f..69111c7b9e2 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java @@ -12,7 +12,9 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.redshift.RedshiftSource; @@ -79,14 +81,17 @@ public class RedshiftSourceAcceptanceTest extends SourceAcceptanceTest { } protected JdbcDatabase createDatabase(final JsonNode config) { - return Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:redshift://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - RedshiftSource.DRIVER_CLASS); + return new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + RedshiftSource.DRIVER_CLASS, + String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()) + ) + ); } protected void createTestUser(final JdbcDatabase database, final JsonNode config, final String testUserName, final String testUserPassword) diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java index a6f40182ee9..dca9f744716 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java @@ -5,25 +5,29 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.redshift.RedshiftSource; public class RedshiftSslSourceAcceptanceTest extends RedshiftSourceAcceptanceTest { @Override protected JdbcDatabase createDatabase(final JsonNode config) { - return Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:redshift://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - RedshiftSource.DRIVER_CLASS, - JdbcUtils.parseJdbcParameters("ssl=true&" + - "sslfactory=com.amazon.redshift.ssl.NonValidatingFactory")); + return new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.REDSHIFT.getDriverClassName(), + String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + config.get("host").asText(), + config.get("port").asInt(), + config.get("database").asText()), + JdbcUtils.parseJdbcParameters("ssl=true&" + + "sslfactory=com.amazon.redshift.ssl.NonValidatingFactory") + ) + ); } } diff --git a/airbyte-integrations/connectors/source-relational-db/build.gradle b/airbyte-integrations/connectors/source-relational-db/build.gradle index 958e6e7dd9b..acbf9bd2cbc 100644 --- a/airbyte-integrations/connectors/source-relational-db/build.gradle +++ b/airbyte-integrations/connectors/source-relational-db/build.gradle @@ -16,8 +16,8 @@ dependencies { testImplementation project(':airbyte-test-utils') - testImplementation "org.postgresql:postgresql:42.2.18" - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.postgresql + testImplementation libs.testcontainers.postgresql implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index a22abe03a10..0163044b233 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -11,6 +11,7 @@ import static io.airbyte.integrations.source.snowflake.SnowflakeDataSourceUtils. import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; @@ -30,7 +31,7 @@ import org.slf4j.LoggerFactory; public class SnowflakeSource extends AbstractJdbcSource implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeSource.class); - public static final String DRIVER_CLASS = "net.snowflake.client.jdbc.SnowflakeDriver"; + public static final String DRIVER_CLASS = DatabaseDriver.SNOWFLAKE.getDriverClassName(); public static final ScheduledExecutorService SCHEDULED_EXECUTOR_SERVICE = Executors.newScheduledThreadPool(1); public SnowflakeSource() { diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java index 9b676083f03..4d4c8a5b742 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java @@ -12,7 +12,9 @@ import com.google.common.collect.Lists; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.snowflake.SnowflakeSource; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; @@ -126,15 +128,17 @@ public class SnowflakeSourceAcceptanceTest extends SourceAcceptanceTest { protected JdbcDatabase setupDataBase() { config = Jsons.clone(getStaticConfig()); - return Databases.createJdbcDatabase( - config.get("credentials").get("username").asText(), - config.get("credentials").get("password").asText(), - String.format("jdbc:snowflake://%s/", - config.get("host").asText()), - SnowflakeSource.DRIVER_CLASS, - Map.of("role", config.get("role").asText(), - "warehouse", config.get("warehouse").asText(), - "database", config.get("database").asText())); + return new DefaultJdbcDatabase( + DataSourceFactory.create( + config.get("credentials").get("username").asText(), + config.get("credentials").get("password").asText(), + SnowflakeSource.DRIVER_CLASS, + String.format(DatabaseDriver.SNOWFLAKE.getUrlFormatString(), config.get("host").asText()), + Map.of("role", config.get("role").asText(), + "warehouse", config.get("warehouse").asText(), + "database", config.get("database").asText()) + ) + ); } @Test diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java index df49c9884d3..05a581e8417 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java @@ -8,7 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.source.snowflake.SnowflakeSource; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; @@ -17,7 +18,9 @@ import io.airbyte.protocol.models.JsonSchemaType; import java.nio.file.Path; import java.util.Map; import org.apache.commons.lang3.RandomStringUtils; +import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.BeforeEach; public class SnowflakeSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { @@ -27,6 +30,7 @@ public class SnowflakeSourceDatatypeTest extends AbstractSourceDatabaseTypeTest private JsonNode config; private Database database; + private DSLContext dslContext; @Override protected String getImageName() { @@ -50,12 +54,17 @@ public class SnowflakeSourceDatatypeTest extends AbstractSourceDatabaseTypeTest } private Database getDatabase() { - return Databases.createDatabase( + return new Database(dslContext); + } + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + super.setupEnvironment(environment); + dslContext = DSLContextFactory.create( config.get("credentials").get("username").asText(), config.get("credentials").get("password").asText(), - String.format("jdbc:snowflake://%s/", - config.get("host").asText()), SnowflakeSource.DRIVER_CLASS, + String.format(DatabaseDriver.SNOWFLAKE.getUrlFormatString(), config.get("host").asText()), SQLDialect.DEFAULT, Map.of( "role", config.get("role").asText(), @@ -69,7 +78,7 @@ public class SnowflakeSourceDatatypeTest extends AbstractSourceDatabaseTypeTest .format("DROP SCHEMA IF EXISTS %s", SCHEMA_NAME); database = getDatabase(); database.query(ctx -> ctx.fetch(dropSchemaQuery)); - database.close(); + dslContext.close(); } @Override diff --git a/airbyte-integrations/connectors/source-tidb/build.gradle b/airbyte-integrations/connectors/source-tidb/build.gradle index efcb2bf76b0..91f88f58cde 100755 --- a/airbyte-integrations/connectors/source-tidb/build.gradle +++ b/airbyte-integrations/connectors/source-tidb/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation 'mysql:mysql-connector-java:8.0.22' // Add testcontainers and use GenericContainer for TiDB - implementation "org.testcontainers:testcontainers:1.16.3" + implementation libs.testcontainers testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) @@ -29,7 +29,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-tidb') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation "org.testcontainers:testcontainers:1.16.3" + integrationTestJavaImplementation libs.testcontainers implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java index f10efc8b6e1..c74df03c7a0 100644 --- a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java +++ b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.mysql.cj.MysqlType; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; @@ -22,7 +23,7 @@ public class TiDBSource extends AbstractJdbcSource implements Source private static final Logger LOGGER = LoggerFactory.getLogger(TiDBSource.class); - static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + static final String DRIVER_CLASS = DatabaseDriver.MYSQL.getDriverClassName(); public static final List SSL_PARAMETERS = List.of( "useSSL=true", "requireSSL=true", @@ -38,9 +39,9 @@ public class TiDBSource extends AbstractJdbcSource implements Source @Override public JsonNode toDatabaseConfig(final JsonNode config) { - final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:mysql://%s:%s/%s", + final StringBuilder jdbcUrl = new StringBuilder(String.format(DatabaseDriver.MYSQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), + config.get("port").asInt(), config.get("database").asText())); if (config.get("jdbc_url_params") != null diff --git a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java index 7dd89937de5..834e55177ef 100755 --- a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java @@ -9,12 +9,21 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.*; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.SyncMode; import java.util.HashMap; +import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; @@ -39,25 +48,25 @@ public class TiDBSourceAcceptanceTest extends SourceAcceptanceTest { .put("username", "root") .put("database", "test") .build()); - final Database database = Databases.createDatabase( + + try (final DSLContext dslContext = DSLContextFactory.create( config.get("username").asText(), "", - String.format("jdbc:mysql://%s:%s/%s", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL); + config.get("port").asInt(), + config.get("database").asText()), SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - - database.close(); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } } @Override diff --git a/airbyte-metrics/lib/build.gradle b/airbyte-metrics/lib/build.gradle index fddad3b6e31..56a14d77370 100644 --- a/airbyte-metrics/lib/build.gradle +++ b/airbyte-metrics/lib/build.gradle @@ -11,5 +11,6 @@ dependencies { implementation 'com.datadoghq:java-dogstatsd-client:4.0.0' testImplementation project(':airbyte-config:persistence') - testImplementation 'org.testcontainers:postgresql:1.15.3' + testImplementation project(':airbyte-test-utils') + testImplementation libs.testcontainers.postgresql } diff --git a/airbyte-metrics/lib/src/test/java/io/airbyte/metrics/lib/MetrisQueriesTest.java b/airbyte-metrics/lib/src/test/java/io/airbyte/metrics/lib/MetricsQueriesTest.java similarity index 97% rename from airbyte-metrics/lib/src/test/java/io/airbyte/metrics/lib/MetrisQueriesTest.java rename to airbyte-metrics/lib/src/test/java/io/airbyte/metrics/lib/MetricsQueriesTest.java index 0dd5496868b..46bd59b1859 100644 --- a/airbyte-metrics/lib/src/test/java/io/airbyte/metrics/lib/MetrisQueriesTest.java +++ b/airbyte-metrics/lib/src/test/java/io/airbyte/metrics/lib/MetricsQueriesTest.java @@ -13,25 +13,30 @@ import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_CATALOG_FETCH_EVE import static io.airbyte.db.instance.configs.jooq.Tables.ACTOR_DEFINITION; import static io.airbyte.db.instance.configs.jooq.Tables.CONNECTION; import static io.airbyte.db.instance.configs.jooq.Tables.WORKSPACE; -import static io.airbyte.db.instance.jobs.jooq.Tables.*; +import static io.airbyte.db.instance.jobs.jooq.Tables.JOBS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.instance.configs.jooq.enums.ActorType; import io.airbyte.db.instance.configs.jooq.enums.NamespaceDefinitionType; import io.airbyte.db.instance.configs.jooq.enums.ReleaseStage; import io.airbyte.db.instance.configs.jooq.enums.StatusType; import io.airbyte.db.instance.jobs.jooq.enums.JobStatus; import io.airbyte.db.instance.test.TestDatabaseProviders; +import io.airbyte.test.utils.DatabaseConnectionHelper; import java.io.IOException; import java.sql.SQLException; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.UUID; +import javax.sql.DataSource; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.jooq.DSLContext; import org.jooq.JSONB; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; @@ -39,7 +44,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.testcontainers.containers.PostgreSQLContainer; -public class MetrisQueriesTest { +public class MetricsQueriesTest { private static final String USER = "user"; private static final String PASS = "hunter2"; @@ -51,12 +56,14 @@ public class MetrisQueriesTest { @BeforeAll static void setUpAll() throws IOException, SQLException { - PostgreSQLContainer container = new PostgreSQLContainer<>("postgres:13-alpine") + final PostgreSQLContainer container = new PostgreSQLContainer<>("postgres:13-alpine") .withUsername(USER) .withPassword(PASS); container.start(); - final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(container); + final DataSource dataSource = DatabaseConnectionHelper.createDataSource(container); + final DSLContext dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(dataSource, dslContext); configDb = databaseProviders.createNewConfigsDatabase(); databaseProviders.createNewJobsDatabase(); @@ -332,7 +339,7 @@ public class MetrisQueriesTest { @Test @DisplayName("should return only connections per workspace") void shouldReturnNumConnectionsBasic() throws SQLException { - var workspaceId = UUID.randomUUID(); + final var workspaceId = UUID.randomUUID(); configDb.transaction( ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.TOMBSTONE).values(workspaceId, "test-0", false) .execute()); @@ -363,7 +370,7 @@ public class MetrisQueriesTest { @Test @DisplayName("should ignore deleted connections") void shouldIgnoreNonRunningConnections() throws SQLException { - var workspaceId = UUID.randomUUID(); + final var workspaceId = UUID.randomUUID(); configDb.transaction( ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.TOMBSTONE).values(workspaceId, "test-0", false) .execute()); @@ -396,7 +403,7 @@ public class MetrisQueriesTest { @Test @DisplayName("should ignore deleted connections") void shouldIgnoreDeletedWorkspaces() throws SQLException { - var workspaceId = UUID.randomUUID(); + final var workspaceId = UUID.randomUUID(); configDb.transaction( ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.TOMBSTONE).values(workspaceId, "test-0", true) .execute()); diff --git a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java index be2a17ae645..4e5f13c335f 100644 --- a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java +++ b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java @@ -4,16 +4,23 @@ package io.airbyte.metrics.reporter; +import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.config.Configs; import io.airbyte.config.EnvConfigs; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.metrics.lib.DatadogClientConfiguration; import io.airbyte.metrics.lib.DogStatsDMetricSingleton; import io.airbyte.metrics.lib.MetricEmittingApps; import java.io.IOException; import java.util.concurrent.Executors; +import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; @Slf4j public class ReporterApp { @@ -25,18 +32,27 @@ public class ReporterApp { DogStatsDMetricSingleton.initialize(MetricEmittingApps.METRICS_REPORTER, new DatadogClientConfiguration(configs)); - configDatabase = new ConfigsDatabaseInstance( + final DataSource dataSource = DataSourceFactory.create( configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), - configs.getConfigDatabaseUrl()) - .getInitialized(); + DatabaseDriver.POSTGRESQL.getDriverClassName(), + configs.getConfigDatabaseUrl()); - final var toEmits = ToEmit.values(); - final var pollers = Executors.newScheduledThreadPool(toEmits.length); + try (final DSLContext dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES)) { - log.info("Scheduling {} metrics for emission..", toEmits.length); - for (ToEmit toEmit : toEmits) { - pollers.scheduleAtFixedRate(toEmit.emitRunnable, 0, toEmit.period, toEmit.timeUnit); + // Ensure that the database resources are closed on application shutdown + CloseableShutdownHook.registerRuntimeShutdownHook(dataSource, dslContext); + + configDatabase = new ConfigsDatabaseInstance(dslContext) + .getInitialized(); + + final var toEmits = ToEmit.values(); + final var pollers = Executors.newScheduledThreadPool(toEmits.length); + + log.info("Scheduling {} metrics for emission..", toEmits.length); + for (final ToEmit toEmit : toEmits) { + pollers.scheduleAtFixedRate(toEmit.emitRunnable, 0, toEmit.period, toEmit.timeUnit); + } } } diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 656dab8ecc6..b5f11046869 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -18,7 +18,7 @@ dependencies { implementation project(':airbyte-scheduler:persistence') implementation project(':airbyte-workers') - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.testcontainers.postgresql } application { diff --git a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java index 2f665b4e30c..81d55e9e19d 100644 --- a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java +++ b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java @@ -15,6 +15,7 @@ import io.airbyte.api.client.model.HealthCheckRead; import io.airbyte.commons.concurrency.GracefulShutdownHandler; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; import io.airbyte.config.Configs.WorkerEnvironment; @@ -26,6 +27,8 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.DatabaseConfigPersistence; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.jobs.JobsDatabaseInstance; import io.airbyte.metrics.lib.DatadogClientConfiguration; @@ -52,6 +55,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -76,6 +82,7 @@ public class SchedulerApp { private static final Duration SCHEDULING_DELAY = Duration.ofSeconds(5); private static final Duration CLEANING_DELAY = Duration.ofHours(2); private static final ThreadFactory THREAD_FACTORY = new ThreadFactoryBuilder().setNameFormat("worker-%d").build(); + private static final String DRIVER_CLASS_NAME = "org.postgresql.Driver"; private final Path workspaceRoot; private final JobPersistence jobPersistence; @@ -234,65 +241,71 @@ public class SchedulerApp { final String temporalHost = configs.getTemporalHost(); LOGGER.info("temporalHost = " + temporalHost); - // Wait for the server to initialize the database and run migration - // This should be converted into check for the migration version. Everything else as per. - waitForServer(configs); - LOGGER.info("Creating Job DB connection pool..."); - final Database jobDatabase = new JobsDatabaseInstance( - configs.getDatabaseUser(), - configs.getDatabasePassword(), - configs.getDatabaseUrl()) - .getInitialized(); + final DataSource configsDataSource = DataSourceFactory.create(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), + DRIVER_CLASS_NAME, configs.getConfigDatabaseUrl()); - final Database configDatabase = new ConfigsDatabaseInstance( - configs.getConfigDatabaseUser(), - configs.getConfigDatabasePassword(), - configs.getConfigDatabaseUrl()) - .getInitialized(); - final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); - final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder() - .maskSecrets(!featureFlags.exposeSecretsInExport()) - .copySecrets(true) - .build(); - final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor); - final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase); + final DataSource jobsDataSource = DataSourceFactory.create(configs.getDatabaseUser(), configs.getDatabasePassword(), + DRIVER_CLASS_NAME, configs.getDatabaseUrl()); - final JobPersistence jobPersistence = new DefaultJobPersistence(jobDatabase); - final JobCleaner jobCleaner = new JobCleaner( - configs.getWorkspaceRetentionConfig(), - workspaceRoot, - jobPersistence); - AirbyteVersion.assertIsCompatible( - configs.getAirbyteVersion(), - jobPersistence.getVersion().map(AirbyteVersion::new).orElseThrow()); + // Manual configuration that will be replaced by Dependency Injection in the future + try (final DSLContext configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); + final DSLContext jobsDslContext = DSLContextFactory.create(jobsDataSource, SQLDialect.POSTGRES)) { - TrackingClientSingleton.initialize( - configs.getTrackingStrategy(), - new Deployment(configs.getDeploymentMode(), jobPersistence.getDeployment().orElseThrow(), configs.getWorkerEnvironment()), - configs.getAirbyteRole(), - configs.getAirbyteVersion(), - configRepository); - final JobNotifier jobNotifier = new JobNotifier( - configs.getWebappUrl(), - configRepository, - new WorkspaceHelper(configRepository, jobPersistence), - TrackingClientSingleton.get()); - final TemporalClient temporalClient = TemporalClient.production(temporalHost, workspaceRoot, configs); + // Ensure that the database resources are closed on application shutdown + CloseableShutdownHook.registerRuntimeShutdownHook(configsDataSource, jobsDataSource, configsDslContext, jobsDslContext); - DogStatsDMetricSingleton.initialize(MetricEmittingApps.SCHEDULER, new DatadogClientConfiguration(configs)); + // Wait for the server to initialize the database and run migration + // This should be converted into check for the migration version. Everything else as per. + waitForServer(configs); + LOGGER.info("Creating Job DB connection pool..."); + final Database jobDatabase = new JobsDatabaseInstance(jobsDslContext).getInitialized(); - LOGGER.info("Launching scheduler..."); - new SchedulerApp( - workspaceRoot, - jobPersistence, - configRepository, - jobCleaner, - jobNotifier, - temporalClient, - Integer.parseInt(configs.getSubmitterNumThreads()), - configs.getSyncJobMaxAttempts(), - configs.getAirbyteVersionOrWarning(), configs.getWorkerEnvironment(), configs.getLogConfigs()) - .start(); + final Database configDatabase = new ConfigsDatabaseInstance(configsDslContext).getInitialized(); + final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); + final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder() + .maskSecrets(!featureFlags.exposeSecretsInExport()) + .copySecrets(true) + .build(); + final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor); + final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase); + + final JobPersistence jobPersistence = new DefaultJobPersistence(jobDatabase); + final JobCleaner jobCleaner = new JobCleaner( + configs.getWorkspaceRetentionConfig(), + workspaceRoot, + jobPersistence); + AirbyteVersion.assertIsCompatible( + configs.getAirbyteVersion(), + jobPersistence.getVersion().map(AirbyteVersion::new).orElseThrow()); + + TrackingClientSingleton.initialize( + configs.getTrackingStrategy(), + new Deployment(configs.getDeploymentMode(), jobPersistence.getDeployment().orElseThrow(), configs.getWorkerEnvironment()), + configs.getAirbyteRole(), + configs.getAirbyteVersion(), + configRepository); + final JobNotifier jobNotifier = new JobNotifier( + configs.getWebappUrl(), + configRepository, + new WorkspaceHelper(configRepository, jobPersistence), + TrackingClientSingleton.get()); + final TemporalClient temporalClient = TemporalClient.production(temporalHost, workspaceRoot, configs); + + DogStatsDMetricSingleton.initialize(MetricEmittingApps.SCHEDULER, new DatadogClientConfiguration(configs)); + + LOGGER.info("Launching scheduler..."); + new SchedulerApp( + workspaceRoot, + jobPersistence, + configRepository, + jobCleaner, + jobNotifier, + temporalClient, + Integer.parseInt(configs.getSubmitterNumThreads()), + configs.getSyncJobMaxAttempts(), + configs.getAirbyteVersionOrWarning(), configs.getWorkerEnvironment(), configs.getLogConfigs()) + .start(); + } } } diff --git a/airbyte-scheduler/persistence/build.gradle b/airbyte-scheduler/persistence/build.gradle index 73ae0358208..58cff030876 100644 --- a/airbyte-scheduler/persistence/build.gradle +++ b/airbyte-scheduler/persistence/build.gradle @@ -15,6 +15,7 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-scheduler:models') - testImplementation "org.flywaydb:flyway-core:7.14.0" - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.flyway.core + testImplementation libs.testcontainers.postgresql + testImplementation project(':airbyte-test-utils') } diff --git a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java index 81f22283719..1d283d767f4 100644 --- a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java +++ b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java @@ -31,6 +31,7 @@ import io.airbyte.config.JobGetSpecConfig; import io.airbyte.config.JobOutput; import io.airbyte.config.JobSyncConfig; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.instance.jobs.JobsDatabaseSchema; import io.airbyte.db.instance.test.TestDatabaseProviders; import io.airbyte.scheduler.models.Attempt; @@ -39,8 +40,10 @@ import io.airbyte.scheduler.models.AttemptWithJobInfo; import io.airbyte.scheduler.models.Job; import io.airbyte.scheduler.models.JobStatus; import io.airbyte.scheduler.models.JobWithStatusAndTimestamp; +import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; +import java.io.Closeable; import java.io.IOException; import java.nio.file.Path; import java.sql.SQLException; @@ -59,8 +62,11 @@ import java.util.UUID; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -98,9 +104,10 @@ class DefaultJobPersistenceTest { private static PostgreSQLContainer container; private Database jobDatabase; - private Database configDatabase; private Supplier timeSupplier; private JobPersistence jobPersistence; + private DataSource dataSource; + private DSLContext dslContext; @BeforeAll public static void dbSetup() { @@ -168,7 +175,9 @@ class DefaultJobPersistenceTest { @SuppressWarnings("unchecked") @BeforeEach public void setup() throws Exception { - final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(container); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(dataSource, dslContext); jobDatabase = databaseProviders.createNewJobsDatabase(); resetDb(); @@ -180,8 +189,11 @@ class DefaultJobPersistenceTest { } @AfterEach - void tearDown() throws Exception { - jobDatabase.close(); + void tearDown() throws IOException { + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } private void resetDb() throws SQLException { diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index e4e8d26966b..c108f322221 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -63,7 +63,7 @@ dependencies { implementation 'org.glassfish.jersey.inject:jersey-hk2' implementation 'org.glassfish.jersey.media:jersey-media-json-jackson' implementation 'org.glassfish.jersey.ext:jersey-bean-validation' - implementation "org.flywaydb:flyway-core:7.14.0" + implementation libs.flyway.core implementation 'com.github.slugify:slugify:2.4' @@ -85,11 +85,10 @@ dependencies { implementation project(':airbyte-scheduler:persistence') implementation project(':airbyte-workers') - testImplementation "org.postgresql:postgresql:42.2.18" - + testImplementation libs.postgresql testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' - - testImplementation "org.testcontainers:postgresql:1.15.3" + testImplementation libs.testcontainers.postgresql + testImplementation project(':airbyte-test-utils') } // we want to be able to access the generated db files from config/init when we build the server docker image. diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java index 53ab8a3ea48..a2290a9d378 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java @@ -25,6 +25,7 @@ import io.temporal.serviceclient.WorkflowServiceStubs; import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Map; +import org.flywaydb.core.Flyway; import org.glassfish.hk2.api.Factory; import org.slf4j.MDC; @@ -52,6 +53,8 @@ public class ConfigurationApiFactory implements Factory { private static HttpClient httpClient; private static FeatureFlags featureFlags; private static EventRunner eventRunner; + private static Flyway configsFlyway; + private static Flyway jobsFlyway; public static void setValues( final WorkflowServiceStubs temporalService, @@ -75,7 +78,9 @@ public class ConfigurationApiFactory implements Factory { final Path workspaceRoot, final HttpClient httpClient, final FeatureFlags featureFlags, - final EventRunner eventRunner) { + final EventRunner eventRunner, + final Flyway configsFlyway, + final Flyway jobsFlyway) { ConfigurationApiFactory.configRepository = configRepository; ConfigurationApiFactory.jobPersistence = jobPersistence; ConfigurationApiFactory.seed = seed; @@ -98,6 +103,8 @@ public class ConfigurationApiFactory implements Factory { ConfigurationApiFactory.httpClient = httpClient; ConfigurationApiFactory.featureFlags = featureFlags; ConfigurationApiFactory.eventRunner = eventRunner; + ConfigurationApiFactory.configsFlyway = configsFlyway; + ConfigurationApiFactory.jobsFlyway = jobsFlyway; } @Override @@ -125,7 +132,9 @@ public class ConfigurationApiFactory implements Factory { ConfigurationApiFactory.workspaceRoot, ConfigurationApiFactory.httpClient, ConfigurationApiFactory.featureFlags, - ConfigurationApiFactory.eventRunner); + ConfigurationApiFactory.eventRunner, + ConfigurationApiFactory.configsFlyway, + ConfigurationApiFactory.jobsFlyway); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 5095586381a..bfc1911cb5e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -9,6 +9,7 @@ import io.airbyte.analytics.TrackingClient; import io.airbyte.analytics.TrackingClientSingleton; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; @@ -26,6 +27,9 @@ import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.instance.DatabaseInstance; import io.airbyte.db.instance.MinimumFlywayMigrationVersionCheck; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; @@ -48,6 +52,7 @@ import io.airbyte.server.errors.InvalidJsonInputExceptionMapper; import io.airbyte.server.errors.KnownExceptionMapper; import io.airbyte.server.errors.NotFoundExceptionMapper; import io.airbyte.server.errors.UncaughtExceptionMapper; +import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.temporal.TemporalClient; @@ -60,13 +65,17 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import javax.sql.DataSource; import lombok.val; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.flywaydb.core.Flyway; import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletContainer; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -75,6 +84,7 @@ public class ServerApp implements ServerRunnable { private static final Logger LOGGER = LoggerFactory.getLogger(ServerApp.class); private static final int PORT = 8001; + private static final String DRIVER_CLASS_NAME = "org.postgresql.Driver"; private final AirbyteVersion airbyteVersion; private final Set> customComponentClasses; @@ -127,24 +137,36 @@ public class ServerApp implements ServerRunnable { private static void assertDatabasesReady(final Configs configs, final DatabaseInstance configsDatabaseInstance, - final DatabaseInstance jobsDatabaseInstance) + final DataSource configsDataSource, + final DatabaseInstance jobsDatabaseInstance, + final DataSource jobsDataSource) throws InterruptedException { LOGGER.info("Checking configs database flyway migration version.."); MinimumFlywayMigrationVersionCheck.assertDatabase(configsDatabaseInstance, MinimumFlywayMigrationVersionCheck.DEFAULT_ASSERT_DATABASE_TIMEOUT_MS); - val configsMigrator = new ConfigsDatabaseMigrator(configsDatabaseInstance.getInitialized(), ServerApp.class.getName()); + final Flyway configsFlyway = FlywayFactory.create(configsDataSource, ServerApp.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + val configsMigrator = new ConfigsDatabaseMigrator(configsDatabaseInstance.getInitialized(), configsFlyway); MinimumFlywayMigrationVersionCheck.assertMigrations(configsMigrator, configs.getConfigsDatabaseMinimumFlywayMigrationVersion(), configs.getConfigsDatabaseInitializationTimeoutMs()); LOGGER.info("Checking jobs database flyway migration version.."); MinimumFlywayMigrationVersionCheck.assertDatabase(jobsDatabaseInstance, MinimumFlywayMigrationVersionCheck.DEFAULT_ASSERT_DATABASE_TIMEOUT_MS); - val jobsMigrator = new JobsDatabaseMigrator(jobsDatabaseInstance.getInitialized(), ServerApp.class.getName()); + final Flyway jobsFlyway = FlywayFactory.create(jobsDataSource, ServerApp.class.getName(), JobsDatabaseMigrator.DB_IDENTIFIER, + JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); + val jobsMigrator = new JobsDatabaseMigrator(jobsDatabaseInstance.getInitialized(), jobsFlyway); MinimumFlywayMigrationVersionCheck.assertMigrations(jobsMigrator, configs.getJobsDatabaseMinimumFlywayMigrationVersion(), configs.getJobsDatabaseInitializationTimeoutMs()); } - public static ServerRunnable getServer(final ServerFactory apiFactory, final ConfigPersistence seed) throws Exception { - final Configs configs = new EnvConfigs(); + public static ServerRunnable getServer(final ServerFactory apiFactory, + final ConfigPersistence seed, + final Configs configs, + final DSLContext configsDslContext, + final DataSource configsDataSource, + final DSLContext jobsDslContext, + final DataSource jobsDataSource) + throws Exception { final WorkerConfigs workerConfigs = new WorkerConfigs(configs); LogClientSingleton.getInstance().setWorkspaceMdc( @@ -154,10 +176,10 @@ public class ServerApp implements ServerRunnable { LOGGER.info("Checking databases.."); final DatabaseInstance configsDatabaseInstance = - new ConfigsDatabaseInstance(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), configs.getConfigDatabaseUrl()); + new ConfigsDatabaseInstance(configsDslContext); final DatabaseInstance jobsDatabaseInstance = - new JobsDatabaseInstance(configs.getDatabaseUser(), configs.getDatabasePassword(), configs.getDatabaseUrl()); - assertDatabasesReady(configs, configsDatabaseInstance, jobsDatabaseInstance); + new JobsDatabaseInstance(jobsDslContext); + assertDatabasesReady(configs, configsDatabaseInstance, configsDataSource, jobsDatabaseInstance, jobsDataSource); LOGGER.info("Creating Staged Resource folder..."); ConfigDumpImporter.initStagedResourceFolder(); @@ -170,9 +192,9 @@ public class ServerApp implements ServerRunnable { .copySecrets(false) .build(); final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor); - final SecretsHydrator secretsHydrator = SecretPersistence.getSecretsHydrator(configs); - final Optional secretPersistence = SecretPersistence.getLongLived(configs); - final Optional ephemeralSecretPersistence = SecretPersistence.getEphemeral(configs); + final SecretsHydrator secretsHydrator = SecretPersistence.getSecretsHydrator(configsDslContext, configs); + final Optional secretPersistence = SecretPersistence.getLongLived(configsDslContext, configs); + final Optional ephemeralSecretPersistence = SecretPersistence.getEphemeral(configsDslContext, configs); final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase); final SecretsRepositoryReader secretsRepositoryReader = new SecretsRepositoryReader(configRepository, secretsHydrator); final SecretsRepositoryWriter secretsRepositoryWriter = @@ -206,6 +228,11 @@ public class ServerApp implements ServerRunnable { final EventRunner eventRunner = new TemporalEventRunner( TemporalClient.production(configs.getTemporalHost(), configs.getWorkspaceRoot(), configs)); + final Flyway configsFlyway = FlywayFactory.create(configsDataSource, DbMigrationHandler.class.getSimpleName(), + ConfigsDatabaseMigrator.DB_IDENTIFIER, ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final Flyway jobsFlyway = FlywayFactory.create(jobsDataSource, DbMigrationHandler.class.getSimpleName(), JobsDatabaseMigrator.DB_IDENTIFIER, + JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -228,7 +255,9 @@ public class ServerApp implements ServerRunnable { configs.getWorkspaceRoot(), httpClient, featureFlags, - eventRunner); + eventRunner, + configsFlyway, + jobsFlyway); } private static void migrateExistingConnection(final ConfigRepository configRepository, final EventRunner eventRunner) @@ -244,7 +273,24 @@ public class ServerApp implements ServerRunnable { public static void main(final String[] args) throws Exception { try { - getServer(new ServerFactory.Api(), YamlSeedConfigPersistence.getDefault()).start(); + final Configs configs = new EnvConfigs(); + + // Manual configuration that will be replaced by Dependency Injection in the future + final DataSource configsDataSource = + DataSourceFactory.create(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), DRIVER_CLASS_NAME, + configs.getConfigDatabaseUrl()); + final DataSource jobsDataSource = + DataSourceFactory.create(configs.getDatabaseUser(), configs.getDatabasePassword(), DRIVER_CLASS_NAME, configs.getDatabaseUrl()); + + try (final DSLContext configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); + final DSLContext jobsDslContext = DSLContextFactory.create(jobsDataSource, SQLDialect.POSTGRES)) { + + // Ensure that the database resources are closed on application shutdown + CloseableShutdownHook.registerRuntimeShutdownHook(configsDataSource, jobsDataSource, configsDslContext, jobsDslContext); + + getServer(new ServerFactory.Api(), YamlSeedConfigPersistence.getDefault(), + configs, configsDslContext, configsDataSource, jobsDslContext, jobsDataSource).start(); + } } catch (final Throwable e) { LOGGER.error("Server failed", e); System.exit(1); // so the app doesn't hang on background threads diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 881e17f686c..8433e1c8395 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -26,6 +26,7 @@ import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.flywaydb.core.Flyway; import org.slf4j.MDC; public interface ServerFactory { @@ -49,7 +50,9 @@ public interface ServerFactory { Path workspaceRoot, HttpClient httpClient, FeatureFlags featureFlags, - EventRunner eventRunner); + EventRunner eventRunner, + Flyway configsFlyway, + Flyway jobsFlyway); class Api implements ServerFactory { @@ -73,7 +76,9 @@ public interface ServerFactory { final Path workspaceRoot, final HttpClient httpClient, final FeatureFlags featureFlags, - final EventRunner eventRunner) { + final EventRunner eventRunner, + final Flyway configsFlyway, + final Flyway jobsFlyway) { // set static values for factory ConfigurationApiFactory.setValues( temporalService, @@ -97,7 +102,9 @@ public interface ServerFactory { workspaceRoot, httpClient, featureFlags, - eventRunner); + eventRunner, + configsFlyway, + jobsFlyway); // server configurations final Set> componentClasses = Set.of(ConfigurationApi.class); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 464fa12e1b4..1c2667d2c8b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -141,6 +141,7 @@ import java.io.IOException; import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Map; +import org.flywaydb.core.Flyway; @javax.ws.rs.Path("/v1") public class ConfigurationApi implements io.airbyte.api.V1Api { @@ -185,7 +186,9 @@ public class ConfigurationApi implements io.airbyte.api.V1Api { final Path workspaceRoot, final HttpClient httpClient, final FeatureFlags featureFlags, - final EventRunner eventRunner) { + final EventRunner eventRunner, + final Flyway configsFlyway, + final Flyway jobsFlyway) { this.workerEnvironment = workerEnvironment; this.logConfigs = logConfigs; this.workspaceRoot = workspaceRoot; @@ -259,7 +262,7 @@ public class ConfigurationApi implements io.airbyte.api.V1Api { true); logsHandler = new LogsHandler(); openApiConfigHandler = new OpenApiConfigHandler(); - dbMigrationHandler = new DbMigrationHandler(configsDatabase, jobsDatabase); + dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); } // WORKSPACE @@ -821,7 +824,7 @@ public class ConfigurationApi implements io.airbyte.api.V1Api { } @Override - public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(WebBackendWorkspaceState webBackendWorkspaceState) { + public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(final WebBackendWorkspaceState webBackendWorkspaceState) { return execute(() -> webBackendConnectionsHandler.getWorkspaceState(webBackendWorkspaceState)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DbMigrationHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DbMigrationHandler.java index 2f0f187a49c..b669ea27fea 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DbMigrationHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DbMigrationHandler.java @@ -14,6 +14,7 @@ import io.airbyte.db.instance.DatabaseMigrator; import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; import io.airbyte.db.instance.jobs.JobsDatabaseMigrator; import java.util.stream.Collectors; +import org.flywaydb.core.Flyway; import org.flywaydb.core.api.MigrationInfo; import org.flywaydb.core.api.output.MigrateOutput; import org.flywaydb.core.api.output.MigrateResult; @@ -23,9 +24,9 @@ public class DbMigrationHandler { private final DatabaseMigrator configDbMigrator; private final DatabaseMigrator jobDbMigrator; - public DbMigrationHandler(final Database configsDatabase, final Database jobsDatabase) { - this.configDbMigrator = new ConfigsDatabaseMigrator(configsDatabase, DbMigrationHandler.class.getSimpleName()); - this.jobDbMigrator = new JobsDatabaseMigrator(jobsDatabase, DbMigrationHandler.class.getSimpleName()); + public DbMigrationHandler(final Database configsDatabase, final Flyway configsFlyway, final Database jobsDatabase, final Flyway jobsFlyway) { + this.configDbMigrator = new ConfigsDatabaseMigrator(configsDatabase, configsFlyway); + this.jobDbMigrator = new JobsDatabaseMigrator(jobsDatabase, jobsFlyway); } public DbMigrationReadList list(final DbMigrationRequestBody request) { diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java index 051db770179..0688b462d75 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java @@ -28,6 +28,7 @@ import io.airbyte.workers.WorkerConfigs; import io.temporal.serviceclient.WorkflowServiceStubs; import java.net.http.HttpClient; import java.nio.file.Path; +import org.flywaydb.core.Flyway; import org.junit.jupiter.api.Test; public class ConfigurationApiTest { @@ -59,7 +60,9 @@ public class ConfigurationApiTest { Path.of(""), mock(HttpClient.class), mock(FeatureFlags.class), - mock(EventRunner.class)); + mock(EventRunner.class), + mock(Flyway.class), + mock(Flyway.class)); assertTrue(configurationApi.canImportDefinitons()); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java index 303808aed45..fae5cc83d61 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java @@ -38,12 +38,15 @@ import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.NoOpSecretsHydrator; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.instance.test.TestDatabaseProviders; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.persistence.DefaultJobPersistence; import io.airbyte.scheduler.persistence.JobPersistence; import io.airbyte.scheduler.persistence.WorkspaceHelper; +import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonValidationException; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -56,6 +59,9 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -74,6 +80,8 @@ public class ArchiveHandlerTest { private static final AirbyteVersion VERSION = new AirbyteVersion("0.6.8"); private static PostgreSQLContainer container; + private DataSource dataSource; + private DSLContext dslContext; private Database jobDatabase; private Database configDatabase; private JobPersistence jobPersistence; @@ -111,7 +119,9 @@ public class ArchiveHandlerTest { @BeforeEach public void setup() throws Exception { - final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(container); + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(dataSource, dslContext); jobDatabase = databaseProviders.createNewJobsDatabase(); configDatabase = databaseProviders.createNewConfigsDatabase(); jobPersistence = new DefaultJobPersistence(jobDatabase); @@ -142,9 +152,11 @@ public class ArchiveHandlerTest { } @AfterEach - void tearDown() throws Exception { - jobDatabase.close(); - configDatabase.close(); + void tearDown() throws IOException { + dslContext.close(); + if (dataSource instanceof Closeable closeable) { + closeable.close(); + } } /** diff --git a/airbyte-test-utils/build.gradle b/airbyte-test-utils/build.gradle index ae566810026..74ebc96d5dd 100644 --- a/airbyte-test-utils/build.gradle +++ b/airbyte-test-utils/build.gradle @@ -5,8 +5,9 @@ plugins { dependencies { implementation project(':airbyte-db:lib') - implementation 'org.testcontainers:postgresql:1.15.3' - implementation "org.testcontainers:cockroachdb:1.15.3" + implementation libs.testcontainers.jdbc + implementation libs.testcontainers.postgresql + implementation libs.testcontainers.cockroachdb implementation 'org.junit.jupiter:junit-jupiter-api:5.7.2' } diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/CockroachDBContainerHelper.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/CockroachDBContainerHelper.java index 378453f8d86..72c6f10e72b 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/CockroachDBContainerHelper.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/CockroachDBContainerHelper.java @@ -10,11 +10,15 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; import java.io.IOException; import java.util.UUID; -import org.jooq.SQLDialect; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.testcontainers.containers.CockroachContainer; import org.testcontainers.utility.MountableFile; @@ -61,27 +65,23 @@ public class CockroachDBContainerHelper { .build()); } - public static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + public static DataSource getDataSourceFromConfig(final JsonNode config) { + return DataSourceFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText())); } - public static JdbcDatabase getJdbcDatabaseFromConfig(final JsonNode config) { - return Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver"); + public static Database getDatabaseFromConfig(final DSLContext dslContext) {// final JsonNode config) { + return new Database(dslContext); + } + + public static JdbcDatabase getJdbcDatabaseFromConfig(final DataSource dataSource) { // final JsonNode config) { + return new DefaultJdbcDatabase(dataSource, JdbcUtils.getDefaultSourceOperations()); } } diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/DatabaseConnectionHelper.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/DatabaseConnectionHelper.java new file mode 100644 index 00000000000..85345f89963 --- /dev/null +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/DatabaseConnectionHelper.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.test.utils; + +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** + * Helper class that facilitates the creation of database connection objects for testing purposes. + */ +public class DatabaseConnectionHelper { + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param container A JDBC Test Container instance. + * @return The configured {@link DataSource}. + */ + public static DataSource createDataSource(final JdbcDatabaseContainer container) { + return DataSourceFactory.create(container.getUsername(), + container.getPassword(), + container.getDriverClassName(), + container.getJdbcUrl()); + } + + /** + * Constructs a configured {@link DSLContext} instance using the provided configuration. + * + * @param container A JDBC Test Container instance. + * @param dialect The SQL dialect to use with objects created from this context. + * @return The configured {@link DSLContext}. + */ + public static DSLContext createDslContext(final JdbcDatabaseContainer container, final SQLDialect dialect) { + return DSLContextFactory.create( + container.getUsername(), + container.getPassword(), + container.getDriverClassName(), + container.getJdbcUrl(), + dialect); + } + +} diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/PostgreSQLContainerHelper.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/PostgreSQLContainerHelper.java index 6d0cfdbd139..3115934f377 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/PostgreSQLContainerHelper.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/PostgreSQLContainerHelper.java @@ -10,11 +10,15 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; import java.io.IOException; import java.util.UUID; -import org.jooq.SQLDialect; +import javax.sql.DataSource; +import org.jooq.DSLContext; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; @@ -57,27 +61,23 @@ public class PostgreSQLContainerHelper { .build()); } - public static Database getDatabaseFromConfig(final JsonNode config) { - return Databases.createDatabase( + public static DataSource getDataSourceFromConfig(final JsonNode config) { + return DataSourceFactory.create( config.get("username").asText(), config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver", - SQLDialect.POSTGRES); + config.get("port").asInt(), + config.get("database").asText())); } - public static JdbcDatabase getJdbcDatabaseFromConfig(final JsonNode config) { - return Databases.createJdbcDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:postgresql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "org.postgresql.Driver"); + public static Database getDatabaseFromConfig(final DSLContext dslContext) { + return new Database(dslContext); + } + + public static JdbcDatabase getJdbcDatabaseFromConfig(final DataSource dataSource) { + return new DefaultJdbcDatabase(dataSource, JdbcUtils.getDefaultSourceOperations()); } } diff --git a/airbyte-test-utils/src/test/java/io/airbyte/test/utils/DatabaseConnectionHelperTest.java b/airbyte-test-utils/src/test/java/io/airbyte/test/utils/DatabaseConnectionHelperTest.java new file mode 100644 index 00000000000..8698da73f30 --- /dev/null +++ b/airbyte-test-utils/src/test/java/io/airbyte/test/utils/DatabaseConnectionHelperTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.test.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.zaxxer.hikari.HikariDataSource; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; + +public class DatabaseConnectionHelperTest { + + private static final String DATABASE_NAME = "airbyte_test_database"; + + protected static PostgreSQLContainer container; + + @BeforeAll + public static void dbSetup() { + container = new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName(DATABASE_NAME) + .withUsername("docker") + .withPassword("docker"); + container.start(); + } + + @AfterAll + public static void dbDown() { + container.close(); + } + + @Test + void testCreatingFromATestContainer() { + final DataSource dataSource = DatabaseConnectionHelper.createDataSource(container); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); + } + + @Test + void testCreatingADslContextFromATestContainer() { + final SQLDialect dialect = SQLDialect.POSTGRES; + final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(container, dialect); + assertNotNull(dslContext); + assertEquals(dialect, dslContext.configuration().dialect()); + } + +} diff --git a/airbyte-tests/build.gradle b/airbyte-tests/build.gradle index 7c65475095c..ccc287e6845 100644 --- a/airbyte-tests/build.gradle +++ b/airbyte-tests/build.gradle @@ -39,7 +39,7 @@ dependencies { implementation project(':airbyte-container-orchestrator') implementation 'io.fabric8:kubernetes-client:5.12.2' - implementation 'org.testcontainers:testcontainers:1.15.3' + implementation libs.testcontainers acceptanceTestsImplementation project(':airbyte-api') acceptanceTestsImplementation project(':airbyte-commons') @@ -54,14 +54,14 @@ dependencies { acceptanceTestsImplementation 'io.github.cdimascio:java-dotenv:3.0.0' acceptanceTestsImplementation 'io.temporal:temporal-sdk:1.8.1' acceptanceTestsImplementation 'org.apache.commons:commons-csv:1.4' - acceptanceTestsImplementation 'org.testcontainers:postgresql:1.15.3' - acceptanceTestsImplementation 'org.postgresql:postgresql:42.2.18' + acceptanceTestsImplementation libs.testcontainers.postgresql + acceptanceTestsImplementation libs.postgresql automaticMigrationAcceptanceTestImplementation project(':airbyte-api') automaticMigrationAcceptanceTestImplementation project(':airbyte-commons') automaticMigrationAcceptanceTestImplementation project(':airbyte-tests') - automaticMigrationAcceptanceTestImplementation 'org.testcontainers:testcontainers:1.15.3' + automaticMigrationAcceptanceTestImplementation libs.testcontainers } // test should run using the current version of the docker compose configuration. diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java index 6f18ec170c8..fbee095ff3a 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java @@ -80,8 +80,8 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreProperties; import io.airbyte.container_orchestrator.ContainerOrchestratorApp; import io.airbyte.db.Database; -import io.airbyte.db.Databases; import io.airbyte.test.airbyte_test_container.AirbyteTestContainer; +import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.test.utils.PostgreSQLContainerHelper; import io.airbyte.workers.temporal.TemporalUtils; import io.airbyte.workers.temporal.scheduling.ConnectionManagerWorkflow; @@ -117,6 +117,7 @@ import java.util.stream.Collectors; import org.jooq.JSONB; import org.jooq.Record; import org.jooq.Result; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -596,7 +597,6 @@ public class AcceptanceTests { // full refreshing, this record will appear in the output and cause the test to fail. if we are, // correctly, doing incremental, we will not find this value in the destination. source.query(ctx -> ctx.execute("UPDATE id_and_name SET name='yennefer' WHERE id=2")); - source.close(); LOGGER.info("Starting testIncrementalSync() sync 2"); final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() @@ -747,7 +747,6 @@ public class AcceptanceTests { // retrieve latest snapshot of source records after modifications; the deduplicated table in // destination should mirror this latest state of records final List expectedNormalizedRecords = retrieveSourceRecords(source, STREAM_NAME); - source.close(); final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); @@ -1332,7 +1331,7 @@ public class AcceptanceTests { } private Database getDatabase(final PostgreSQLContainer db) { - return Databases.createPostgresDatabase(db.getUsername(), db.getPassword(), db.getJdbcUrl()); + return new Database(DatabaseConnectionHelper.createDslContext(db, SQLDialect.POSTGRES)); } private Set listAllTables(final Database database) throws SQLException { diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/GKEPostgresConfig.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/GKEPostgresConfig.java index 123a0b7b932..5ec03a0c306 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/GKEPostgresConfig.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/GKEPostgresConfig.java @@ -5,10 +5,12 @@ package io.airbyte.test.acceptance; import io.airbyte.db.Database; -import io.airbyte.db.Databases; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.test.acceptance.AcceptanceTests.Type; import java.util.HashMap; import java.util.Map; +import org.jooq.SQLDialect; /** * This class is used to provide information related to the test databases for running the @@ -43,11 +45,13 @@ public class GKEPostgresConfig { } public static Database getSourceDatabase() { - return Databases.createPostgresDatabase(USERNAME, PASSWORD, "jdbc:postgresql://localhost:2000/postgresdb"); + return new Database(DSLContextFactory.create(USERNAME, PASSWORD, DatabaseDriver.POSTGRESQL.getDriverClassName(), + "jdbc:postgresql://localhost:2000/postgresdb", SQLDialect.POSTGRES)); } public static Database getDestinationDatabase() { - return Databases.createPostgresDatabase(USERNAME, PASSWORD, "jdbc:postgresql://localhost:3000/postgresdb"); + return new Database(DSLContextFactory.create(USERNAME, PASSWORD, DatabaseDriver.POSTGRESQL.getDriverClassName(), + "jdbc:postgresql://localhost:3000/postgresdb", SQLDialect.POSTGRES)); } } diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 3830e60097e..a85cea79581 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -33,11 +33,11 @@ dependencies { testImplementation 'io.temporal:temporal-testing:1.8.1' testImplementation 'com.jayway.jsonpath:json-path:2.7.0' - testImplementation "org.flywaydb:flyway-core:7.14.0" + testImplementation libs.flyway.core testImplementation 'org.mockito:mockito-inline:4.0.0' - testImplementation 'org.postgresql:postgresql:42.2.18' - testImplementation 'org.testcontainers:testcontainers:1.15.3' - testImplementation 'org.testcontainers:postgresql:1.15.3' + testImplementation libs.postgresql + testImplementation libs.testcontainers + testImplementation libs.testcontainers.postgresql testImplementation project(':airbyte-commons-docker') testImplementation project(':airbyte-test-utils') diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index d87ffd380b5..e3c17a35a43 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -9,6 +9,7 @@ import io.airbyte.analytics.TrackingClient; import io.airbyte.analytics.TrackingClientSingleton; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.config.Configs; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.EnvConfigs; @@ -22,6 +23,9 @@ import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import io.airbyte.db.instance.jobs.JobsDatabaseInstance; import io.airbyte.metrics.lib.DatadogClientConfiguration; @@ -80,7 +84,10 @@ import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.concurrent.Executors; +import javax.sql.DataSource; import lombok.AllArgsConstructor; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -90,6 +97,7 @@ public class WorkerApp { private static final Logger LOGGER = LoggerFactory.getLogger(WorkerApp.class); public static final int KUBE_HEARTBEAT_PORT = 9000; + private static final String DRIVER_CLASS_NAME = DatabaseDriver.POSTGRESQL.getDriverClassName(); // IMPORTANT: Changing the storage location will orphan already existing kube pods when the new // version is deployed! @@ -332,9 +340,7 @@ public class WorkerApp { } } - private static void launchWorkerApp() throws IOException { - final Configs configs = new EnvConfigs(); - + private static void launchWorkerApp(final Configs configs, final DSLContext configsDslContext, final DSLContext jobsDslContext) throws IOException { DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration(configs)); final WorkerConfigs defaultWorkerConfigs = new WorkerConfigs(configs); @@ -358,7 +364,7 @@ public class WorkerApp { final String temporalHost = configs.getTemporalHost(); LOGGER.info("temporalHost = " + temporalHost); - final SecretsHydrator secretsHydrator = SecretPersistence.getSecretsHydrator(configs); + final SecretsHydrator secretsHydrator = SecretPersistence.getSecretsHydrator(configsDslContext, configs); if (configs.getWorkerEnvironment().equals(WorkerEnvironment.KUBERNETES)) { KubePortManagerSingleton.init(configs.getTemporalWorkerPorts()); @@ -368,11 +374,7 @@ public class WorkerApp { TemporalUtils.configureTemporalNamespace(temporalService); - final Database configDatabase = new ConfigsDatabaseInstance( - configs.getConfigDatabaseUser(), - configs.getConfigDatabasePassword(), - configs.getConfigDatabaseUrl()) - .getInitialized(); + final Database configDatabase = new ConfigsDatabaseInstance(configsDslContext).getInitialized(); final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder() .maskSecrets(!featureFlags.exposeSecretsInExport()) @@ -381,11 +383,7 @@ public class WorkerApp { final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor); final ConfigRepository configRepository = new ConfigRepository(configPersistence, configDatabase); - final Database jobDatabase = new JobsDatabaseInstance( - configs.getDatabaseUser(), - configs.getDatabasePassword(), - configs.getDatabaseUrl()) - .getInitialized(); + final Database jobDatabase = new JobsDatabaseInstance(jobsDslContext).getInitialized(); final JobPersistence jobPersistence = new DefaultJobPersistence(jobDatabase); TrackingClientSingleton.initialize( @@ -456,7 +454,22 @@ public class WorkerApp { public static void main(final String[] args) { try { - launchWorkerApp(); + final Configs configs = new EnvConfigs(); + + final DataSource configsDataSource = DataSourceFactory.create(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), + DRIVER_CLASS_NAME, configs.getConfigDatabaseUrl()); + final DataSource jobsDataSource = DataSourceFactory.create(configs.getDatabaseUser(), configs.getDatabasePassword(), + DRIVER_CLASS_NAME, configs.getDatabaseUrl()); + + // Manual configuration that will be replaced by Dependency Injection in the future + try (final DSLContext configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); + final DSLContext jobsDslContext = DSLContextFactory.create(jobsDataSource, SQLDialect.POSTGRES)) { + + // Ensure that the database resources are closed on application shutdown + CloseableShutdownHook.registerRuntimeShutdownHook(configsDataSource, jobsDataSource, configsDslContext, jobsDslContext); + + launchWorkerApp(configs, configsDslContext, jobsDslContext); + } } catch (final Throwable t) { LOGGER.error("Worker app failed", t); System.exit(1); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java index e73171176fc..c5157ba1cdb 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java @@ -15,16 +15,21 @@ import static org.mockito.Mockito.when; import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.config.Configs; import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.instance.test.TestDatabaseProviders; import io.airbyte.scheduler.models.JobRunConfig; import io.airbyte.scheduler.persistence.DefaultJobPersistence; import io.airbyte.scheduler.persistence.JobPersistence; +import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.workers.Worker; import io.temporal.serviceclient.CheckedExceptionWrapper; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.function.Consumer; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -42,6 +47,8 @@ class TemporalAttemptExecutionTest { private static PostgreSQLContainer container; private static Configs configs; + private static DataSource dataSource; + private static DSLContext dslContext; private Path jobRoot; @@ -57,12 +64,15 @@ class TemporalAttemptExecutionTest { .withPassword(SOURCE_PASSWORD); container.start(); configs = mock(Configs.class); + + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); } @SuppressWarnings("unchecked") @BeforeEach void setup() throws IOException { - final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(container); + final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(dataSource, dslContext); final Database jobDatabase = databaseProviders.createNewJobsDatabase(); final JobPersistence jobPersistence = new DefaultJobPersistence(jobDatabase); diff --git a/deps.toml b/deps.toml index 5d637e2d3ef..e0cf8f42493 100644 --- a/deps.toml +++ b/deps.toml @@ -1,11 +1,16 @@ [versions] fasterxml_version = "2.13.0" +flyway = "7.14.0" glassfish_version = "2.31" +hikaricp = "5.0.1" commons_io = "2.7" log4j = "2.17.1" slf4j = "1.7.30" lombok = "1.18.22" +jooq = "3.13.4" junit-jupiter = "5.7.2" +postgresql = "42.3.4" +testcontainers = "1.17.1" [libraries] fasterxml = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "fasterxml_version" } @@ -25,6 +30,27 @@ log4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j-impl", version.ref log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j" } jul-to-slf4j = { module = "org.slf4j:jul-to-slf4j", version.ref = "slf4j" } jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } +hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" } +jooq = { module = "org.jooq:jooq", version.ref = "jooq" } +jooq-codegen = { module = "org.jooq:jooq-codegen", version.ref = "jooq" } +jooq-meta = { module = "org.jooq:jooq-meta", version.ref = "jooq" } +postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" } +flyway-core = { module = "org.flywaydb:flyway-core", version.ref = "flyway" } +testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-cassandra = { module = "org.testcontainers:cassandra", version.ref = "testcontainers" } +testcontainers-clickhouse = { module = "org.testcontainers:clickhouse", version.ref = "testcontainers" } +testcontainers-cockroachdb = { module = "org.testcontainers:cockroachdb", version.ref = "testcontainers" } +testcontainers-db2 = { module = "org.testcontainers:db2", version.ref = "testcontainers" } +testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "testcontainers" } +testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref = "testcontainers" } +testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } +testcontainers-mariadb = { module = "org.testcontainers:mariadb", version.ref = "testcontainers" } +testcontainers-mongodb = { module = "org.testcontainers:mongodb", version.ref = "testcontainers" } +testcontainers-mssqlserver = { module = "org.testcontainers:mssqlserver", version.ref = "testcontainers" } +testcontainers-mysql = { module = "org.testcontainers:mysql", version.ref = "testcontainers" } +testcontainers-oracle-xe = { module = "org.testcontainers:oracle-xe", version.ref = "testcontainers" } +testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "testcontainers" } +testcontainers-pulsar = { module = "org.testcontainers:pulsar", version.ref = "testcontainers" } log4j-over-slf4j = { module = "org.slf4j:log4j-over-slf4j", version.ref = "slf4j" } appender-log4j2 = { module = "com.therealvan:appender-log4j2", version = "3.6.0" } aws-java-sdk-s3 = { module = "com.amazonaws:aws-java-sdk-s3", version = "1.12.6" }