jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-23 21:03:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

introduce BaseImage to MySql source (#32899)

Browse Source 
introduce baseImage to MySql so we don't use strings everywhere
This commit is contained in:
Stephane Geneix
2023-12-07 15:38:42 -08:00
committed by GitHub
parent f6d03ec860
commit 0b8b11b3e5
11 changed files with 37 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java
View File

@@ -7,6 +7,7 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources;
import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.cdk.db.Database;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
public class CDCMySqlDatatypeAccuracyTest extends MySqlDatatypeAccuracyTest {
@@ -21,7 +22,7 @@ public class CDCMySqlDatatypeAccuracyTest extends MySqlDatatypeAccuracyTest {
@Override
protected Database setupDatabase() {
testdb = MySQLTestDatabase.in("mysql:8.0").withoutStrictMode().withCdcPermissions();
testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions();
return testdb.getDatabase();
}

3
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java
View File

@@ -12,6 +12,7 @@ import io.airbyte.commons.features.FeatureFlags;
import io.airbyte.commons.features.FeatureFlagsWrapper;
import io.airbyte.commons.json.Jsons;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import io.airbyte.protocol.models.JsonSchemaType;
import io.airbyte.protocol.models.v0.AirbyteMessage;
import io.airbyte.protocol.models.v0.AirbyteStateMessage;
@@ -38,7 +39,7 @@ public class CdcBinlogsMySqlSourceDatatypeTest extends AbstractMySqlSourceDataty
@Override
protected Database setupDatabase() {
testdb = MySQLTestDatabase.in("mysql:8.0").withoutStrictMode().withCdcPermissions();
testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions();
return testdb.getDatabase();
}

3
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java
View File

@@ -9,6 +9,7 @@ import io.airbyte.cdk.db.Database;
import io.airbyte.commons.features.FeatureFlags;
import io.airbyte.commons.features.FeatureFlagsWrapper;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
public class CdcInitialSnapshotMySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest {
@@ -28,7 +29,7 @@ public class CdcInitialSnapshotMySqlSourceDatatypeTest extends AbstractMySqlSour
@Override
protected Database setupDatabase() {
testdb = MySQLTestDatabase.in("mysql:8.0").withoutStrictMode().withCdcPermissions();
testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions();
return testdb.getDatabase();
}

3
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java
View File

@@ -19,6 +19,7 @@ import io.airbyte.commons.features.FeatureFlags;
import io.airbyte.commons.features.FeatureFlagsWrapper;
import io.airbyte.commons.json.Jsons;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import io.airbyte.protocol.models.Field;
import io.airbyte.protocol.models.JsonSchemaType;
import io.airbyte.protocol.models.v0.AirbyteMessage;
@@ -101,7 +102,7 @@ public class CdcMySqlSourceAcceptanceTest extends SourceAcceptanceTest {
@Override
protected void setupEnvironment(final TestDestinationEnv environment) {
testdb = MySQLTestDatabase.in("mysql:8.0", extraContainerFactoryMethods().toArray(String[]::new))
testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, extraContainerFactoryMethods().toArray(String[]::new))
.withCdcPermissions()
.with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));")
.with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');")

3
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java
View File

@@ -13,6 +13,7 @@ import io.airbyte.commons.features.FeatureFlags;
import io.airbyte.commons.features.FeatureFlagsWrapper;
import io.airbyte.commons.json.Jsons;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import io.airbyte.protocol.models.Field;
import io.airbyte.protocol.models.JsonSchemaType;
import io.airbyte.protocol.models.v0.CatalogHelpers;
@@ -33,7 +34,7 @@ public class MySqlSourceAcceptanceTest extends SourceAcceptanceTest {
@Override
protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
testdb = MySQLTestDatabase.in("mysql:8.0", extraContainerFactoryMethods().toArray(String[]::new))
testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, extraContainerFactoryMethods().toArray(String[]::new))
.with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));")
.with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');")
.with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));")

3
airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java
View File

@@ -12,6 +12,7 @@ import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer;
import io.airbyte.cdk.integrations.base.ssh.SshTunnel;
import io.airbyte.commons.exceptions.ConfigErrorException;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import io.airbyte.integrations.source.mysql.MySqlSource;
import java.nio.file.Path;
import org.junit.jupiter.api.Test;
@@ -25,7 +26,7 @@ public class SshPasswordMySqlSourceAcceptanceTest extends AbstractSshMySqlSource
@Test
public void sshTimeoutExceptionMarkAsConfigErrorTest() throws Exception {
try (final var testdb = MySQLTestDatabase.in("mysql:8.0", "withNetwork")) {
try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, "withNetwork")) {
final SshBastionContainer bastion = new SshBastionContainer();
bastion.initAndStartBastion(testdb.getContainer().getNetwork());
final var config = testdb.integrationTestConfigBuilder()

3
airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java
View File

@@ -39,6 +39,7 @@ import io.airbyte.commons.features.FeatureFlagsWrapper;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.util.AutoCloseableIterator;
import io.airbyte.commons.util.AutoCloseableIterators;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import io.airbyte.protocol.models.Field;
import io.airbyte.protocol.models.JsonSchemaType;
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus;
@@ -77,7 +78,7 @@ public class CdcMysqlSourceTest extends CdcSourceTest<MySqlSource, MySQLTestData
@Override
protected MySQLTestDatabase createTestDatabase() {
return MySQLTestDatabase.in("mysql:8.0", "withInvalidTimezoneCEST").withCdcPermissions();
return MySQLTestDatabase.in(BaseImage.MYSQL_8, "withInvalidTimezoneCEST").withCdcPermissions();
}
@Override

3
airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java
View File

@@ -28,6 +28,7 @@ import io.airbyte.commons.features.FeatureFlagsWrapper;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.resources.MoreResources;
import io.airbyte.commons.util.MoreIterators;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import io.airbyte.integrations.source.mysql.internal.models.CursorBasedStatus;
import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType;
import io.airbyte.protocol.models.Field;
@@ -78,7 +79,7 @@ class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest<MySqlSource
@Override
protected MySQLTestDatabase createTestDatabase() {
return MySQLTestDatabase.in("mysql:8.0");
return MySQLTestDatabase.in(BaseImage.MYSQL_8);
}
@Override

3
airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceOperationsTest.java
View File

@@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.mysql.cj.MysqlType;
import io.airbyte.cdk.db.jdbc.DateTimeConverter;
import io.airbyte.commons.json.Jsons;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -91,7 +92,7 @@ public class MySqlSourceOperationsTest {
throws SQLException {
final var sqlSourceOperations = new MySqlSourceOperations();
final String cursorColumn = "cursor_column";
try (final var testdb = MySQLTestDatabase.in("mysql:8.0")
try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8)
.with("CREATE TABLE cursor_table (id INTEGER PRIMARY KEY, %s %s);", cursorColumn, sqlType)) {
final List<JsonNode> expectedRecords = new ArrayList<>();

7
airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java
View File

@@ -21,6 +21,7 @@ import io.airbyte.commons.features.EnvVariableFeatureFlags;
import io.airbyte.commons.features.FeatureFlagsWrapper;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.util.MoreIterators;
import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage;
import io.airbyte.protocol.models.Field;
import io.airbyte.protocol.models.JsonSchemaType;
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus;
@@ -46,7 +47,7 @@ public class MySqlSourceTests {
@Test
public void testSettingTimezones() throws Exception {
try (final var testdb = MySQLTestDatabase.in("mysql:8.0", "withMoscowTimezone")) {
try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, "withMoscowTimezone")) {
final var config = testdb.testConfigBuilder()
.with(JdbcUtils.JDBC_URL_PARAMS_KEY, "serverTimezone=Europe/Moscow")
.withoutSsl()
@@ -76,7 +77,7 @@ public class MySqlSourceTests {
@Test
@Disabled("See https://github.com/airbytehq/airbyte/pull/23908#issuecomment-1463753684, enable once communication is out")
public void testNullCursorValueShouldThrowException() {
try (final var testdb = MySQLTestDatabase.in("mysql:8.0")
try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8)
.with("CREATE TABLE null_cursor_table(id INTEGER NULL);")
.with("INSERT INTO null_cursor_table(id) VALUES (1), (2), (NULL);")
.with("CREATE VIEW null_cursor_view(id) AS SELECT null_cursor_table.id FROM null_cursor_table;")) {
@@ -126,7 +127,7 @@ public class MySqlSourceTests {
@Test
public void testJDBCSessionVariable() throws Exception {
try (final var testdb = MySQLTestDatabase.in("mysql:8.0")) {
try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8)) {
final var config = testdb.testConfigBuilder()
.with(JdbcUtils.JDBC_URL_PARAMS_KEY, "sessionVariables=MAX_EXECUTION_TIME=28800000")
.withoutSsl()

17
airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java
View File

@@ -17,8 +17,21 @@ import org.testcontainers.containers.MySQLContainer;
public class MySQLTestDatabase extends
TestDatabase<MySQLContainer<?>, MySQLTestDatabase, MySQLTestDatabase.MySQLConfigBuilder> {
static public MySQLTestDatabase in(String imageName, String... methods) {
final var container = new MySQLContainerFactory().shared(imageName, methods);
public static enum BaseImage {
MYSQL_8("mysql:8.0"),
;
private final String reference;
private BaseImage(String reference) {
this.reference = reference;
}
}
static public MySQLTestDatabase in(BaseImage baseImage, String... methods) {
final var container = new MySQLContainerFactory().shared(baseImage.reference, methods);
return new MySQLTestDatabase(container).initialized();
}