jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

[13309] Destination-mssql: enabled DAT tests (#16226)

Browse Source
This commit is contained in:
Eugene
2022-09-02 10:20:53 +03:00
committed by GitHub
parent 29cef6635f
commit db8df934b1
3 changed files with 56 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java
View File

@@ -78,4 +78,8 @@ public class SshBastionContainer {
db.close();
}
public GenericContainer getContainer() {
return bastion;
}
}

44
airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java
View File

@@ -15,11 +15,11 @@ import io.airbyte.db.factory.DatabaseDriver;
import io.airbyte.db.jdbc.JdbcUtils;
import io.airbyte.integrations.base.JavaBaseConstants;
import io.airbyte.integrations.destination.ExtendedNameTransformer;
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import io.airbyte.integrations.util.HostPortResolver;
import io.airbyte.test.utils.DatabaseConnectionHelper;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.jooq.DSLContext;
@@ -29,7 +29,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.MSSQLServerContainer;
import org.testcontainers.utility.DockerImageName;
public class MSSQLDestinationAcceptanceTestSSL extends DestinationAcceptanceTest {
public class MSSQLDestinationAcceptanceTestSSL extends JdbcDestinationAcceptanceTest {
private static MSSQLServerContainer<?> db;
private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
@@ -90,7 +90,7 @@ public class MSSQLDestinationAcceptanceTestSSL extends DestinationAcceptanceTest
throws Exception {
return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace)
.stream()
.map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText()))
.map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA))
.collect(Collectors.toList());
}
@@ -112,19 +112,6 @@ public class MSSQLDestinationAcceptanceTestSSL extends DestinationAcceptanceTest
return retrieveRecordsFromTable(tableName, namespace);
}
@Override
protected List<String> resolveIdentifier(final String identifier) {
final List<String> result = new ArrayList<>();
final String resolved = namingResolver.getIdentifier(identifier);
result.add(identifier);
result.add(resolved);
if (!resolved.startsWith("\"")) {
result.add(resolved.toLowerCase());
result.add(resolved.toUpperCase());
}
return result;
}
private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException {
final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, SQLDialect.DEFAULT);
return new Database(dslContext).query(
@@ -133,8 +120,7 @@ public class MSSQLDestinationAcceptanceTestSSL extends DestinationAcceptanceTest
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.stream()
.map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat()))
.map(Jsons::deserialize)
.map(this::getJsonFromRecord)
.collect(Collectors.toList());
});
}
@@ -194,4 +180,24 @@ public class MSSQLDestinationAcceptanceTestSSL extends DestinationAcceptanceTest
db.close();
}
@Override
protected TestDataComparator getTestDataComparator() {
return new MSSQLTestDataComparator();
}
@Override
protected boolean supportBasicDataTypeTest() {
return true;
}
@Override
protected boolean supportArrayDataTypeTest() {
return true;
}
@Override
protected boolean supportObjectDataTypeTest() {
return true;
}
}

47
airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java
View File

@@ -16,8 +16,8 @@ import io.airbyte.integrations.base.JavaBaseConstants;
import io.airbyte.integrations.base.ssh.SshBastionContainer;
import io.airbyte.integrations.base.ssh.SshTunnel;
import io.airbyte.integrations.destination.ExtendedNameTransformer;
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
import java.util.ArrayList;
import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.RandomStringUtils;
@@ -29,7 +29,7 @@ import org.testcontainers.containers.Network;
* Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file
* or with a password.
*/
public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAcceptanceTest {
public abstract class SshMSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest {
private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
@@ -73,7 +73,7 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAccep
throws Exception {
return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace)
.stream()
.map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText()))
.map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA))
.collect(Collectors.toList());
}
@@ -92,19 +92,6 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAccep
return true;
}
@Override
protected List<String> resolveIdentifier(final String identifier) {
final List<String> result = new ArrayList<>();
final String resolved = namingResolver.getIdentifier(identifier);
result.add(identifier);
result.add(resolved);
if (!resolved.startsWith("\"")) {
result.add(resolved.toLowerCase());
result.add(resolved.toUpperCase());
}
return result;
}
private static Database getDatabaseFromConfig(final JsonNode config) {
final DSLContext dslContext = DSLContextFactory.create(
config.get(JdbcUtils.USERNAME_KEY).asText(),
@@ -132,8 +119,7 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAccep
database, schema, tableName.toLowerCase(),
JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.stream()
.map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat()))
.map(Jsons::deserialize)
.map(this::getJsonFromRecord)
.collect(Collectors.toList())));
}
@@ -164,7 +150,8 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAccep
private void initAndStartJdbcContainer() {
db = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-CU16-ubuntu-20.04")
.withNetwork(network)
.acceptLicense();
.acceptLicense()
.dependsOn(bastion.getContainer());
db.start();
}
@@ -173,4 +160,24 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAccep
bastion.stopAndCloseContainers(db);
}
@Override
protected TestDataComparator getTestDataComparator() {
return new MSSQLTestDataComparator();
}
@Override
protected boolean supportBasicDataTypeTest() {
return true;
}
@Override
protected boolean supportArrayDataTypeTest() {
return true;
}
@Override
protected boolean supportObjectDataTypeTest() {
return true;
}
}