jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 18:14:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

MSSQL remove normalization (#36050)

Browse Source
This commit is contained in:
Joe Bell
2024-04-22 13:23:17 -07:00
committed by GitHub
parent 8f6036e871
commit 54b0a7b86b
15 changed files with 243 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle
View File

@@ -4,10 +4,12 @@ plugins {
}
airbyteJavaConnector {
cdkVersionRequired = '0.2.0'
cdkVersionRequired = '0.30.2'
features = [
'db-sources', // required for tests
'db-destinations',
's3-destinations',
'typing-deduping'
]
useLocalCdk = false
}

16
airbyte-integrations/connectors/destination-mssql-strict-encrypt/metadata.yaml
View File

@@ -7,17 +7,23 @@ data:
connectorSubtype: database
connectorType: destination
definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
dockerImageTag: 0.2.0
dockerImageTag: 1.0.0
dockerRepository: airbyte/destination-mssql-strict-encrypt
githubIssueLabel: destination-mssql
icon: mssql.svg
license: ELv2
name: MS SQL Server
normalizationConfig:
normalizationIntegrationType: mssql
normalizationRepository: airbyte/normalization-mssql
normalizationTag: 0.4.1
releaseStage: alpha
releases:
breakingChanges:
1.0.0:
upgradeDeadline: "2024-05-25"
message: >
This version removes the option to use "normalization" with MSSQL. It also changes
the schema and database of Airbyte's "raw" tables to be compatible with the new
[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2)
format. These changes will likely require updates to downstream dbt / SQL models.
Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync.
documentationUrl: https://docs.airbyte.com/integrations/destinations/mssql
supportsDbt: true
tags:

4
airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java
View File

@@ -30,9 +30,11 @@ import java.util.stream.Collectors;
import org.jooq.DSLContext;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.MSSQLServerContainer;
@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.")
public class MssqlStrictEncryptDestinationAcceptanceTest extends DestinationAcceptanceTest {
private static MSSQLServerContainer<?> db;
@@ -167,7 +169,7 @@ public class MssqlStrictEncryptDestinationAcceptanceTest extends DestinationAcce
@Override
protected void tearDown(final TestDestinationEnv testEnv) {
dslContext.close();
// do nothing
}
@AfterAll

4
airbyte-integrations/connectors/destination-mssql/build.gradle
View File

@@ -4,10 +4,12 @@ plugins {
}
airbyteJavaConnector {
cdkVersionRequired = '0.2.0'
cdkVersionRequired = '0.30.2'
features = [
'db-sources', // required for tests
'db-destinations',
's3-destinations',
'typing-deduping'
]
useLocalCdk = false
}

16
airbyte-integrations/connectors/destination-mssql/metadata.yaml
View File

@@ -2,16 +2,12 @@ data:
connectorSubtype: database
connectorType: destination
definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
dockerImageTag: 0.2.0
dockerImageTag: 1.0.0
dockerRepository: airbyte/destination-mssql
githubIssueLabel: destination-mssql
icon: mssql.svg
license: ELv2
name: MS SQL Server
normalizationConfig:
normalizationIntegrationType: mssql
normalizationRepository: airbyte/normalization-mssql
normalizationTag: 0.4.3
registries:
cloud:
dockerRepository: airbyte/destination-mssql-strict-encrypt
@@ -19,6 +15,16 @@ data:
oss:
enabled: true
releaseStage: alpha
releases:
breakingChanges:
1.0.0:
upgradeDeadline: "2024-05-25"
message: >
This version removes the option to use "normalization" with MSSQL. It also changes
the schema and database of Airbyte's "raw" tables to be compatible with the new
[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2)
format. These changes will likely require updates to downstream dbt / SQL models.
Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync.
documentationUrl: https://docs.airbyte.com/integrations/destinations/mssql
supportsDbt: true
tags:

46
airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java
View File

@@ -7,16 +7,28 @@ package io.airbyte.integrations.destination.mssql;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import io.airbyte.cdk.db.factory.DatabaseDriver;
import io.airbyte.cdk.db.jdbc.JdbcDatabase;
import io.airbyte.cdk.db.jdbc.JdbcUtils;
import io.airbyte.cdk.integrations.base.Destination;
import io.airbyte.cdk.integrations.base.IntegrationRunner;
import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination;
import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination;
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler;
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator;
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.NoOpJdbcDestinationHandler;
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator;
import io.airbyte.commons.json.Jsons;
import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler;
import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator;
import io.airbyte.integrations.base.destination.typing_deduping.migrators.Migration;
import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState;
import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.jetbrains.annotations.NotNull;
import org.jooq.SQLDialect;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +42,7 @@ public class MSSQLDestination extends AbstractJdbcDestination implements Destina
super(DRIVER_CLASS, new MSSQLNameTransformer(), new SqlServerOperations());
}
@NotNull
@Override
protected Map<String, String> getDefaultConnectionProperties(final JsonNode config) {
final HashMap<String, String> properties = new HashMap<>();
@@ -57,6 +70,7 @@ public class MSSQLDestination extends AbstractJdbcDestination implements Destina
return properties;
}
@NotNull
@Override
public JsonNode toJdbcConfig(final JsonNode config) {
final String schema = Optional.ofNullable(config.get("schema")).map(JsonNode::asText).orElse("public");
@@ -81,6 +95,22 @@ public class MSSQLDestination extends AbstractJdbcDestination implements Destina
return Jsons.jsonNode(configBuilder.build());
}
@Override
protected JdbcDestinationHandler<? extends MinimumDestinationState> getDestinationHandler(final String databaseName,
final JdbcDatabase database,
final String rawTableSchema) {
return new NoOpJdbcDestinationHandler<>(databaseName, database, rawTableSchema, SQLDialect.DEFAULT);
}
@NotNull
@Override
protected List<Migration> getMigrations(final JdbcDatabase database,
final String databaseName,
final SqlGenerator sqlGenerator,
final DestinationHandler destinationHandler) {
return List.of();
}
private String getTrustStoreLocation() {
// trust store location code found at https://stackoverflow.com/a/56570588
final String trustStoreLocation = Optional.ofNullable(System.getProperty("javax.net.ssl.trustStore"))
@@ -104,4 +134,20 @@ public class MSSQLDestination extends AbstractJdbcDestination implements Destina
LOGGER.info("completed destination: {}", MSSQLDestination.class);
}
@Override
public boolean isV2Destination() {
return true;
}
@Override
protected boolean shouldAlwaysDisableTypeDedupe() {
return true;
}
@NotNull
@Override
protected JdbcSqlGenerator getSqlGenerator(@NotNull final JsonNode config) {
return new RawOnlySqlGenerator(new MSSQLNameTransformer());
}
}

77
airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/SqlServerOperations.java
View File

@@ -5,17 +5,26 @@
package io.airbyte.integrations.destination.mssql;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import io.airbyte.cdk.db.jdbc.JdbcDatabase;
import io.airbyte.cdk.integrations.base.JavaBaseConstants;
import io.airbyte.cdk.integrations.destination.async.model.PartialAirbyteMessage;
import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations;
import io.airbyte.cdk.integrations.destination.jdbc.SqlOperationsUtils;
import io.airbyte.protocol.models.v0.AirbyteRecordMessage;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SqlServerOperations implements SqlOperations {
private static final Logger LOGGER = LoggerFactory.getLogger(SqlServerOperations.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@Override
public void createSchemaIfNotExists(final JdbcDatabase database, final String schemaName) throws Exception {
final String query = String.format("IF NOT EXISTS ( SELECT * FROM sys.schemas WHERE name = '%s') EXEC('CREATE SCHEMA [%s]')",
@@ -37,10 +46,12 @@ public class SqlServerOperations implements SqlOperations {
+ "CREATE TABLE %s.%s ( \n"
+ "%s VARCHAR(64) PRIMARY KEY,\n"
+ "%s NVARCHAR(MAX),\n" // Microsoft SQL Server specific: NVARCHAR can store Unicode meanwhile VARCHAR - not
+ "%s DATETIMEOFFSET(7) DEFAULT SYSDATETIMEOFFSET()\n"
+ "%s DATETIMEOFFSET(7) DEFAULT SYSDATETIMEOFFSET(),\n"
+ "%s DATETIMEOFFSET(7),\n"
+ "%s NVARCHAR(MAX),\n"
+ ");\n",
schemaName, tableName, schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA,
JavaBaseConstants.COLUMN_NAME_EMITTED_AT);
schemaName, tableName, schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA,
JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, JavaBaseConstants.COLUMN_NAME_AB_META);
}
@Override
@@ -60,30 +71,60 @@ public class SqlServerOperations implements SqlOperations {
@Override
public void insertRecords(final JdbcDatabase database,
final List<AirbyteRecordMessage> records,
final List<PartialAirbyteMessage> records,
final String schemaName,
final String tempTableName)
throws SQLException {
// MSSQL has a limitation of 2100 parameters used in a query
// Airbyte inserts data with 3 columns (raw table) this limits to 700 records.
// Limited the variable to 500 records to
final int MAX_BATCH_SIZE = 500;
final int MAX_BATCH_SIZE = 400;
final String insertQueryComponent = String.format(
"INSERT INTO %s.%s (%s, %s, %s) VALUES\n",
"INSERT INTO %s.%s (%s, %s, %s, %s, %s) VALUES\n",
schemaName,
tempTableName,
JavaBaseConstants.COLUMN_NAME_AB_ID,
JavaBaseConstants.COLUMN_NAME_AB_RAW_ID,
JavaBaseConstants.COLUMN_NAME_DATA,
JavaBaseConstants.COLUMN_NAME_EMITTED_AT);
final String recordQueryComponent = "(?, ?, ?),\n";
final List<List<AirbyteRecordMessage>> batches = Lists.partition(records, MAX_BATCH_SIZE);
batches.forEach(record -> {
try {
SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, database, record);
} catch (final SQLException e) {
e.printStackTrace();
JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT,
JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT,
JavaBaseConstants.COLUMN_NAME_AB_META);
final String recordQueryComponent = "(?, ?, ?, ?, ?),\n";
final List<List<PartialAirbyteMessage>> batches = Lists.partition(records, MAX_BATCH_SIZE);
for (List<PartialAirbyteMessage> batch : batches) {
if (batch.isEmpty()) {
continue;
}
});
database.execute(connection -> {
final StringBuilder sqlStatement = new StringBuilder(insertQueryComponent);
for (PartialAirbyteMessage ignored : batch) {
sqlStatement.append(recordQueryComponent);
}
final var sql = sqlStatement.substring(0, sqlStatement.length() - 2) + ";";
try (final var statement = connection.prepareStatement(sql)) {
int i = 1;
for (PartialAirbyteMessage record : batch) {
final var id = UUID.randomUUID().toString();
statement.setString(i++, id);
statement.setString(i++, record.getSerialized());
statement.setTimestamp(i++, Timestamp.from(Instant.ofEpochMilli(Objects.requireNonNull(record.getRecord()).getEmittedAt())));
statement.setTimestamp(i++, null);
String metadata;
if (record.getRecord().getMeta() != null) {
try {
metadata = OBJECT_MAPPER.writeValueAsString(record.getRecord().getMeta());
} catch (Exception e) {
LOGGER.error("Failed to serialize record metadata for record {}", id, e);
metadata = null;
}
} else {
metadata = null;
}
statement.setString(i++, metadata);
}
statement.execute();
}
});
}
}
@Override

6
airbyte-integrations/connectors/destination-mssql/src/main/resources/spec.json
View File

@@ -114,6 +114,12 @@
}
}
]
},
"raw_data_schema": {
"type": "string",
"description": "The schema to write raw tables into (default: airbyte_internal)",
"title": "Raw Table Schema Name",
"order": 7
}
}
}

31
airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java
View File

@@ -26,14 +26,15 @@ import java.util.stream.Collectors;
import org.jooq.DSLContext;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.testcontainers.containers.MSSQLServerContainer;
@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.")
public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest {
private static MSSQLServerContainer<?> db;
private final StandardNameTransformer namingResolver = new StandardNameTransformer();
private JsonNode config;
private DSLContext dslContext;
@Override
protected String getImageName() {
@@ -93,17 +94,16 @@ public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes
}
private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException {
try (final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, null)) {
return getDatabase(dslContext).query(
ctx -> {
ctx.fetch(String.format("USE %s;", config.get(JdbcUtils.DATABASE_KEY)));
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.stream()
.map(this::getJsonFromRecord)
.collect(Collectors.toList());
});
}
final DSLContext dslContext = DatabaseConnectionHelper.createDslContext(db, null);
return getDatabase(dslContext).query(
ctx -> {
ctx.fetch(String.format("USE %s;", config.get(JdbcUtils.DATABASE_KEY)));
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))
.stream()
.map(this::getJsonFromRecord)
.collect(Collectors.toList());
});
}
@BeforeAll
@@ -134,7 +134,7 @@ public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes
protected void setup(final TestDestinationEnv testEnv, final HashSet<String> TEST_SCHEMAS) throws SQLException {
final JsonNode configWithoutDbName = getConfig(db);
final String dbName = Strings.addRandomSuffix("db", "_", 10);
dslContext = getDslContext(configWithoutDbName);
DSLContext dslContext = getDslContext(configWithoutDbName);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch(String.format("CREATE DATABASE %s;", dbName));
@@ -150,8 +150,9 @@ public class MSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes
}
@Override
protected void tearDown(final TestDestinationEnv testEnv) {
dslContext.close();
protected void tearDown(final TestDestinationEnv testEnv) throws Exception {
db.stop();
db.close();
}
@Override

13
airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java
View File

@@ -27,16 +27,16 @@ import org.jooq.DSLContext;
import org.jooq.SQLDialect;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.testcontainers.containers.MSSQLServerContainer;
import org.testcontainers.utility.DockerImageName;
@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.")
public class MSSQLDestinationAcceptanceTestSSL extends JdbcDestinationAcceptanceTest {
private static MSSQLServerContainer<?> db;
private final StandardNameTransformer namingResolver = new StandardNameTransformer();
private JsonNode configWithoutDbName;
private JsonNode config;
private DSLContext dslContext;
@Override
protected String getImageName() {
@@ -109,7 +109,7 @@ public class MSSQLDestinationAcceptanceTestSSL extends JdbcDestinationAcceptance
ctx -> {
ctx.fetch(String.format("USE %s;", config.get(JdbcUtils.DATABASE_KEY)));
return ctx
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))
.stream()
.map(this::getJsonFromRecord)
.collect(Collectors.toList());
@@ -143,9 +143,9 @@ public class MSSQLDestinationAcceptanceTestSSL extends JdbcDestinationAcceptance
// 2. /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "A_Str0ng_Required_Password"
@Override
protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) throws SQLException {
configWithoutDbName = getConfig(db);
JsonNode configWithoutDbName = getConfig(db);
final String dbName = Strings.addRandomSuffix("db", "_", 10);
dslContext = getDslContext(configWithoutDbName);
DSLContext dslContext = getDslContext(configWithoutDbName);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch(String.format("CREATE DATABASE %s;", dbName));
@@ -162,7 +162,8 @@ public class MSSQLDestinationAcceptanceTestSSL extends JdbcDestinationAcceptance
@Override
protected void tearDown(final TestDestinationEnv testEnv) {
dslContext.close();
// no op, called in {@link
// io.airbyte.integrations.destination.mssql.MSSQLDestinationAcceptanceTestSSL.cleanUp}
}
@AfterAll

2
airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshKeyMSSQLDestinationAcceptanceTest.java
View File

@@ -5,7 +5,9 @@
package io.airbyte.integrations.destination.mssql;
import io.airbyte.cdk.integrations.base.ssh.SshTunnel;
import org.junit.jupiter.api.Disabled;
@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.")
public class SshKeyMSSQLDestinationAcceptanceTest extends SshMSSQLDestinationAcceptanceTest {
@Override

12
airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java
View File

@@ -18,11 +18,13 @@ import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAccep
import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator;
import io.airbyte.commons.functional.CheckedFunction;
import io.airbyte.commons.json.Jsons;
import io.airbyte.integrations.base.destination.typing_deduping.StreamId;
import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.RandomStringUtils;
import org.jooq.DSLContext;
import org.junit.jupiter.api.Disabled;
import org.testcontainers.containers.MSSQLServerContainer;
import org.testcontainers.containers.Network;
@@ -30,6 +32,7 @@ import org.testcontainers.containers.Network;
* Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file
* or with a password.
*/
@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.")
public abstract class SshMSSQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest {
private final StandardNameTransformer namingResolver = new StandardNameTransformer();
@@ -62,8 +65,7 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends JdbcDestinationA
@Override
protected List<JsonNode> retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace)
throws Exception {
final String tableName = namingResolver.getIdentifier(streamName);
return retrieveRecordsFromTable(tableName, namespace);
return List.of();
}
@Override
@@ -72,7 +74,7 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends JdbcDestinationA
final String namespace,
final JsonNode streamSchema)
throws Exception {
return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace)
return retrieveRecordsFromTable(StreamId.concatenateRawTableName(namespace, streamName), "airbyte_internal")
.stream()
.map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA))
.collect(Collectors.toList());
@@ -107,8 +109,8 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends JdbcDestinationA
ctx -> ctx
.fetch(String.format("USE %s;"
+ "SELECT * FROM %s.%s ORDER BY %s ASC;",
database, schema, tableName.toLowerCase(),
JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
database, "airbyte_internal", tableName.toLowerCase(),
JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))
.stream()
.map(this::getJsonFromRecord)
.collect(Collectors.toList())));

2
airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshPasswordMSSQLDestinationAcceptanceTest.java
View File

@@ -5,7 +5,9 @@
package io.airbyte.integrations.destination.mssql;
import io.airbyte.cdk.integrations.base.ssh.SshTunnel;
import org.junit.jupiter.api.Disabled;
@Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.")
public class SshPasswordMSSQLDestinationAcceptanceTest extends SshMSSQLDestinationAcceptanceTest {
@Override

65
docs/integrations/destinations/mssql-migrations.md Normal file
View File

@@ -0,0 +1,65 @@
# MS SQL Server Migration Guide
## Upgrading to 1.0.0
This version removes the option to use "normalization" with Microsoft SQL Server. It also changes
the schema and database of Airbyte's "raw" tables to be compatible with the new
[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2)
format. These changes will likely require updates to downstream dbt / SQL models. After this update,
Airbyte will only produce the "raw" v2 tables, which store all content in JSON. These changes remove
the ability to do deduplicated syncs with Microsoft SQL Server.
If you are interested in the Microsoft SQL Server destination gaining the full features
of Destinations V2 (including final tables), click [[https://github.com/airbytehq/airbyte/discussions/37010]]
to register your interest.
This upgrade will ignore any existing raw tables and will not migrate any data to the new schema.
For each stream, you should perform the following query to migrate the data from the old raw table
to the new raw table:
```sql
-- assumes your schema was 'default'
-- replace `{{stream_name}}` with replace your stream name
CREATE TABLE airbyte_internal.default_raw__stream_{{stream_name}} (
_airbyte_raw_id VARCHAR(64) PRIMARY KEY,
_airbyte_data NVARCHAR(MAX),
_airbyte_extracted_at DATETIMEOFFSET(7) DEFAULT SYSDATETIMEOFFSET(),
_airbyte_loaded_at DATETIMEOFFSET(7),
_airbyte_meta NVARCHAR(MAX)
);
INSERT INTO airbyte_internal.default_raw__stream_{{stream_name}}
SELECT
_airbyte_ab_id AS _airbyte_raw_id,
_airbyte_data as _airbyte_data,
_airbyte_emitted_at as _airbyte_extracted_at,
NULL as _airbyte_loaded_at,
NULL as _airbyte_meta
FROM airbyte._airbyte_raw_{{stream_name}}
```
**Airbyte will not delete any of your v1 data.**
### Schema and the Internal Schema
We have split the raw and final tables into their own schemas. For the Microsoft SQL Server destination, this means that
we will only write into the raw table which will live in the `airbyte_internal` schema.
The tables written into this schema will be prefixed with either the default database provided in
the `DB Name` field when configuring Microsoft SQL Server (but can also be overridden in the connection). You can
change the "raw" database from the default `airbyte_internal` by supplying a value for
`Raw Table Schema Name`.
For Example:
- Schema: `default`
- Stream Name: `my_stream`
Writes to `airbyte_intneral.default_raw__stream_my_stream`
Where as:
- Schema: `default`
- Stream Name: `my_stream`
- Raw Table Schema Name: `raw_data`
Writes to `raw_data.default_raw__stream_my_stream`

3
docs/integrations/destinations/mssql.md
View File

@@ -115,7 +115,8 @@ Using this feature requires additional configuration, when creating the source.
## Changelog
| Version | Date | Pull Request | Subject |
| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- |
|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
| 1.0.0 | 2024-04-11 | [\#36050](https://github.com/airbytehq/airbyte/pull/36050) | Update to Dv2 Table Format and Remove normalization |
| 0.2.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 |
| 0.1.25 | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size |
| 0.1.24 | 2023-06-05 | [\#27034](https://github.com/airbytehq/airbyte/pull/27034) | Internal code change for future development (install normalization packages inside connector) |