jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Destination Mssql: install normalization (#27034)

Browse Source 
* install in mssql

* changelog+metadata

* update changelog message
This commit is contained in:
Edward Gao
2023-06-05 17:00:39 -07:00
committed by GitHub
parent 3aa4512b46
commit bd22b25ccf
8 changed files with 62 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile
View File

@@ -8,9 +8,29 @@
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
RUN yum install -y python3 python3-devel jq sshpass git && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
pip3 install dbt-sqlserver==1.0.0
# Luckily, none of normalization's files conflict with destination-mssql's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-mssql-strict-encrypt
ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
@@ -24,5 +44,8 @@ ENV APPLICATION destination-mssql-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.23
LABEL io.airbyte.version=0.1.24
LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

4
airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle
View File

@@ -29,3 +29,7 @@ dependencies {
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)
}
tasks.named("airbyteDocker") {
dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql
}

2
airbyte-integrations/connectors/destination-mssql-strict-encrypt/metadata.yaml
View File

@@ -7,7 +7,7 @@ data:
connectorSubtype: database
connectorType: destination
definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
dockerImageTag: 0.1.23
dockerImageTag: 0.1.24
dockerRepository: airbyte/destination-mssql-strict-encrypt
githubIssueLabel: destination-mssql
icon: mssql.svg

25
airbyte-integrations/connectors/destination-mssql/Dockerfile
View File

@@ -8,9 +8,29 @@
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
RUN yum install -y python3 python3-devel jq sshpass git && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
pip3 install dbt-sqlserver==1.0.0
# Luckily, none of normalization's files conflict with destination-mssql's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-mssql
ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
@@ -24,5 +44,8 @@ ENV APPLICATION destination-mssql
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.23
LABEL io.airbyte.version=0.1.24
LABEL io.airbyte.name=airbyte/destination-mssql
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

4
airbyte-integrations/connectors/destination-mssql/build.gradle
View File

@@ -27,3 +27,7 @@ dependencies {
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)
}
tasks.named("airbyteDocker") {
dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql
}

2
airbyte-integrations/connectors/destination-mssql/metadata.yaml
View File

@@ -2,7 +2,7 @@ data:
connectorSubtype: database
connectorType: destination
definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
dockerImageTag: 0.1.23
dockerImageTag: 0.1.24
dockerRepository: airbyte/destination-mssql
githubIssueLabel: destination-mssql
icon: mssql.svg

4
docs/integrations/destinations/mssql.md
View File

@@ -56,7 +56,7 @@ Make sure your SQL Server database can be accessed by Airbyte. If your database
#### **Permissions**
You need a user configured in SQL Server that can create tables and write rows. We highly recommend creating an Airbyte-specific user for this purpose.
In order to allow for normalization, please grant ALTER permissions for the user configured.
In order to allow for normalization, please grant ALTER permissions for the user configured.
#### Target Database
@@ -116,6 +116,7 @@ Using this feature requires additional configuration, when creating the source.
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
| 0.1.24 | 2023-06-05 | [\#27034](https://github.com/airbytehq/airbyte/pull/27034) | Internal code change for future development (install normalization packages inside connector) |
| 0.1.23 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing |
| 0.1.22 | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 |
| 0.1.20 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors |
@@ -142,6 +143,7 @@ Using this feature requires additional configuration, when creating the source.
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
| 0.1.24 | 2023-06-05 | [\#27034](https://github.com/airbytehq/airbyte/pull/27034) | Internal code change for future development (install normalization packages inside connector) |
| 0.1.23 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing |
| 0.1.22 | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 |
| 0.1.21 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage |

2
tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
View File

@@ -539,7 +539,7 @@ BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = {
"dockerfile": "mssql.Dockerfile",
"dbt_adapter": "dbt-sqlserver==1.0.0",
"integration_name": "mssql",
"supports_in_connector_normalization": False,
"supports_in_connector_normalization": True,
"yum_packages": [],
},
"destination-mysql": {