jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 18:14:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

airbyte-ci,gradle: replace airbyte-docker with airbyte-ci (#30743)

Browse Source
This commit is contained in:
Marius Posta
2023-10-04 08:38:17 -07:00
committed by GitHub
parent 5d6597e5b6
commit f8edc18039
127 changed files with 353 additions and 2601 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-cdk/java/airbyte-cdk/archive/base-standard-source-test-file/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'application'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}
dependencies {

2
airbyte-cdk/java/airbyte-cdk/archive/bases-destination-jdbc/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'application'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
id 'airbyte-integration-test-java'
}

2
airbyte-cdk/java/airbyte-cdk/archive/source-jdbc/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'application'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
id 'airbyte-integration-test-java'
id "java-library"
// https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures

2
airbyte-cdk/java/airbyte-cdk/archive/standard-source-test/build.gradle
View File

@@ -6,7 +6,7 @@ buildscript {
plugins {
id 'application'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}
import org.jsoup.Jsoup;

1
airbyte-cdk/java/airbyte-cdk/build.gradle
View File

@@ -1,7 +1,6 @@
allprojects {
apply plugin: 'java-library'
apply plugin: 'maven-publish'
apply plugin: 'airbyte-docker'
apply plugin: 'airbyte-java-cdk'
apply plugin: 'airbyte-integration-test-java'
apply plugin: 'airbyte-performance-test-java'

2
airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java
View File

@@ -27,6 +27,7 @@ import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
class DebeziumEventUtilsTest {
@@ -60,6 +61,7 @@ class DebeziumEventUtilsTest {
deepCompare(expectedDelete, actualDelete);
}
@Disabled("See https://github.com/airbytehq/airbyte/issues/31032")
@Test
void testConvertMongoDbChangeEvent() throws IOException {
final String objectId = "64f24244f95155351c4185b1";

2
airbyte-cdk/python/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}
def generateComponentManifestClassFiles = tasks.register('generateComponentManifestClassFiles', Exec) {

8
airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py
View File

@@ -7,7 +7,7 @@ import sys
from pathlib import Path
from typing import Iterable, Optional, Set, Tuple
from connector_ops.utils import Connector
from connector_ops.utils import Connector, ConnectorLanguage
from pydash.objects import get
@@ -235,7 +235,11 @@ def check_connector_has_no_critical_vulnerabilities(connector: Connector) -> boo
def check_metadata_version_matches_dockerfile_label(connector: Connector) -> bool:
return connector.version_in_dockerfile_label == connector.version
version_in_dockerfile = connector.version_in_dockerfile_label
if version_in_dockerfile is None:
# Java connectors don't have Dockerfiles.
return connector.language == ConnectorLanguage.JAVA
return version_in_dockerfile == connector.version
QA_CHECKS = [

21
airbyte-ci/connectors/connector_ops/connector_ops/utils.py
View File

@@ -314,12 +314,8 @@ class Connector:
return ConnectorLanguage.LOW_CODE
if Path(self.code_directory / "setup.py").is_file() or Path(self.code_directory / "pyproject.toml").is_file():
return ConnectorLanguage.PYTHON
try:
with open(self.code_directory / "Dockerfile") as dockerfile:
if "FROM airbyte/integration-base-java" in dockerfile.read():
return ConnectorLanguage.JAVA
except FileNotFoundError:
pass
if Path(self.code_directory / "src" / "main" / "java").exists():
return ConnectorLanguage.JAVA
return None
@property
@@ -329,11 +325,14 @@ class Connector:
return self.metadata["dockerImageTag"]
@property
def version_in_dockerfile_label(self) -> str:
with open(self.code_directory / "Dockerfile") as f:
for line in f:
if "io.airbyte.version" in line:
return line.split("=")[1].strip()
def version_in_dockerfile_label(self) -> Optional[str]:
try:
with open(self.code_directory / "Dockerfile") as f:
for line in f:
if "io.airbyte.version" in line:
return line.split("=")[1].strip()
except FileNotFoundError as e:
return None
raise ConnectorVersionNotFound(
"""
Could not find the connector version from its Dockerfile.

10
airbyte-ci/connectors/connector_ops/poetry.lock generated
View File

@@ -1058,6 +1058,7 @@ files = [
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
{file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
{file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
{file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
{file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
@@ -1065,8 +1066,15 @@ files = [
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
{file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
{file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
{file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
{file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
{file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
{file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
@@ -1083,6 +1091,7 @@ files = [
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
{file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
{file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
{file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
{file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
@@ -1090,6 +1099,7 @@ files = [
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
{file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
{file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
{file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
{file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},

2
airbyte-ci/connectors/connector_ops/pyproject.toml
View File

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "connector_ops"
version = "0.2.3"
version = "0.2.4"
description = "Packaged maintained by the connector operations team to perform CI for connectors"
authors = ["Airbyte <contact@airbyte.io>"]

2
airbyte-ci/connectors/pipelines/README.md
View File

@@ -398,6 +398,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
## Changelog
| Version | PR | Description |
|---------| --------------------------------------------------------- |-----------------------------------------------------------------------------------------------------------|
| 1.4.4 | [#30743](https://github.com/airbytehq/airbyte/pull/30743) | Add `--disable-report-auto-open` and `--use-host-gradle-dist-tar` to allow gradle integration. |
| 1.4.3 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Add --version and version check |
| 1.4.2 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Remove directory name requirement |
| 1.4.1 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Load base migration guide into QA Test container for strict encrypt variants |
@@ -486,3 +487,4 @@ and when in doubt, you can reinstall the CLI with the `--force` option:
```bash
$ pipx reinstall pipelines --force
```

5
airbyte-ci/connectors/pipelines/pipelines/bases.py
View File

@@ -624,8 +624,9 @@ class ConnectorReport(Report):
absolute_path = await local_html_path.resolve()
if self.pipeline_context.is_local:
self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}")
self.pipeline_context.logger.info("Opening HTML report in browser.")
webbrowser.open(absolute_path.as_uri())
if self.pipeline_context.open_report_in_browser:
self.pipeline_context.logger.info("Opening HTML report in browser.")
webbrowser.open(absolute_path.as_uri())
if self.remote_storage_enabled:
await self.save_remote(local_html_path, self.html_report_remote_storage_key, "text/html")
self.pipeline_context.logger.info(f"HTML report uploaded to {self.html_report_url}")

13
airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py
View File

@@ -9,9 +9,10 @@ import platform
import anyio
from connector_ops.utils import ConnectorLanguage
from dagger import Platform
from pipelines.bases import ConnectorReport, StepResult
from pipelines.builds import common, java_connectors, python_connectors
from pipelines.builds import java_connectors, python_connectors
from pipelines.builds.common import LoadContainerToLocalDockerHost, StepStatus
from pipelines.consts import LOCAL_BUILD_PLATFORM
from pipelines.contexts import ConnectorContext
@@ -25,9 +26,6 @@ LANGUAGE_BUILD_CONNECTOR_MAPPING = {
ConnectorLanguage.JAVA: java_connectors.run_connector_build,
}
BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}")
async def run_connector_build(context: ConnectorContext) -> StepResult:
"""Run a build pipeline for a single connector."""
@@ -50,9 +48,8 @@ async def run_connector_build_pipeline(context: ConnectorContext, semaphore: any
async with context:
build_result = await run_connector_build(context)
step_results.append(build_result)
if context.is_local and build_result.status is common.StepStatus.SUCCESS:
connector_to_load_to_local_docker_host = build_result.output_artifact[LOCAL_BUILD_PLATFORM]
load_image_result = await common.LoadContainerToLocalDockerHost(context, connector_to_load_to_local_docker_host).run()
if context.is_local and build_result.status is StepStatus.SUCCESS:
load_image_result = await LoadContainerToLocalDockerHost(context, LOCAL_BUILD_PLATFORM, build_result.output_artifact).run()
step_results.append(load_image_result)
context.report = ConnectorReport(context, step_results, name="BUILD RESULTS")
return context.report

62
airbyte-ci/connectors/pipelines/pipelines/builds/common.py
View File

@@ -3,51 +3,69 @@
#
from abc import ABC
from typing import Tuple
from typing import List, Tuple
import docker
from dagger import Container, Platform
from dagger import Container, ExecError, Platform, QueryError
from pipelines.bases import Step, StepResult, StepStatus
from pipelines.consts import BUILD_PLATFORMS
from pipelines.contexts import ConnectorContext
from pipelines.utils import export_container_to_tarball
class BuildConnectorImageBase(Step, ABC):
class BuildConnectorImagesBase(Step, ABC):
"""
A step to build connector images for a set of platforms.
"""
@property
def title(self):
return f"Build {self.context.connector.technical_name} docker image for platform {self.build_platform}"
return f"Build {self.context.connector.technical_name} docker image for platform(s) {', '.join(self.build_platforms)}"
def __init__(self, context: ConnectorContext, build_platform: Platform) -> None:
self.build_platform = build_platform
def __init__(self, context: ConnectorContext, *build_platforms: List[Platform]) -> None:
self.build_platforms = build_platforms if build_platforms else BUILD_PLATFORMS
super().__init__(context)
class BuildConnectorImageForAllPlatformsBase(Step, ABC):
ALL_PLATFORMS = BUILD_PLATFORMS
title = f"Build connector image for {BUILD_PLATFORMS}"
def get_success_result(self, build_results_per_platform: dict[Platform, Container]) -> StepResult:
return StepResult(
self,
StepStatus.SUCCESS,
stdout="The connector image was successfully built for all platforms.",
output_artifact=build_results_per_platform,
async def _run(self, *args) -> StepResult:
build_results_per_platform = {}
for platform in self.build_platforms:
try:
connector = await self._build_connector(platform, *args)
try:
await connector.with_exec(["spec"])
except ExecError:
return StepResult(
self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {platform}."
)
build_results_per_platform[platform] = connector
except QueryError as e:
return StepResult(self, StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}")
success_message = (
f"The {self.context.connector.technical_name} docker image "
f"was successfully built for platform(s) {', '.join(self.build_platforms)}"
)
return StepResult(self, StepStatus.SUCCESS, stdout=success_message, output_artifact=build_results_per_platform)
async def _build_connector(self, platform: Platform, *args) -> Container:
"""Implement the generation of the image for the platform and return the corresponding container.
Returns:
Container: The container to package as a docker image for this platform.
"""
raise NotImplementedError("`BuildConnectorImagesBase`s must define a '_build_connector' attribute.")
class LoadContainerToLocalDockerHost(Step):
IMAGE_TAG = "dev"
def __init__(self, context: ConnectorContext, container: Container) -> None:
def __init__(self, context: ConnectorContext, platform: Platform, containers: dict[Platform, Container]) -> None:
super().__init__(context)
self.container = container
self.platform = platform
self.container = containers[platform]
@property
def title(self):
return f"Load {self.image_name}:{self.IMAGE_TAG} to the local docker host."
return f"Load {self.image_name}:{self.IMAGE_TAG} for platform {self.platform} to the local docker host."
@property
def image_name(self) -> Tuple:

91
airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py
View File

@@ -2,10 +2,13 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
from dagger import ExecError, File, QueryError
from typing import List, Optional, Tuple, Union
from dagger import Container, Directory, ExecError, File, Host, Platform, QueryError
from pipelines.actions import environments
from pipelines.bases import StepResult, StepStatus
from pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase
from pipelines.builds.common import BuildConnectorImagesBase
from pipelines.consts import LOCAL_BUILD_PLATFORM
from pipelines.contexts import ConnectorContext
from pipelines.gradle import GradleTask
@@ -18,76 +21,50 @@ class BuildConnectorDistributionTar(GradleTask):
title = "Build connector tar"
gradle_task_name = "distTar"
async def _run(self) -> StepResult:
result = await super()._run()
if result.status is not StepStatus.SUCCESS:
return result
with_built_tar = result.output_artifact.with_workdir(f"{self.context.connector.code_directory}/build/distributions")
distributions = await with_built_tar.directory(".").entries()
tar_files = [f for f in distributions if f.endswith(".tar")]
if len(tar_files) == 1:
return StepResult(
self,
StepStatus.SUCCESS,
stdout="The distribution tar file for the current java connector was built.",
output_artifact=with_built_tar.file(tar_files[0]),
)
elif len(tar_files) == 0:
return StepResult(
self,
StepStatus.FAILURE,
stderr="The distribution tar file for the current java connector was not built.",
)
else:
return StepResult(
self,
StepStatus.FAILURE,
stderr="The distributions directory for this java connector contains multiple tar files. "
"We can't infer which one should be used. "
"Please review and delete any unnecessary tar files.",
)
class BuildConnectorImage(BuildConnectorImageBase):
class BuildConnectorImages(BuildConnectorImagesBase):
"""
A step to build a Java connector image using the distTar Gradle task.
A step to build Java connector images using the distTar Gradle task.
"""
async def _run(self, distribution_tar: File) -> StepResult:
async def _run(self, dist_dir: Directory) -> StepResult:
dist_tar: File
try:
java_connector = await environments.with_airbyte_java_connector(self.context, distribution_tar, self.build_platform)
try:
await java_connector.with_exec(["spec"])
except ExecError:
return StepResult(
self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {self.build_platform}."
dir_files = await dist_dir.entries()
tar_files = [f for f in dir_files if f.endswith(".tar")]
num_files = len(tar_files)
if num_files != 1:
error_message = (
"The distribution tar file for the current java connector was not built."
if num_files == 0
else "More than one distribution tar file was built for the current java connector."
)
return StepResult(
self, StepStatus.SUCCESS, stdout="The connector image was successfully built.", output_artifact=java_connector
)
return StepResult(self, StepStatus.FAILURE, stderr=error_message)
dist_tar = dist_dir.file(tar_files[0])
except QueryError as e:
return StepResult(self, StepStatus.FAILURE, stderr=str(e))
return await super()._run(dist_tar)
class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase):
"""Build a Java connector image for all platforms."""
async def _run(self, distribution_tar: File) -> StepResult:
build_results_per_platform = {}
for platform in self.ALL_PLATFORMS:
build_connector_step_result = await BuildConnectorImage(self.context, platform).run(distribution_tar)
if build_connector_step_result.status is not StepStatus.SUCCESS:
return build_connector_step_result
build_results_per_platform[platform] = build_connector_step_result.output_artifact
return self.get_success_result(build_results_per_platform)
async def _build_connector(self, platform: Platform, dist_tar: File) -> Container:
return await environments.with_airbyte_java_connector(self.context, dist_tar, platform)
async def run_connector_build(context: ConnectorContext) -> StepResult:
"""Create the java connector distribution tar file and build the connector image."""
if context.use_host_gradle_dist_tar and context.is_local:
# Special case: use a local dist tar to speed up local development.
dist_dir = await context.dagger_client.host().directory(dist_tar_directory_path(context), include=["*.tar"])
# Speed things up by only building for the local platform.
return await BuildConnectorImages(context, LOCAL_BUILD_PLATFORM).run(dist_dir)
# Default case: distribution tar is built by the dagger pipeline.
build_connector_tar_result = await BuildConnectorDistributionTar(context).run()
if build_connector_tar_result.status is not StepStatus.SUCCESS:
return build_connector_tar_result
dist_dir = await build_connector_tar_result.output_artifact.directory(dist_tar_directory_path(context))
return await BuildConnectorImages(context).run(dist_dir)
return await BuildConnectorImageForAllPlatforms(context).run(build_connector_tar_result.output_artifact)
def dist_tar_directory_path(context: ConnectorContext) -> str:
return f"{context.connector.code_directory}/build/distributions"

31
airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py
View File

@@ -2,39 +2,22 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
from dagger import QueryError
from dagger import Container, Platform
from pipelines.actions.environments import with_airbyte_python_connector
from pipelines.bases import StepResult, StepStatus
from pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase
from pipelines.bases import StepResult
from pipelines.builds.common import BuildConnectorImagesBase
from pipelines.contexts import ConnectorContext
class BuildConnectorImage(BuildConnectorImageBase):
class BuildConnectorImages(BuildConnectorImagesBase):
"""
A step to build a Python connector image.
A spec command is run on the container to validate it was built successfully.
"""
async def _run(self) -> StepResult:
connector = await with_airbyte_python_connector(self.context, self.build_platform)
try:
return await self.get_step_result(connector.with_exec(["spec"]))
except QueryError as e:
return StepResult(self, StepStatus.FAILURE, stderr=str(e))
class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase):
"""Build a Python connector image for all platforms."""
async def _run(self) -> StepResult:
build_results_per_platform = {}
for platform in self.ALL_PLATFORMS:
build_connector_step_result = await BuildConnectorImage(self.context, platform).run()
if build_connector_step_result.status is not StepStatus.SUCCESS:
return build_connector_step_result
build_results_per_platform[platform] = build_connector_step_result.output_artifact
return self.get_success_result(build_results_per_platform)
async def _build_connector(self, platform: Platform) -> Container:
return await with_airbyte_python_connector(self.context, platform)
async def run_connector_build(context: ConnectorContext) -> StepResult:
return await BuildConnectorImageForAllPlatforms(context).run()
return await BuildConnectorImages(context).run()

22
airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py
View File

@@ -165,6 +165,13 @@ def get_selected_connectors_with_modified_files(
default=False,
type=bool,
)
@click.option(
"--enable-report-auto-open/--disable-report-auto-open",
is_flag=True,
help=("When enabled, finishes by opening a browser window to display an HTML report."),
default=True,
type=bool,
)
@click.pass_context
def connectors(
ctx: click.Context,
@@ -179,6 +186,7 @@ def connectors(
execute_timeout: int,
enable_dependency_scanning: bool,
use_local_cdk: bool,
enable_report_auto_open: bool,
):
"""Group all the connectors-ci command."""
validate_environment(ctx.obj["is_local"], use_remote_secrets)
@@ -188,6 +196,7 @@ def connectors(
ctx.obj["concurrency"] = concurrency
ctx.obj["execute_timeout"] = execute_timeout
ctx.obj["use_local_cdk"] = use_local_cdk
ctx.obj["open_report_in_browser"] = enable_report_auto_open
ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files(
names,
support_levels,
@@ -295,8 +304,15 @@ def test(
@connectors.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.")
@click.option(
"--use-host-gradle-dist-tar",
is_flag=True,
help="Use gradle distTar output from host for java connectors.",
default=False,
type=bool,
)
@click.pass_context
def build(ctx: click.Context) -> bool:
def build(ctx: click.Context, use_host_gradle_dist_tar: bool) -> bool:
"""Runs a build pipeline for the selected connectors."""
connectors_contexts = [
@@ -315,9 +331,13 @@ def build(ctx: click.Context) -> bool:
ci_context=ctx.obj.get("ci_context"),
ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
use_local_cdk=ctx.obj.get("use_local_cdk"),
open_report_in_browser=ctx.obj.get("open_report_in_browser"),
use_host_gradle_dist_tar=use_host_gradle_dist_tar,
)
for connector in ctx.obj["selected_connectors_with_modified_files"]
]
if use_host_gradle_dist_tar and not ctx.obj["is_local"]:
raise Exception("flag --use-host-gradle-dist-tar requires --is-local")
anyio.run(
run_connectors_pipelines,
connectors_contexts,

6
airbyte-ci/connectors/pipelines/pipelines/contexts.py
View File

@@ -334,6 +334,8 @@ class ConnectorContext(PipelineContext):
fast_tests_only: bool = False,
code_tests_only: bool = False,
use_local_cdk: bool = False,
use_host_gradle_dist_tar: bool = False,
open_report_in_browser: bool = True,
):
"""Initialize a connector context.
@@ -355,6 +357,8 @@ class ConnectorContext(PipelineContext):
fail_fast (bool, optional): Whether to fail fast. Defaults to False.
fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False.
code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False.
use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False.
open_report_in_browser (bool, optional): Open HTML report in browser window. Defaults to True.
"""
self.pipeline_name = pipeline_name
@@ -370,6 +374,8 @@ class ConnectorContext(PipelineContext):
self.fast_tests_only = fast_tests_only
self.code_tests_only = code_tests_only
self.use_local_cdk = use_local_cdk
self.use_host_gradle_dist_tar = use_host_gradle_dist_tar
self.open_report_in_browser = open_report_in_browser
super().__init__(
pipeline_name=pipeline_name,

14
airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py
View File

@@ -10,9 +10,9 @@ import anyio
from dagger import File, QueryError
from pipelines.actions import environments, secrets
from pipelines.bases import StepResult, StepStatus
from pipelines.builds import LOCAL_BUILD_PLATFORM
from pipelines.builds.java_connectors import BuildConnectorDistributionTar, BuildConnectorImage
from pipelines.builds.java_connectors import BuildConnectorDistributionTar, BuildConnectorImages, dist_tar_directory_path
from pipelines.builds.normalization import BuildOrPullNormalization
from pipelines.consts import LOCAL_BUILD_PLATFORM
from pipelines.contexts import ConnectorContext
from pipelines.gradle import GradleTask
from pipelines.tests.common import AcceptanceTests
@@ -23,7 +23,7 @@ class IntegrationTests(GradleTask):
"""A step to run integrations tests for Java connectors using the integrationTestJava Gradle task."""
title = "Java Connector Integration Tests"
gradle_task_name = "integrationTestJava -x airbyteDocker -x assemble"
gradle_task_name = "integrationTestJava -x buildConnectorImage -x assemble"
mount_connector_secrets = True
bind_to_docker_host = True
@@ -81,9 +81,8 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]:
if build_distribution_tar_results.status is StepStatus.FAILURE:
return step_results
build_connector_image_results = await BuildConnectorImage(context, LOCAL_BUILD_PLATFORM).run(
build_distribution_tar_results.output_artifact
)
dist_tar_dir = await build_distribution_tar_results.output_artifact.directory(dist_tar_directory_path(context))
build_connector_image_results = await BuildConnectorImages(context, LOCAL_BUILD_PLATFORM).run(dist_tar_dir)
step_results.append(build_connector_image_results)
if build_connector_image_results.status is StepStatus.FAILURE:
return step_results
@@ -105,7 +104,8 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]:
else:
normalization_tar_file = None
connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact)
connector_container = build_connector_image_results.output_artifact[LOCAL_BUILD_PLATFORM]
connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container)
integration_tests_results = await IntegrationTests(context).run(connector_image_tar_file, normalization_tar_file)
step_results.append(integration_tests_results)

9
airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py
View File

@@ -11,8 +11,8 @@ import asyncer
from dagger import Container
from pipelines.actions import environments, secrets
from pipelines.bases import Step, StepResult, StepStatus
from pipelines.builds import LOCAL_BUILD_PLATFORM
from pipelines.builds.python_connectors import BuildConnectorImage
from pipelines.builds.python_connectors import BuildConnectorImages
from pipelines.consts import LOCAL_BUILD_PLATFORM
from pipelines.contexts import ConnectorContext
from pipelines.helpers.steps import run_steps
from pipelines.tests.common import AcceptanceTests, PytestStep
@@ -126,13 +126,14 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]:
step_results = await run_steps(
[
ConnectorPackageInstall(context),
BuildConnectorImage(context, LOCAL_BUILD_PLATFORM),
BuildConnectorImages(context, LOCAL_BUILD_PLATFORM),
]
)
if any([step_result.status is StepStatus.FAILURE for step_result in step_results]):
return step_results
connector_package_install_results, build_connector_image_results = step_results[0], step_results[1]
connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact)
connector_container = build_connector_image_results.output_artifact[LOCAL_BUILD_PLATFORM]
connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container)
connector_container = connector_package_install_results.output_artifact
context.connector_secrets = await secrets.get_connector_secrets(context)

2
airbyte-ci/connectors/pipelines/poetry.lock generated
View File

@@ -380,7 +380,7 @@ url = "../common_utils"
[[package]]
name = "connector-ops"
version = "0.2.3"
version = "0.2.4"
description = "Packaged maintained by the connector operations team to perform CI for connectors"
optional = false
python-versions = "^3.10"

2
airbyte-ci/connectors/pipelines/pyproject.toml
View File

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "pipelines"
version = "1.4.3"
version = "1.4.4"
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
authors = ["Airbyte <contact@airbyte.io>"]

2
airbyte-integrations/bases/base-java/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'java-library'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}
dependencies {

34
airbyte-integrations/bases/base-normalization/build.gradle
View File

@@ -1,7 +1,7 @@
import org.apache.tools.ant.taskdefs.condition.Os
plugins {
id 'airbyte-docker'
id 'airbyte-docker-legacy'
id 'airbyte-python'
}
@@ -43,20 +43,6 @@ tasks.named('check').configure {
dependsOn generate
}
def customIntegrationTestPython = tasks.register('customIntegrationTestPython', PythonTask) {
module = "pytest"
command = "-s integration_tests"
}
customIntegrationTestPython.configure {
dependsOn tasks.named('installTestReqs')
dependsOn tasks.named('assemble')
}
def customIntegrationTestsCoverage = tasks.named('_customIntegrationTestsCoverage')
customIntegrationTestsCoverage.configure {
dependsOn tasks.named('assemble')
}
[
'bigquery',
'mysql',
@@ -69,25 +55,13 @@ customIntegrationTestsCoverage.configure {
'tidb',
'duckdb',
].each {destinationName ->
customIntegrationTestPython.configure {
def integrationTestPython = tasks.named('integrationTestPython')
integrationTestPython.configure {
dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble')
}
// Not really sure what this task does differently from customIntegrationTestPython,
// but it seems to also run integration tests and as such it depends on the docker images.
customIntegrationTestsCoverage.configure {
integrationTestPython.configure {
dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble')
}
}
// DATs have some additional tests that exercise normalization code paths,
// so we want to run these in addition to the base-normalization integration tests.
// If you add more items here, make sure to also to have CI fetch their credentials.
// See git history for an example.
// TODO reenable these - they're causing flakiness in our test results, need to figure that out
// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest")
// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest")
// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest")
tasks.named('customIntegrationTests').configure {
dependsOn customIntegrationTestPython
}

2
airbyte-integrations/bases/base/build.gradle
View File

@@ -1,3 +1,3 @@
plugins {
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}

2
airbyte-integrations/bases/connector-acceptance-test/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
// airbyte-docker is kept to support /legacy-publish until airbyte-ci cat publish command exists.
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}

2
airbyte-integrations/connector-templates/destination-java/README.md.hbs
View File

@@ -50,7 +50,7 @@ Airbyte has a standard test suite that all destination connectors must pass. Imp
All commands should be run from airbyte project root.
To run unit tests:
```
./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:unitTest
./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:check
```
To run acceptance and custom integration tests:
```

8
airbyte-integrations/connector-templates/destination-python/build.gradle
View File

@@ -1,8 +0,0 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
}
airbytePython {
moduleDirectory 'destination_{{snakeCase name}}'
}

2
airbyte-integrations/connector-templates/source-configuration-based/README.md.hbs
View File

@@ -60,7 +60,7 @@ To run your integration tests with Docker, run:
All commands should be run from airbyte project root.
To run unit tests:
```
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:check
```
To run acceptance and custom integration tests:
```

8
airbyte-integrations/connector-templates/source-configuration-based/build.gradle.hbs
View File

@@ -1,8 +0,0 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
}
airbytePython {
moduleDirectory 'source_{{snakeCase name}}'
}

5
airbyte-integrations/connector-templates/source-generic/build.gradle
View File

@@ -1,5 +0,0 @@
plugins {
// Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image
// via ./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build
id 'airbyte-docker'
}

2
airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs
View File

@@ -111,7 +111,7 @@ To run your integration tests with docker
All commands should be run from airbyte project root.
To run unit tests:
```
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:check
```
To run acceptance and custom integration tests:
```

8
airbyte-integrations/connector-templates/source-python-http-api/build.gradle.hbs
View File

@@ -1,8 +0,0 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
}
airbytePython {
moduleDirectory 'source_{{snakeCase name}}'
}

2
airbyte-integrations/connector-templates/source-python/README.md.hbs
View File

@@ -107,7 +107,7 @@ To run your integration tests with docker
All commands should be run from airbyte project root.
To run unit tests:
```
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:check
```
To run acceptance and custom integration tests:
```

8
airbyte-integrations/connector-templates/source-python/build.gradle.hbs
View File

@@ -1,8 +0,0 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
}
airbytePython {
moduleDirectory 'source_{{snakeCase name}}_singer'
}

2
airbyte-integrations/connector-templates/source-singer/README.md.hbs
View File

@@ -107,7 +107,7 @@ To run your integration tests with docker
All commands should be run from airbyte project root.
To run unit tests:
```
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}-singer:unitTest
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}-singer:check
```
To run acceptance and custom integration tests:
```

8
airbyte-integrations/connector-templates/source-singer/build.gradle.hbs
View File

@@ -1,8 +0,0 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
}
airbytePython {
moduleDirectory 'source_{{snakeCase name}}_singer'
}

2
airbyte-integrations/connectors-performance/destination-harness/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'application'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}
application {

2
airbyte-integrations/connectors-performance/source-harness/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'application'
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}
application {

28
airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-azure-blob-storage
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-azure-blob-storage
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.1
LABEL io.airbyte.name=airbyte/destination-azure-blob-storage

3
airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle
View File

@@ -1,3 +0,0 @@
plugins {
}

28
airbyte-integrations/connectors/destination-cassandra/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-cassandra
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-cassandra
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.name=airbyte/destination-cassandra

51
airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile
View File

@@ -1,51 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
pip3 install dbt-clickhouse>=1.4.0
# Luckily, none of normalization's files conflict with destination-clickhouse's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-clickhouse-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-clickhouse-strict-encrypt
ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.5
LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

54
airbyte-integrations/connectors/destination-clickhouse/Dockerfile
View File

@@ -1,54 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
# these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601
pip3 install wheel && \
pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \
pip3 install dbt-clickhouse>=1.4.0
# Luckily, none of normalization's files conflict with destination-clickhouse's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-clickhouse
ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-clickhouse
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.5
LABEL io.airbyte.name=airbyte/destination-clickhouse
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

28
airbyte-integrations/connectors/destination-csv/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-csv
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-csv
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=1.0.0
LABEL io.airbyte.name=airbyte/destination-csv

28
airbyte-integrations/connectors/destination-databricks/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-databricks
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-databricks
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=1.1.0
LABEL io.airbyte.name=airbyte/destination-databricks

28
airbyte-integrations/connectors/destination-dev-null/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-dev-null
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-dev-null
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.3.0
LABEL io.airbyte.name=airbyte/destination-dev-null

26
airbyte-integrations/connectors/destination-doris/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-doris
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-doris
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/destination-doris

2
airbyte-integrations/connectors/destination-duckdb/build.gradle
View File

@@ -1,3 +1,3 @@
plugins {
id 'airbyte-docker'
id 'airbyte-docker-legacy'
}

28
airbyte-integrations/connectors/destination-dynamodb/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-dynamodb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-dynamodb
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.7
LABEL io.airbyte.name=airbyte/destination-dynamodb

28
airbyte-integrations/connectors/destination-e2e-test/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-e2e-test
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-e2e-test
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.3.0
LABEL io.airbyte.name=airbyte/destination-e2e-test

28
airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-elasticsearch-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-elasticsearch-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.6
LABEL io.airbyte.name=airbyte/destination-elasticsearch-strict-encrypt

28
airbyte-integrations/connectors/destination-elasticsearch/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-elasticsearch
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-elasticsearch
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.6
LABEL io.airbyte.name=airbyte/destination-elasticsearch

26
airbyte-integrations/connectors/destination-exasol/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-exasol
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-exasol
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.1
LABEL io.airbyte.name=airbyte/destination-exasol

28
airbyte-integrations/connectors/destination-gcs/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-gcs
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-gcs
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.4.4
LABEL io.airbyte.name=airbyte/destination-gcs

33
airbyte-integrations/connectors/destination-iceberg/Dockerfile
View File

@@ -1,33 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-iceberg
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-iceberg
ENV JAVA_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED \
--add-opens java.base/java.util=ALL-UNNAMED \
--add-opens java.base/java.lang.reflect=ALL-UNNAMED \
--add-opens java.base/java.text=ALL-UNNAMED \
--add-opens java.base/sun.nio.ch=ALL-UNNAMED \
--add-opens java.base/java.nio=ALL-UNNAMED "
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.name=airbyte/destination-iceberg

28
airbyte-integrations/connectors/destination-kafka/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-kafka
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-kafka
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.10
LABEL io.airbyte.name=airbyte/destination-kafka

28
airbyte-integrations/connectors/destination-keen/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-keen
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-keen
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.4
LABEL io.airbyte.name=airbyte/destination-keen

28
airbyte-integrations/connectors/destination-kinesis/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-kinesis
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-kinesis
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.5
LABEL io.airbyte.name=airbyte/destination-kinesis

4
airbyte-integrations/connectors/destination-kvdb/build.gradle
View File

@@ -1,4 +0,0 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
}

28
airbyte-integrations/connectors/destination-local-json/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-local-json
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-local-json
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.11
LABEL io.airbyte.name=airbyte/destination-local-json

28
airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-mariadb-columnstore
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mariadb-columnstore
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.7
LABEL io.airbyte.name=airbyte/destination-mariadb-columnstore

28
airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-mongodb-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mongodb-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt

28
airbyte-integrations/connectors/destination-mongodb/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-mongodb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mongodb
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-mongodb

28
airbyte-integrations/connectors/destination-mqtt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-mqtt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mqtt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.3
LABEL io.airbyte.name=airbyte/destination-mqtt

51
airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile
View File

@@ -1,51 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
pip3 install dbt-sqlserver==1.0.0
# Luckily, none of normalization's files conflict with destination-mssql's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-mssql-strict-encrypt
ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mssql-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

54
airbyte-integrations/connectors/destination-mssql/Dockerfile
View File

@@ -1,54 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
# these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601
pip3 install wheel && \
pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \
pip3 install dbt-sqlserver==1.0.0
# Luckily, none of normalization's files conflict with destination-mssql's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-mssql
ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mssql
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-mssql
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

28
airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-mysql-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mysql-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt

28
airbyte-integrations/connectors/destination-mysql/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-mysql
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-mysql
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-mysql

28
airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-oracle-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-oracle-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt

28
airbyte-integrations/connectors/destination-oracle/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-oracle
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-oracle
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-oracle

28
airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-postgres-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-postgres-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.4.0
LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt

28
airbyte-integrations/connectors/destination-postgres/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-postgres
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-postgres
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.4.0
LABEL io.airbyte.name=airbyte/destination-postgres

28
airbyte-integrations/connectors/destination-pubsub/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-pubsub
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-pubsub
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.2.0
LABEL io.airbyte.name=airbyte/destination-pubsub

28
airbyte-integrations/connectors/destination-pulsar/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-pulsar
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-pulsar
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.3
LABEL io.airbyte.name=airbyte/destination-pulsar

26
airbyte-integrations/connectors/destination-r2/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-r2
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-r2
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/destination-r2

28
airbyte-integrations/connectors/destination-redis/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-redis
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-redis
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.name=airbyte/destination-redis

26
airbyte-integrations/connectors/destination-redpanda/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-redpanda
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-redpanda
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/destination-redpanda

53
airbyte-integrations/connectors/destination-redshift/Dockerfile
View File

@@ -1,53 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
# these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601
pip3 install wheel && \
pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \
pip3 install dbt-redshift==1.0.0 "urllib3<2"
# Luckily, none of normalization's files conflict with destination-bigquery's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-redshift:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-redshift
ENV AIRBYTE_NORMALIZATION_INTEGRATION redshift
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-redshift
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.6.5
LABEL io.airbyte.name=airbyte/destination-redshift
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

28
airbyte-integrations/connectors/destination-rockset/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-rockset
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-rockset
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.name=airbyte/destination-rockset

26
airbyte-integrations/connectors/destination-s3-glue/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-s3-glue
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-s3-glue
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.7
LABEL io.airbyte.name=airbyte/destination-s3-glue

35
airbyte-integrations/connectors/destination-s3/Dockerfile
View File

@@ -1,35 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-s3
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-s3
COPY --from=build /airbyte /airbyte
RUN /bin/bash -c 'set -e && \
ARCH=`uname -m` && \
if [ "$ARCH" == "x86_64" ] || [ "$ARCH" = "amd64" ]; then \
echo "$ARCH" && \
yum install lzop lzo lzo-dev -y; \
fi'
RUN yum clean all
LABEL io.airbyte.version=0.5.1
LABEL io.airbyte.name=airbyte/destination-s3

28
airbyte-integrations/connectors/destination-scylla/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-scylla
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-scylla
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.3
LABEL io.airbyte.name=airbyte/destination-scylla

26
airbyte-integrations/connectors/destination-selectdb/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-selectdb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-selectdb
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/destination-selectdb

26
airbyte-integrations/connectors/destination-starburst-galaxy/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-starburst-galaxy
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-starburst-galaxy
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.0.1
LABEL io.airbyte.name=airbyte/destination-starburst-galaxy

52
airbyte-integrations/connectors/destination-tidb/Dockerfile
View File

@@ -1,52 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
# these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601
pip3 install wheel && \
pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \
pip3 install dbt-tidb==1.0.1
# Luckily, none of normalization's files conflict with destination-tidb's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-tidb:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps
WORKDIR /airbyte
ENV APPLICATION destination-tidb
ENV AIRBYTE_NORMALIZATION_INTEGRATION tidb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-tidb
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.name=airbyte/destination-tidb
ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]

26
airbyte-integrations/connectors/destination-yugabytedb/Dockerfile
View File

@@ -1,26 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION destination-yugabytedb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION destination-yugabytedb
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.1
LABEL io.airbyte.name=airbyte/destination-yugabytedb

28
airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-alloydb-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-alloydb-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=3.1.5
LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt

28
airbyte-integrations/connectors/source-alloydb/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-alloydb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-alloydb
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=3.1.8
LABEL io.airbyte.name=airbyte/source-alloydb

29
airbyte-integrations/connectors/source-azure-blob-storage/Dockerfile
View File

@@ -1,29 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-azure-blob-storage
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-azure-blob-storage
COPY --from=build /airbyte /airbyte
# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/source-azure-blob-storage

29
airbyte-integrations/connectors/source-bigquery/Dockerfile
View File

@@ -1,29 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-bigquery
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-bigquery
COPY --from=build /airbyte /airbyte
# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.
LABEL io.airbyte.version=0.3.0
LABEL io.airbyte.name=airbyte/source-bigquery

28
airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-clickhouse-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-clickhouse-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.17
LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt

28
airbyte-integrations/connectors/source-clickhouse/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-clickhouse
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-clickhouse
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.17
LABEL io.airbyte.name=airbyte/source-clickhouse

28
airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-cockroachdb-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-cockroachdb-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.22
LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt

28
airbyte-integrations/connectors/source-cockroachdb/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-cockroachdb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-cockroachdb
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.22
LABEL io.airbyte.name=airbyte/source-cockroachdb

28
airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-db2-strict-encrypt
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-db2-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.19
LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt

1
airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle
View File

@@ -1,6 +1,5 @@
plugins {
id 'application'
id 'airbyte-docker'
id 'airbyte-java-connector'
}

28
airbyte-integrations/connectors/source-db2/Dockerfile
View File

@@ -1,28 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-db2
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-db2
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=0.1.20
LABEL io.airbyte.name=airbyte/source-db2

29
airbyte-integrations/connectors/source-dynamodb/Dockerfile
View File

@@ -1,29 +0,0 @@
### WARNING ###
# The Java connector Dockerfiles will soon be deprecated.
# This Dockerfile is not used to build the connector image we publish to DockerHub.
# The new logic to build the connector image is declared with Dagger here:
# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649
# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder.
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build
WORKDIR /airbyte
ENV APPLICATION source-dynamodb
COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
FROM airbyte/integration-base-java:dev
WORKDIR /airbyte
ENV APPLICATION source-dynamodb
COPY --from=build /airbyte /airbyte
# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.
LABEL io.airbyte.version=0.1.2
LABEL io.airbyte.name=airbyte/source-dynamodb

Some files were not shown because too many files have changed in this diff Show More