From 8f7abc2cc04d4e54c2a9fd3d61bf473c8be786a2 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 30 Nov 2023 09:48:18 +0100 Subject: [PATCH] S3, Azure Blob Storage, GCS, Weviate, Milvus, Chroma, Qdrant: Bump cdk version (#32608) Co-authored-by: flash1293 --- airbyte-integrations/connectors/destination-chroma/Dockerfile | 2 +- .../connectors/destination-chroma/metadata.yaml | 2 +- airbyte-integrations/connectors/destination-chroma/setup.py | 2 +- .../connectors/destination-milvus/metadata.yaml | 2 +- airbyte-integrations/connectors/destination-milvus/setup.py | 2 +- airbyte-integrations/connectors/destination-qdrant/Dockerfile | 2 +- .../connectors/destination-qdrant/destination_qdrant/config.py | 2 +- .../connectors/destination-qdrant/metadata.yaml | 2 +- airbyte-integrations/connectors/destination-qdrant/setup.py | 2 +- .../connectors/destination-weaviate/metadata.yaml | 2 +- airbyte-integrations/connectors/destination-weaviate/setup.py | 2 +- .../connectors/source-azure-blob-storage/metadata.yaml | 2 +- .../connectors/source-azure-blob-storage/setup.py | 2 +- airbyte-integrations/connectors/source-gcs/metadata.yaml | 2 +- airbyte-integrations/connectors/source-gcs/setup.py | 2 +- airbyte-integrations/connectors/source-s3/metadata.yaml | 2 +- airbyte-integrations/connectors/source-s3/setup.py | 2 +- docs/integrations/destinations/chroma.md | 1 + docs/integrations/destinations/milvus.md | 3 ++- docs/integrations/destinations/qdrant.md | 1 + docs/integrations/destinations/weaviate.md | 3 ++- docs/integrations/sources/azure-blob-storage.md | 1 + docs/integrations/sources/gcs.md | 1 + docs/integrations/sources/s3.md | 1 + 24 files changed, 26 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/connectors/destination-chroma/Dockerfile b/airbyte-integrations/connectors/destination-chroma/Dockerfile index da9d91f9af9..243f07e4571 100644 --- a/airbyte-integrations/connectors/destination-chroma/Dockerfile +++ b/airbyte-integrations/connectors/destination-chroma/Dockerfile @@ -41,5 +41,5 @@ COPY destination_chroma ./destination_chroma ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.0.6 +LABEL io.airbyte.version=0.0.7 LABEL io.airbyte.name=airbyte/destination-chroma diff --git a/airbyte-integrations/connectors/destination-chroma/metadata.yaml b/airbyte-integrations/connectors/destination-chroma/metadata.yaml index 3e57dfca3ab..0d1ce0b7d0f 100644 --- a/airbyte-integrations/connectors/destination-chroma/metadata.yaml +++ b/airbyte-integrations/connectors/destination-chroma/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 0b75218b-f702-4a28-85ac-34d3d84c0fc2 - dockerImageTag: 0.0.6 + dockerImageTag: 0.0.7 dockerRepository: airbyte/destination-chroma githubIssueLabel: destination-chroma icon: chroma.svg diff --git a/airbyte-integrations/connectors/destination-chroma/setup.py b/airbyte-integrations/connectors/destination-chroma/setup.py index 07c0cffe1f3..cba67dc9de0 100644 --- a/airbyte-integrations/connectors/destination-chroma/setup.py +++ b/airbyte-integrations/connectors/destination-chroma/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[vector-db-based]==0.53.3", + "airbyte-cdk[vector-db-based]==0.53.8", "chromadb", ] diff --git a/airbyte-integrations/connectors/destination-milvus/metadata.yaml b/airbyte-integrations/connectors/destination-milvus/metadata.yaml index bf7b68027e9..6d3064094d6 100644 --- a/airbyte-integrations/connectors/destination-milvus/metadata.yaml +++ b/airbyte-integrations/connectors/destination-milvus/metadata.yaml @@ -22,7 +22,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 65de8962-48c9-11ee-be56-0242ac120002 - dockerImageTag: 0.0.9 + dockerImageTag: 0.0.10 dockerRepository: airbyte/destination-milvus githubIssueLabel: destination-milvus icon: milvus.svg diff --git a/airbyte-integrations/connectors/destination-milvus/setup.py b/airbyte-integrations/connectors/destination-milvus/setup.py index fdefdea8303..51c4d2cdd13 100644 --- a/airbyte-integrations/connectors/destination-milvus/setup.py +++ b/airbyte-integrations/connectors/destination-milvus/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.3", "pymilvus==2.3.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.8", "pymilvus==2.3.0"] TEST_REQUIREMENTS = ["pytest~=6.2"] diff --git a/airbyte-integrations/connectors/destination-qdrant/Dockerfile b/airbyte-integrations/connectors/destination-qdrant/Dockerfile index ff82da252da..132b09db701 100644 --- a/airbyte-integrations/connectors/destination-qdrant/Dockerfile +++ b/airbyte-integrations/connectors/destination-qdrant/Dockerfile @@ -41,5 +41,5 @@ COPY destination_qdrant ./destination_qdrant ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.0.7 +LABEL io.airbyte.version=0.0.8 LABEL io.airbyte.name=airbyte/destination-qdrant diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py index 1f79cffa756..1da1e0dee36 100644 --- a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py @@ -44,7 +44,7 @@ class QdrantIndexingConfigModel(BaseModel): title="Prefer gRPC", description="Whether to prefer gRPC over HTTP. Set to true for Qdrant cloud clusters", default=True ) collection: str = Field(..., title="Collection Name", description="The collection to load data into", order=2) - distance_metric: Union[Literal["dot"], Literal["cos"], Literal["euc"]] = Field( + distance_metric: str = Field( default="cos", title="Distance Metric", enum=["dot", "cos", "euc"], diff --git a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml index 29d3047f24c..501ed9a349b 100644 --- a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml +++ b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml @@ -20,7 +20,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 6eb1198a-6d38-43e5-aaaa-dccd8f71db2b - dockerImageTag: 0.0.7 + dockerImageTag: 0.0.8 dockerRepository: airbyte/destination-qdrant githubIssueLabel: destination-qdrant icon: qdrant.svg diff --git a/airbyte-integrations/connectors/destination-qdrant/setup.py b/airbyte-integrations/connectors/destination-qdrant/setup.py index c5b16e110ee..72c0d4ba4d1 100644 --- a/airbyte-integrations/connectors/destination-qdrant/setup.py +++ b/airbyte-integrations/connectors/destination-qdrant/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.3", "qdrant-client", "fastembed"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.9", "qdrant-client", "fastembed"] TEST_REQUIREMENTS = ["pytest~=6.2"] diff --git a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml index a885f0d025d..36babd07992 100644 --- a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml +++ b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 7b7d7a0d-954c-45a0-bcfc-39a634b97736 - dockerImageTag: 0.2.9 + dockerImageTag: 0.2.10 dockerRepository: airbyte/destination-weaviate documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate githubIssueLabel: destination-weaviate diff --git a/airbyte-integrations/connectors/destination-weaviate/setup.py b/airbyte-integrations/connectors/destination-weaviate/setup.py index fffa30e1200..5c15352cfd4 100644 --- a/airbyte-integrations/connectors/destination-weaviate/setup.py +++ b/airbyte-integrations/connectors/destination-weaviate/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.3", "weaviate-client==3.25.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.8", "weaviate-client==3.25.2"] TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker"] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index fb96d5cf2fa..5a3a0487ed4 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 dockerRepository: airbyte/source-azure-blob-storage documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage githubIssueLabel: source-azure-blob-storage diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py index efb01f6a4af..52176bff869 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.53.5", + "airbyte-cdk[file-based]>=0.53.8", "smart_open[azure]", "pytz", ] diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index c531579684c..d81dec19ac5 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.2 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index 70d0679e6bd..f18be61d098 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.53.5", + "airbyte-cdk[file-based]>=0.53.8", "google-cloud-storage==2.12.0", "smart-open[s3]==5.1.0", "pandas==1.5.3", diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index 4c99fb3cc67..7b8cdbdee77 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.2.2 + dockerImageTag: 4.2.3 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py index a0bfb240328..7d7304efb56 100644 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ b/airbyte-integrations/connectors/source-s3/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.53.5", + "airbyte-cdk[file-based]>=0.53.8", "smart-open[s3]==5.1.0", "wcmatch==8.4", "dill==0.3.4", diff --git a/docs/integrations/destinations/chroma.md b/docs/integrations/destinations/chroma.md index f99a9cf869a..02997e73a29 100644 --- a/docs/integrations/destinations/chroma.md +++ b/docs/integrations/destinations/chroma.md @@ -76,6 +76,7 @@ You should now have all the requirements needed to configure Chroma as a destina | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.7 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | | 0.0.6 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.5 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | | 0.0.4 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | diff --git a/docs/integrations/destinations/milvus.md b/docs/integrations/destinations/milvus.md index 2e7c225a0b8..724f2d51854 100644 --- a/docs/integrations/destinations/milvus.md +++ b/docs/integrations/destinations/milvus.md @@ -29,7 +29,7 @@ You'll need the following information to configure the destination: | :----------------------------- | :------------------- | :---- | | Full Refresh Sync | Yes | | | Incremental - Append Sync | Yes | | -| Incremental - Append + Deduped | Yes | Deleting records via CDC is not supported (see issue [#29827](https://github.com/airbytehq/airbyte/issues/29827)) | +| Incremental - Append + Deduped | Yes | | | Partitions | No | | | Record-defined ID | No | Auto-id needs to be enabled | @@ -109,6 +109,7 @@ vector_store.similarity_search("test") | Version | Date | Pull Request | Subject | |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.0.10 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | | 0.0.9 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.8 | 2023-11-08 | [#31563](https://github.com/airbytehq/airbyte/pull/32262) | Auto-create collection if it doesn't exist | | 0.0.7 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | diff --git a/docs/integrations/destinations/qdrant.md b/docs/integrations/destinations/qdrant.md index 648d0b01928..7a1892137fc 100644 --- a/docs/integrations/destinations/qdrant.md +++ b/docs/integrations/destinations/qdrant.md @@ -71,6 +71,7 @@ You should now have all the requirements needed to configure Qdrant as a destina | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.8 | 2023-11-29 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources and fix spec schema | | 0.0.7 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.6 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | | 0.0.5 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | diff --git a/docs/integrations/destinations/weaviate.md b/docs/integrations/destinations/weaviate.md index 9e36ca07fe5..18d7da6b155 100644 --- a/docs/integrations/destinations/weaviate.md +++ b/docs/integrations/destinations/weaviate.md @@ -30,7 +30,7 @@ You'll need the following information to configure the destination: | :----------------------------- | :------------------- | :---- | | Full Refresh Sync | Yes | | | Incremental - Append Sync | Yes | | -| Incremental - Append + Deduped | Yes | Deleting records via CDC is not supported (see issue [#29827](https://github.com/airbytehq/airbyte/issues/29827)) | +| Incremental - Append + Deduped | Yes | | | Namespaces | No | | | Provide vector | Yes | Either from field are calculated during the load process | @@ -83,6 +83,7 @@ As properties have to start will a lowercase letter in Weaviate, field names mig | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------- | +| 0.2.10 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | | 0.2.9 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.2.8 | 2023-11-03 | [#32134](https://github.com/airbytehq/airbyte/pull/32134) | Improve test coverage | | 0.2.7 | 2023-11-03 | [#32134](https://github.com/airbytehq/airbyte/pull/32134) | Upgrade weaviate client library | diff --git a/docs/integrations/sources/azure-blob-storage.md b/docs/integrations/sources/azure-blob-storage.md index 02942fa6ee0..cd3a0ed0720 100644 --- a/docs/integrations/sources/azure-blob-storage.md +++ b/docs/integrations/sources/azure-blob-storage.md @@ -193,6 +193,7 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------|:------------------------------------------------------------------------| +| 0.2.4 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | | 0.2.3 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.2.2 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK to support document file types | | 0.2.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index 0fd9e627d4e..daff7cf1682 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -37,6 +37,7 @@ Use the service account ID from above, grant read access to your target bucket. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------| +| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | | 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | | 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index a0e2e8ec9d7..d3fbc938b8b 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -256,6 +256,7 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.2.3 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | | 4.2.2 | 2023-11-20 | [32677](https://github.com/airbytehq/airbyte/pull/32677) | Only read files with ".zip" extension as zipped files | | 4.2.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 4.2.0 | 2023-11-02 | [32109](https://github.com/airbytehq/airbyte/pull/32109) | Fix docs; add HTTPS validation for S3 endpoint; fix coverage |