From f1ddfca1c57c8df609a4b9ef4786649397b40880 Mon Sep 17 00:00:00 2001 From: "oleh.zorenko" <19872253+Zirochkaa@users.noreply.github.com> Date: Mon, 18 Oct 2021 07:05:07 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Twilio:=20update=20sche?= =?UTF-8?q?mas=20+=20turn=20on=20transformer=20for=20all=20streams=20(#703?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update schemas + turn on transformer for all streams * Update docs * Transform RFC2822 datetime into ISO8601 * Bump connector's version + update `accounts` schema --- .../b9dc6155-672e-42ea-b10d-9f1f1fb95ab1.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-twilio/Dockerfile | 2 +- .../integration_tests/configured_catalog.json | 77 ++----------------- .../no_empty_streams_catalog.json | 12 --- .../source_twilio/schemas/addresses.json | 3 + .../source_twilio/schemas/applications.json | 2 +- .../source_twilio/schemas/calls.json | 16 +++- .../schemas/conference_participants.json | 3 + .../source_twilio/schemas/conferences.json | 21 +++-- .../schemas/dependent_phone_numbers.json | 9 +-- .../schemas/incoming_phone_numbers.json | 3 + .../source_twilio/schemas/keys.json | 3 - .../source_twilio/schemas/messages.json | 4 +- .../source_twilio/schemas/queues.json | 13 ++-- .../source_twilio/schemas/recordings.json | 3 +- .../source_twilio/schemas/transcriptions.json | 1 - .../source_twilio/schemas/usage_records.json | 13 ++-- .../source_twilio/schemas/usage_triggers.json | 3 +- .../source-twilio/source_twilio/streams.py | 16 ++++ docs/integrations/sources/twilio.md | 1 + 21 files changed, 82 insertions(+), 127 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b9dc6155-672e-42ea-b10d-9f1f1fb95ab1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b9dc6155-672e-42ea-b10d-9f1f1fb95ab1.json index fc99b37ad3c..66f4337830c 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b9dc6155-672e-42ea-b10d-9f1f1fb95ab1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b9dc6155-672e-42ea-b10d-9f1f1fb95ab1.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "b9dc6155-672e-42ea-b10d-9f1f1fb95ab1", "name": "Twilio", "dockerRepository": "airbyte/source-twilio", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/twilio" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 26136169193..963d291694c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -187,7 +187,7 @@ - sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 name: Twilio dockerRepository: airbyte/source-twilio - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/twilio sourceType: api - sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile index 6dd447b3a7a..3ddbd6516d7 100644 --- a/airbyte-integrations/connectors/source-twilio/Dockerfile +++ b/airbyte-integrations/connectors/source-twilio/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-twilio/integration_tests/configured_catalog.json index 163dbe1dc9e..6461c7b23cd 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/configured_catalog.json @@ -93,16 +93,7 @@ { "stream": { "name": "calls", - "json_schema": { - "properties": { - "end_time": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["end_time"] @@ -114,16 +105,7 @@ { "stream": { "name": "conferences", - "json_schema": { - "properties": { - "date_updated": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_updated"] @@ -153,16 +135,7 @@ { "stream": { "name": "recordings", - "json_schema": { - "properties": { - "date_created": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_created"] @@ -192,16 +165,7 @@ { "stream": { "name": "messages", - "json_schema": { - "properties": { - "date_sent": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_sent"] @@ -213,16 +177,7 @@ { "stream": { "name": "message_media", - "json_schema": { - "properties": { - "date_created": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_created"] @@ -234,16 +189,7 @@ { "stream": { "name": "usage_records", - "json_schema": { - "properties": { - "start_date": { - "format": "date", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["start_date"] @@ -264,16 +210,7 @@ { "stream": { "name": "alerts", - "json_schema": { - "properties": { - "date_updated": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_updated"] diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json b/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json index 30a6e6e4ba6..1de59155ead 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json @@ -188,18 +188,6 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "alerts", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": ["date_updated"] - }, - "cursor_field": ["date_updated"], - "sync_mode": "incremental", - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json index ebe4448c79f..0671f851378 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json @@ -38,6 +38,9 @@ "street": { "type": ["null", "string"] }, + "street_secondary": { + "type": ["null", "string"] + }, "validated": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json index fa8c961ed3b..56fe7b0dcac 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json @@ -47,7 +47,7 @@ "uri": { "type": ["null", "string"] }, - "voice_callback_id_lookup": { + "voice_caller_id_lookup": { "type": ["null", "boolean"] }, "voice_fallback_method": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json index 2b1d19d4f5d..3daf6f426b7 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json @@ -47,7 +47,6 @@ "type": ["null", "integer"] }, "price": { - "multipleOf": 1e-8, "type": ["null", "number"] }, "price_unit": { @@ -92,6 +91,21 @@ }, "notifications": { "type": ["null", "string"] + }, + "feedback": { + "type": ["null", "string"] + }, + "payments": { + "type": ["null", "string"] + }, + "siprec": { + "type": ["null", "string"] + }, + "events": { + "type": ["null", "string"] + }, + "feedback_summaries": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json index 0c184d9bb45..23f5de105e8 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json @@ -6,6 +6,9 @@ "call_sid": { "type": ["null", "string"] }, + "label": { + "type": ["null", "string"] + }, "conference_sid": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json index b3d86fb5f66..4d74793e4e0 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json @@ -8,7 +8,7 @@ "type": ["null", "string"] }, "date_updated": { - "format": "date-time", + "format": "date", "type": ["null", "string"] }, "api_version": { @@ -36,17 +36,14 @@ "type": ["null", "string"] }, "subresource_uris": { - "type": ["array"], - "items": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "participants": { - "type": ["null", "string"] - }, - "recordings": { - "type": ["null", "string"] - } + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "participants": { + "type": ["null", "string"] + }, + "recordings": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json index cac8e1e55fc..58413bbccf6 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json @@ -52,16 +52,13 @@ }, "capabilities": { "properties": { - "voice": { + "Voice": { "type": ["null", "boolean"] }, - "sms": { + "SMS": { "type": ["null", "boolean"] }, - "mms": { - "type": ["null", "boolean"] - }, - "fax": { + "MMS": { "type": ["null", "boolean"] } }, diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json index b990d2602d6..b493a73f0a8 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json @@ -101,6 +101,9 @@ "emergency_address_sid": { "type": ["null", "string"] }, + "emergency_address_status": { + "type": ["null", "string"] + }, "address_sid": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json index b6a13248d82..7f30792f83f 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json @@ -1,8 +1,5 @@ { "properties": { - "account_sid": { - "type": ["null", "string"] - }, "date_created": { "format": "date-time", "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json index 9d9996a7d17..24e0952f49a 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json @@ -36,6 +36,9 @@ "messaging_service_sid": { "type": ["null", "string"] }, + "price_unit": { + "type": ["null", "string"] + }, "num_media": { "type": ["null", "integer"] }, @@ -43,7 +46,6 @@ "type": ["null", "integer"] }, "price": { - "multipleOf": 1e-8, "type": ["null", "number"] }, "sid": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json index 597a129cfb6..594aacf683d 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json @@ -30,14 +30,11 @@ "type": ["null", "string"] }, "subresource_uris": { - "type": "array", - "items": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "members": { - "type": ["null", "string"] - } + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "members": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json index 48be8a7250b..2949880e2d4 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json @@ -31,7 +31,6 @@ "type": ["null", "string"] }, "price": { - "multipleOf": 1e-8, "type": ["null", "number"] }, "price_unit": { @@ -47,7 +46,7 @@ "type": ["null", "string"] }, "error_code": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "uri": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json index 0d184273a29..ca6293568c2 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json @@ -18,7 +18,6 @@ "type": ["null", "integer"] }, "price": { - "multipleOf": 1e-8, "type": ["null", "number"] }, "price_unit": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json index c541009eca7..8551b8ad3d5 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json @@ -4,7 +4,7 @@ "type": ["null", "string"] }, "count": { - "type": ["null", "string", "integer"] + "type": ["null", "integer"] }, "price_unit": { "type": ["null", "string"] @@ -43,14 +43,14 @@ "type": ["null", "string"] }, "end_date": { - "format": "date-time", + "format": "date", "type": ["null", "string"] }, "usage_unit": { "type": ["null", "string"] }, "price": { - "type": ["null", "number", "string"] + "type": ["null", "number"] }, "uri": { "type": ["null", "string"] @@ -59,15 +59,18 @@ "type": ["null", "string"] }, "usage": { - "type": ["null", "number", "string"] + "type": ["null", "number"] }, "start_date": { - "format": "date-time", + "format": "date", "type": ["null", "string"] }, "count_unit": { "type": ["null", "string"] }, + "as_of": { + "type": ["null", "string"] + }, "api_version": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json index ac9a7ef0269..658fa01a241 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json @@ -30,7 +30,7 @@ "type": ["null", "string"] }, "current_value": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "date_created": { "format": "date-time", @@ -46,7 +46,6 @@ "type": ["null", "string"] }, "trigger_value": { - "multipleOf": 1e-8, "type": ["null", "number"] }, "api_version": { diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 5ab4ae9dde5..528e8ddbb0b 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -10,6 +10,7 @@ import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer TWILIO_API_URL_BASE = "https://api.twilio.com" TWILIO_API_URL_BASE_VERSIONED = f"{TWILIO_API_URL_BASE}/2010-04-01/" @@ -20,6 +21,7 @@ class TwilioStream(HttpStream, ABC): url_base = TWILIO_API_URL_BASE primary_key = "sid" page_size = 100 + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) @property def data_field(self): @@ -64,6 +66,20 @@ class TwilioStream(HttpStream, ABC): params.update(**next_page_token) return params + @transformer.registerCustomTransform + def custom_transform_function(original_value: Any, field_schema: Mapping[str, Any]) -> Any: + if original_value and "format" in field_schema and field_schema["format"] == "date-time": + try: + return pendulum.from_format(original_value, "ddd, D MMM YYYY HH:mm:ss ZZ").in_timezone('UTC').to_iso8601_string() + except ValueError: + # Twilio API returns datetime in two formats: + # - RFC2822, like "Fri, 11 Dec 2020 04:28:40 +0000"; + # - ISO8601, like "2020-12-11T04:29:09Z". + # If `ValueError` exception was raised this means that datetime was already in ISO8601 format and there + # is no need in transforming anything. + pass + return original_value + class IncrementalTwilioStream(TwilioStream, ABC): cursor_field = "date_updated" diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index 6c98ba0b914..616f78ae72a 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -66,5 +66,6 @@ See [docs](https://www.twilio.com/docs/iam/api) for more details. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.1 | 2021-10-18 | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas and transform data types according to the API schema | | 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented |