✨Source Datascope: Make Connector Compatible with Builder (#38844)

airbyte-integrations/connectors/source-datascope/metadata.yaml

@@ -2,7 +2,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 8e1ae2d2-4790-44d3-9d83-75b3fc3940ff
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   dockerRepository: airbyte/source-datascope
   githubIssueLabel: source-datascope
   icon: datascope.svg

airbyte-integrations/connectors/source-datascope/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.1.2"
+version = "0.1.3"
 name = "source-datascope"
 description = "Source implementation for Datascope."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte-integrations/connectors/source-datascope/requirements.txt

@@ -1 +0,0 @@
--e .

airbyte-integrations/connectors/source-datascope/source_datascope/manifest.yaml

@@ -1,92 +1,379 @@
-version: "0.29.0"
+version: 0.79.1
 
-definitions:
-  selector:
-    extractor:
-      field_path: []
-  requester:
-    url_base: "https://www.mydatascope.com/api/external/"
-    http_method: "GET"
-    authenticator:
-      type: ApiKeyAuthenticator
-      header: "Authorization"
-      api_token: "{{ config['api_key'] }}"
-  incremental_sync:
-    type: "DatetimeBasedCursor"
-    start_datetime:
-      datetime: "{{ config['start_date'] }}"
-      datetime_format: "%d/%m/%Y %H:%M"
-    end_datetime:
-      datetime: "{{ now_utc().strftime('%d/%m/%Y %H:%M') }}"
-      datetime_format: "%d/%m/%Y %H:%M"
-    step: "P1D"
-    datetime_format: "%d/%m/%Y %H:%M"
-    # There is a discrepancy between the format of the cursor `created_at` and the granularity of the API hence "P1D"
-    # here instead of "PT1M" if we were to check the datetime_format
-    cursor_granularity: "P1D"
-    cursor_field: "created_at"
-    start_time_option:
-      field_name: "start"
-      inject_into: "request_parameter"
-    end_time_option:
-      field_name: "end"
-      inject_into: "request_parameter"
-
-  retriever:
-    record_selector:
-      $ref: "#/definitions/selector"
-    paginator:
-      type: DefaultPaginator
-      page_size_option:
-        inject_into: "request_parameter"
-        field_name: "limit"
-      pagination_strategy:
-        type: "OffsetIncrement"
-        page_size: 200
-      page_token_option:
-        type: RequestOption
-        inject_into: "request_parameter"
-        field_name: "offset"
-    requester:
-      $ref: "#/definitions/requester"
-
-  base_stream:
-    retriever:
-      $ref: "#/definitions/retriever"
-  location_stream:
-    $ref: "#/definitions/base_stream"
-    $parameters:
-      name: "locations"
-      primary_key: "id"
-      path: "/locations"
-  answers_stream:
-    $ref: "#/definitions/base_stream"
-    incremental_sync:
-      $ref: "#/definitions/incremental_sync"
-    $parameters:
-      name: "answers"
-      primary_key: "form_answer_id"
-      path: "/v2/answers"
-  lists_stream:
-    $ref: "#/definitions/base_stream"
-    $parameters:
-      name: "lists"
-      primary_key: "id"
-      path: "/metadata_objects"
-  notifications_stream:
-    $ref: "#/definitions/base_stream"
-    incremental_sync:
-      $ref: "#/definitions/incremental_sync"
-    $parameters:
-      name: "notifications"
-      primary_key: "id"
-      path: "/notifications"
-streams:
-  - "#/definitions/location_stream"
-  - "#/definitions/answers_stream"
-  - "#/definitions/lists_stream"
-  - "#/definitions/notifications_stream"
+type: DeclarativeSource
 
 check:
+  type: CheckStream
   stream_names:
-    - "locations"
+    - locations
+
+definitions:
+  streams:
+    locations:
+      type: DeclarativeStream
+      name: locations
+      primary_key:
+        - id
+      retriever:
+        type: SimpleRetriever
+        requester:
+          $ref: "#/definitions/base_requester"
+          path: /locations
+          http_method: GET
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path: []
+        paginator:
+          type: DefaultPaginator
+          page_token_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: offset
+          page_size_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: limit
+          pagination_strategy:
+            type: OffsetIncrement
+            page_size: 200
+      schema_loader:
+        type: InlineSchemaLoader
+        schema:
+          $ref: "#/schemas/locations"
+    answers:
+      type: DeclarativeStream
+      name: answers
+      primary_key:
+        - form_answer_id
+      retriever:
+        type: SimpleRetriever
+        requester:
+          $ref: "#/definitions/base_requester"
+          path: /v2/answers
+          http_method: GET
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path: []
+        paginator:
+          type: DefaultPaginator
+          page_token_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: offset
+          page_size_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: limit
+          pagination_strategy:
+            type: OffsetIncrement
+            page_size: 200
+      incremental_sync:
+        type: DatetimeBasedCursor
+        cursor_field: created_at
+        name: answers
+        primary_key: form_answer_id
+        path: /v2/answers
+        cursor_datetime_formats:
+          - "%d/%m/%Y %H:%M"
+        datetime_format: "%d/%m/%Y %H:%M"
+        start_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config['start_date'] }}"
+          datetime_format: "%d/%m/%Y %H:%M"
+        start_time_option:
+          type: RequestOption
+          field_name: start
+          inject_into: request_parameter
+        end_time_option:
+          type: RequestOption
+          field_name: end
+          inject_into: request_parameter
+        end_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ now_utc().strftime('%d/%m/%Y %H:%M') }}"
+          datetime_format: "%d/%m/%Y %H:%M"
+        step: P1D
+        cursor_granularity: P1D
+      schema_loader:
+        type: InlineSchemaLoader
+        schema:
+          $ref: "#/schemas/answers"
+    lists:
+      type: DeclarativeStream
+      name: lists
+      primary_key:
+        - id
+      retriever:
+        type: SimpleRetriever
+        requester:
+          $ref: "#/definitions/base_requester"
+          path: /metadata_objects
+          http_method: GET
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path: []
+        paginator:
+          type: DefaultPaginator
+          page_token_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: offset
+          page_size_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: limit
+          pagination_strategy:
+            type: OffsetIncrement
+            page_size: 200
+      schema_loader:
+        type: InlineSchemaLoader
+        schema:
+          $ref: "#/schemas/lists"
+    notifications:
+      type: DeclarativeStream
+      name: notifications
+      primary_key:
+        - id
+      retriever:
+        type: SimpleRetriever
+        requester:
+          $ref: "#/definitions/base_requester"
+          path: /notifications
+          http_method: GET
+        record_selector:
+          type: RecordSelector
+          extractor:
+            type: DpathExtractor
+            field_path: []
+        paginator:
+          type: DefaultPaginator
+          page_token_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: offset
+          page_size_option:
+            type: RequestOption
+            inject_into: request_parameter
+            field_name: limit
+          pagination_strategy:
+            type: OffsetIncrement
+            page_size: 200
+      incremental_sync:
+        type: DatetimeBasedCursor
+        cursor_field: created_at
+        name: notifications
+        primary_key: id
+        path: /notifications
+        cursor_datetime_formats:
+          - "%d/%m/%Y %H:%M"
+        datetime_format: "%d/%m/%Y %H:%M"
+        start_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config['start_date'] }}"
+          datetime_format: "%d/%m/%Y %H:%M"
+        start_time_option:
+          type: RequestOption
+          field_name: start
+          inject_into: request_parameter
+        end_time_option:
+          type: RequestOption
+          field_name: end
+          inject_into: request_parameter
+        end_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ now_utc().strftime('%d/%m/%Y %H:%M') }}"
+          datetime_format: "%d/%m/%Y %H:%M"
+        step: P1D
+        cursor_granularity: P1D
+      schema_loader:
+        type: InlineSchemaLoader
+        schema:
+          $ref: "#/schemas/notifications"
+  base_requester:
+    type: HttpRequester
+    url_base: https://www.mydatascope.com/api/external/
+    authenticator:
+      type: ApiKeyAuthenticator
+      api_token: "{{ config['api_key'] }}"
+      inject_into:
+        type: RequestOption
+        field_name: Authorization
+        inject_into: header
+
+streams:
+  - $ref: "#/definitions/streams/locations"
+  - $ref: "#/definitions/streams/answers"
+  - $ref: "#/definitions/streams/lists"
+  - $ref: "#/definitions/streams/notifications"
+
+spec:
+  type: Spec
+  connection_specification:
+    type: object
+    $schema: http://json-schema.org/draft-07/schema#
+    required:
+      - api_key
+      - start_date
+    properties:
+      api_key:
+        type: string
+        title: Authorization
+        airbyte_secret: true
+        description: API Key
+        order: 0
+      start_date:
+        type: string
+        title: Start Date
+        description: Start date for the data to be replicated
+        examples:
+          - dd/mm/YYYY HH:MM
+        pattern: ^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}$
+        order: 1
+    additionalProperties: true
+
+metadata:
+  autoImportSchema:
+    locations: false
+    answers: false
+    lists: false
+    notifications: false
+
+schemas:
+  locations:
+    "$schema": http://json-schema.org/draft-04/schema#
+    type: object
+    properties:
+      id:
+        type: integer
+      name:
+        type: string
+      description:
+        type:
+          - string
+          - "null"
+      code:
+        type:
+          - string
+          - "null"
+      address:
+        type:
+          - string
+          - "null"
+      city:
+        type:
+          - string
+          - "null"
+      country:
+        type:
+          - string
+          - "null"
+      latitude:
+        type:
+          - number
+          - "null"
+      longitude:
+        type:
+          - number
+          - "null"
+      region:
+        type:
+          - string
+          - "null"
+      phone:
+        type:
+          - string
+          - "null"
+      company_code:
+        type:
+          - string
+          - "null"
+      company_name:
+        type:
+          - string
+          - "null"
+
+  answers:
+    "$schema": http://json-schema.org/draft-07/schema#
+    type: object
+    properties:
+      form_name:
+        type: string
+      form_state:
+        type:
+          - string
+          - "null"
+      user_name:
+        type: string
+      user_identifier:
+        type: string
+      code:
+        type: string
+      form_id:
+        type: integer
+      created_at:
+        type: string
+      form_answer_id:
+        type: integer
+      latitude:
+        type:
+          - number
+          - "null"
+      longitude:
+        type:
+          - number
+          - "null"
+      "[question_name1]":
+        type: string
+      "[question_name2]":
+        type: string
+      "[question_name3]":
+        type: string
+  lists:
+    "$schema": http://json-schema.org/draft-07/schema#
+    type: object
+    properties:
+      id:
+        type: integer
+      name:
+        type: string
+      description:
+        type: string
+      attribute1:
+        type: string
+      attribute2:
+        type: string
+      list_id:
+        type: integer
+      account_id:
+        type: integer
+      code:
+        type: string
+      created_at:
+        type: string
+      updated_at:
+        type: string
+
+  notifications:
+    "$schema": http://json-schema.org/draft-04/schema#
+    type: object
+    properties:
+      id:
+        type: integer
+      type:
+        type: string
+      url:
+        type: string
+      form_name:
+        type: string
+      form_code:
+        type: string
+      user:
+        type: string
+      created_at:
+        type: string

airbyte-integrations/connectors/source-datascope/source_datascope/schemas/answers.json

@@ -1,45 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "type": "object",
-  "properties": {
-    "form_name": {
-      "type": "string"
-    },
-    "form_state": {
-      "type": ["string", "null"]
-    },
-    "user_name": {
-      "type": "string"
-    },
-    "user_identifier": {
-      "type": "string"
-    },
-    "code": {
-      "type": "string"
-    },
-    "form_id": {
-      "type": "integer"
-    },
-    "created_at": {
-      "type": "string"
-    },
-    "form_answer_id": {
-      "type": "integer"
-    },
-    "latitude": {
-      "type": ["number", "null"]
-    },
-    "longitude": {
-      "type": ["number", "null"]
-    },
-    "[question_name1]": {
-      "type": "string"
-    },
-    "[question_name2]": {
-      "type": "string"
-    },
-    "[question_name3]": {
-      "type": "string"
-    }
-  }
-}

airbyte-integrations/connectors/source-datascope/source_datascope/schemas/lists.json

@@ -1,36 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "type": "object",
-  "properties": {
-    "id": {
-      "type": "integer"
-    },
-    "name": {
-      "type": "string"
-    },
-    "description": {
-      "type": "string"
-    },
-    "attribute1": {
-      "type": "string"
-    },
-    "attribute2": {
-      "type": "string"
-    },
-    "list_id": {
-      "type": "integer"
-    },
-    "account_id": {
-      "type": "integer"
-    },
-    "code": {
-      "type": "string"
-    },
-    "created_at": {
-      "type": "string"
-    },
-    "updated_at": {
-      "type": "string"
-    }
-  }
-}

airbyte-integrations/connectors/source-datascope/source_datascope/schemas/locations.json

@@ -1,45 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "type": "object",
-  "properties": {
-    "id": {
-      "type": "integer"
-    },
-    "name": {
-      "type": "string"
-    },
-    "description": {
-      "type": ["string", "null"]
-    },
-    "code": {
-      "type": ["string", "null"]
-    },
-    "address": {
-      "type": ["string", "null"]
-    },
-    "city": {
-      "type": ["string", "null"]
-    },
-    "country": {
-      "type": ["string", "null"]
-    },
-    "latitude": {
-      "type": ["number", "null"]
-    },
-    "longitude": {
-      "type": ["number", "null"]
-    },
-    "region": {
-      "type": ["string", "null"]
-    },
-    "phone": {
-      "type": ["string", "null"]
-    },
-    "company_code": {
-      "type": ["string", "null"]
-    },
-    "company_name": {
-      "type": ["string", "null"]
-    }
-  }
-}

airbyte-integrations/connectors/source-datascope/source_datascope/schemas/notifications.json

@@ -1,27 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "type": "object",
-  "properties": {
-    "id": {
-      "type": "integer"
-    },
-    "type": {
-      "type": "string"
-    },
-    "url": {
-      "type": "string"
-    },
-    "form_name": {
-      "type": "string"
-    },
-    "form_code": {
-      "type": "string"
-    },
-    "user": {
-      "type": "string"
-    },
-    "created_at": {
-      "type": "string"
-    }
-  }
-}

airbyte-integrations/connectors/source-datascope/source_datascope/spec.yaml

@@ -1,22 +0,0 @@
-documentationUrl: "https://docs.airbyte.com/integrations/sources/datascope"
-connectionSpecification:
-  $schema: "http://json-schema.org/draft-07/schema#"
-  title: Datascope Spec
-  type: object
-  required:
-    - api_key
-    - start_date
-  additionalProperties: true
-  properties:
-    start_date:
-      title: Start Date
-      type: string
-      description: Start date for the data to be replicated
-      examples:
-        - "dd/mm/YYYY HH:MM"
-      pattern: "^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}$"
-    api_key:
-      title: Authorization
-      type: string
-      description: API Key
-      airbyte_secret: true

docs/integrations/sources/datascope.md

@@ -64,8 +64,9 @@ GET https://www.mydatascope.com/api/external/locations
 
 | Version | Date       | Pull Request                                              | Subject        |
 | :------ | :--------- | :-------------------------------------------------------- | :------------- |
-| 0.1.2 | 2024-06-06 | [39254](https://github.com/airbytehq/airbyte/pull/39254) | [autopull] Upgrade base image to v1.2.2 |
-| 0.1.1 | 2024-05-20 | [38440](https://github.com/airbytehq/airbyte/pull/38440) | [autopull] base image + poetry + up_to_date |
+| 0.1.3   | 2024-06-15 | [38844](https://github.com/airbytehq/airbyte/pull/38844)  | Make compatible with builder |
+| 0.1.2   | 2024-06-06 | [39254](https://github.com/airbytehq/airbyte/pull/39254)  | [autopull] Upgrade base image to v1.2.2 |
+| 0.1.1   | 2024-05-20 | [38440](https://github.com/airbytehq/airbyte/pull/38440)  | [autopull] base image + poetry + up_to_date |
 | 0.1.0   | 2022-10-31 | [#18725](https://github.com/airbytehq/airbyte/pull/18725) | Initial commit |
 
 </details>