airbyte/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/manifest.yaml

version: "0.29.0"
definitions:
  page_size: 50
  step: "P30D"

  schema_loader:
    type: JsonFileSchemaLoader
    file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json"

  requester:
    type: HttpRequester
    url_base: "https://api.sendgrid.com"
    http_method: "GET"
    authenticator:
      type: "BearerAuthenticator"
      api_token: "{{ config.apikey }}"
  cursor_paginator:
    type: DefaultPaginator
    page_size: "#/definitions/page_size"
    limit_option:
      inject_into: "request_parameter"
      field_name: "page_size"
    page_token_option:
      type: RequestPath
    pagination_strategy:
      type: "CursorPagination"
      cursor_value: "{{ response._metadata.next }}"
  offset_paginator:
    type: DefaultPaginator
    $parameters:
      page_size: "#/definitions/page_size"
    limit_option:
      inject_into: "request_parameter"
      field_name: "limit"
    page_token_option:
      type: RequestOption
      inject_into: "request_parameter"
      field_name: "offset"
    pagination_strategy:
      type: "OffsetIncrement"
  retriever:
    type: SimpleRetriever
  incremental_sync:
    type: "DatetimeBasedCursor"
    start_datetime:
      datetime: "{{ config['start_time'] }}"
      datetime_format: "%s"
    end_datetime:
      datetime: "{{ now_utc() }}"
      datetime_format: "%Y-%m-%d %H:%M:%S.%f%z"
    step: "#/definitions/step"
    start_time_option:
      field_name: "start_time"
      inject_into: "request_parameter"
    end_time_option:
      field_name: "end_time"
      inject_into: "request_parameter"
    datetime_format: "%s"
    cursor_granularity: "PT1S"
  messages_incremental_sync:
    type: "DatetimeBasedCursor"
    start_datetime:
      datetime: "{{ config['start_time'] }}"
      datetime_format: "%s"
    end_datetime:
      datetime: "{{ now_utc() }}"
      datetime_format: "%Y-%m-%d %H:%M:%S.%f%z"
    step: "#/definitions/step"
    datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
    cursor_granularity: "PT0.000001S"

  base_stream:
    type: DeclarativeStream
    schema_loader:
      $ref: "#/definitions/schema_loader"
    retriever:
      $ref: "#/definitions/retriever"
      record_selector:
        extractor:
          field_path: []
      requester:
        $ref: "#/definitions/requester"
      paginator:
        type: NoPagination
streams:
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "lists"
      primary_key: "id"
      path: "/v3/marketing/lists"
      field_path: ["result"]
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/cursor_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "campaigns"
      primary_key: "id"
      path: "/v3/marketing/campaigns"
      field_path: ["result"]
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/cursor_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "contacts"
      primary_key: "id"
      path: "/v3/marketing/contacts"
      field_path: ["result"]
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "stats_automations"
      primary_key: "id"
      path: "/v3/marketing/stats/automations"
      field_path: ["results"]
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/cursor_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "segments"
      primary_key: "id"
      path: "/v3/marketing/segments"
      field_path: ["results"]
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "single_sends"
      primary_key: "id"
      path: "/v3/marketing/stats/singlesends"
      field_path: ["results"]
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/cursor_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "templates"
      primary_key: "id"
      path: "/v3/templates"
      field_path: ["result"]
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      requester:
        $ref: "#/definitions/base_stream/retriever/requester"
        request_parameters:
          generations: "legacy,dynamic"
      paginator:
        $ref: "#/definitions/cursor_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "bounces"
      primary_key: "email"
      cursor_field: "created"
      path: "/v3/suppression/bounces"
      field_path: []
    incremental_sync:
      $ref: "#/definitions/incremental_sync"
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/offset_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "global_suppressions"
      primary_key: "email"
      cursor_field: "created"
      path: "/v3/suppression/unsubscribes"
      field_path: []
    incremental_sync:
      $ref: "#/definitions/incremental_sync"
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/offset_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "blocks"
      primary_key: "email"
      cursor_field: "created"
      path: "/v3/suppression/blocks"
      field_path: []
    incremental_sync:
      $ref: "#/definitions/incremental_sync"
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/offset_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "suppression_groups"
      primary_key: "id"
      path: "/v3/asm/groups"
      field_path: []
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "suppression_group_members"
      primary_key: "group_id"
      path: "/v3/asm/suppressions"
      field_path: []
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/offset_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "invalid_emails"
      primary_key: "email"
      cursor_field: "created"
      path: "/v3/suppression/invalid_emails"
      field_path: []
    incremental_sync:
      $ref: "#/definitions/incremental_sync"
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/offset_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "spam_reports"
      primary_key: "email"
      cursor_field: "created"
      path: "/v3/suppression/spam_reports"
      field_path: []
    incremental_sync:
      $ref: "#/definitions/incremental_sync"
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      paginator:
        $ref: "#/definitions/offset_paginator"
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "messages"
      primary_key: "msg_id"
      cursor_field: "last_event_time"
      path: "/v3/messages"
      field_path: []
    incremental_sync:
      $ref: "#/definitions/incremental_sync"
    retriever:
      $ref: "#/definitions/base_stream/retriever"
      requester:
        $ref: "#/definitions/requester"
        request_parameters:
          limit: "1000"
          query: 'last_event_time BETWEEN TIMESTAMP "{{stream_slice.start_time}}" AND TIMESTAMP "{{stream_slice.end_time}}"'
  - $ref: "#/definitions/base_stream"
    $parameters:
      name: "unsubscribe_groups"
      primary_key: "id"
      path: "/v3/asm/groups"
      field_path: []
check:
  type: CheckStream
  stream_names: ["lists"]