version: 5.10.2

type: DeclarativeSource

description: >-
  Open Data DC source connector which ingests data from the MAR 2 API.

  The District of Columbia government uses the Master Address Repository (MAR)
  to implement intelligent search functionality for finding and verifying
  addresses, place names, blocks and intersections.

  More information can be found here https://developers.data.dc.gov/

check:
  type: CheckStream
  stream_names:
    - locations

definitions:
  streams:
    locations:
      type: DeclarativeStream
      name: locations
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: locations/{{ config["location"] }}
          http_method: GET
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - Result
              - addresses
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/locations"
    units:
      type: DeclarativeStream
      name: units
      primary_key:
        - UnitNum
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: units/{{ config["marid"] }}
          http_method: GET
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - Result
              - units
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/units"
    ssls:
      type: DeclarativeStream
      name: ssls
      primary_key:
        - SSL
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: ssls
          http_method: GET
          request_parameters:
            marid: "{{ config[\"marid\"] }}"
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - Result
              - ssls
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/ssls"
  base_requester:
    type: HttpRequester
    url_base: https://datagate.dc.gov/mar/open/api/v2.2/
    authenticator:
      type: ApiKeyAuthenticator
      inject_into:
        type: RequestOption
        inject_into: request_parameter
        field_name: apikey
      api_token: "{{ config[\"api_key\"] }}"

streams:
  - $ref: "#/definitions/streams/locations"
  - $ref: "#/definitions/streams/units"
  - $ref: "#/definitions/streams/ssls"

spec:
  type: Spec
  connection_specification:
    type: object
    $schema: http://json-schema.org/draft-07/schema#
    required:
      - api_key
    properties:
      api_key:
        type: string
        title: API Key
        airbyte_secret: true
        order: 0
      location:
        type: string
        description: address or place or block
        title: location
        order: 1
      marid:
        type: string
        description: A unique identifier (Master Address Repository).
        title: marid
        order: 2
    additionalProperties: true

metadata:
  autoImportSchema:
    locations: true
    units: false
    ssls: true
  testedStreams:
    locations:
      streamHash: 1651fb10d35593f821d053006bcbc1778a51fb23
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
    units:
      streamHash: 9ffb509aa58bd50d864d4c1266ed70707ef81e5a
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
    ssls:
      streamHash: bfed8da033bc94c8260dabd1a582cb34261fa0fc
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
  assist:
    docsUrl: https://developers.data.dc.gov/guide/overview

schemas:
  locations:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      address:
        type:
          - object
          - "null"
        properties:
          type:
            type:
              - string
              - "null"
          geometry:
            type:
              - object
              - "null"
            properties:
              type:
                type:
                  - string
                  - "null"
              coordinates:
                type:
                  - array
                  - "null"
                items:
                  type:
                    - number
                    - "null"
          properties:
            type:
              - object
              - "null"
            properties:
              AddrNum:
                type:
                  - string
                  - "null"
              Alias:
                type:
                  - string
                  - "null"
              Anc:
                type:
                  - string
                  - "null"
              BlockKey:
                type:
                  - string
                  - "null"
              CensusTract:
                type:
                  - string
                  - "null"
              FullAddress:
                type:
                  - string
                  - "null"
              HasCondoUnit:
                type:
                  - string
                  - "null"
              HasResUnit:
                type:
                  - string
                  - "null"
              Latitude:
                type:
                  - number
                  - "null"
              Longitude:
                type:
                  - number
                  - "null"
              MarId:
                type:
                  - string
                  - "null"
              NationalGrid:
                type:
                  - string
                  - "null"
              Quadrant:
                type:
                  - string
                  - "null"
              ResidenceType:
                type:
                  - string
                  - "null"
              SSL:
                type:
                  - string
                  - "null"
              StName:
                type:
                  - string
                  - "null"
              Status:
                type:
                  - string
                  - "null"
              StreetType:
                type:
                  - string
                  - "null"
              SubBlockKey:
                type:
                  - string
                  - "null"
              Ward:
                type:
                  - string
                  - "null"
              Xcoord:
                type:
                  - number
                  - "null"
              Ycoord:
                type:
                  - number
                  - "null"
              Zipcode:
                type:
                  - string
                  - "null"
              _Score:
                type:
                  - number
                  - "null"
      distance:
        type:
          - number
          - "null"
      units:
        type:
          - string
          - "null"
      zones:
        type:
          - object
          - "null"
  units:
    type: object
    $schema: http://json-schema.org/schema#
    properties:
      FullAddress:
        type:
          - string
          - "null"
      MarId:
        type: string
      Status:
        type:
          - string
          - "null"
      UnitNum:
        type:
          - string
          - "null"
      UnitSSL:
        type:
          - string
          - "null"
      UnitType:
        type:
          - string
          - "null"
    required:
      - MarId
    additionalProperties: true
  ssls:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      Col:
        type:
          - string
          - "null"
      FullAddress:
        type:
          - string
          - "null"
      Lot:
        type:
          - string
          - "null"
      Lot_type:
        type:
          - string
          - "null"
      MarId:
        type:
          - string
          - "null"
      SSL:
        type: string
      Square:
        type:
          - string
          - "null"
    required:
      - SSL