365 lines
8.3 KiB
YAML
365 lines
8.3 KiB
YAML
version: 5.10.2
|
|
|
|
type: DeclarativeSource
|
|
|
|
description: >-
|
|
Open Data DC source connector which ingests data from the MAR 2 API.
|
|
|
|
The District of Columbia government uses the Master Address Repository (MAR)
|
|
to implement intelligent search functionality for finding and verifying
|
|
addresses, place names, blocks and intersections.
|
|
|
|
More information can be found here https://developers.data.dc.gov/
|
|
|
|
check:
|
|
type: CheckStream
|
|
stream_names:
|
|
- locations
|
|
|
|
definitions:
|
|
streams:
|
|
locations:
|
|
type: DeclarativeStream
|
|
name: locations
|
|
retriever:
|
|
type: SimpleRetriever
|
|
requester:
|
|
$ref: "#/definitions/base_requester"
|
|
path: locations/{{ config["location"] }}
|
|
http_method: GET
|
|
record_selector:
|
|
type: RecordSelector
|
|
extractor:
|
|
type: DpathExtractor
|
|
field_path:
|
|
- Result
|
|
- addresses
|
|
schema_loader:
|
|
type: InlineSchemaLoader
|
|
schema:
|
|
$ref: "#/schemas/locations"
|
|
units:
|
|
type: DeclarativeStream
|
|
name: units
|
|
primary_key:
|
|
- UnitNum
|
|
retriever:
|
|
type: SimpleRetriever
|
|
requester:
|
|
$ref: "#/definitions/base_requester"
|
|
path: units/{{ config["marid"] }}
|
|
http_method: GET
|
|
record_selector:
|
|
type: RecordSelector
|
|
extractor:
|
|
type: DpathExtractor
|
|
field_path:
|
|
- Result
|
|
- units
|
|
schema_loader:
|
|
type: InlineSchemaLoader
|
|
schema:
|
|
$ref: "#/schemas/units"
|
|
ssls:
|
|
type: DeclarativeStream
|
|
name: ssls
|
|
primary_key:
|
|
- SSL
|
|
retriever:
|
|
type: SimpleRetriever
|
|
requester:
|
|
$ref: "#/definitions/base_requester"
|
|
path: ssls
|
|
http_method: GET
|
|
request_parameters:
|
|
marid: "{{ config[\"marid\"] }}"
|
|
record_selector:
|
|
type: RecordSelector
|
|
extractor:
|
|
type: DpathExtractor
|
|
field_path:
|
|
- Result
|
|
- ssls
|
|
schema_loader:
|
|
type: InlineSchemaLoader
|
|
schema:
|
|
$ref: "#/schemas/ssls"
|
|
base_requester:
|
|
type: HttpRequester
|
|
url_base: https://datagate.dc.gov/mar/open/api/v2.2/
|
|
authenticator:
|
|
type: ApiKeyAuthenticator
|
|
inject_into:
|
|
type: RequestOption
|
|
inject_into: request_parameter
|
|
field_name: apikey
|
|
api_token: "{{ config[\"api_key\"] }}"
|
|
|
|
streams:
|
|
- $ref: "#/definitions/streams/locations"
|
|
- $ref: "#/definitions/streams/units"
|
|
- $ref: "#/definitions/streams/ssls"
|
|
|
|
spec:
|
|
type: Spec
|
|
connection_specification:
|
|
type: object
|
|
$schema: http://json-schema.org/draft-07/schema#
|
|
required:
|
|
- api_key
|
|
properties:
|
|
api_key:
|
|
type: string
|
|
title: API Key
|
|
airbyte_secret: true
|
|
order: 0
|
|
location:
|
|
type: string
|
|
description: address or place or block
|
|
title: location
|
|
order: 1
|
|
marid:
|
|
type: string
|
|
description: A unique identifier (Master Address Repository).
|
|
title: marid
|
|
order: 2
|
|
additionalProperties: true
|
|
|
|
metadata:
|
|
autoImportSchema:
|
|
locations: true
|
|
units: false
|
|
ssls: true
|
|
testedStreams:
|
|
locations:
|
|
streamHash: 1651fb10d35593f821d053006bcbc1778a51fb23
|
|
hasResponse: true
|
|
responsesAreSuccessful: true
|
|
hasRecords: true
|
|
primaryKeysArePresent: true
|
|
primaryKeysAreUnique: true
|
|
units:
|
|
streamHash: 9ffb509aa58bd50d864d4c1266ed70707ef81e5a
|
|
hasResponse: true
|
|
responsesAreSuccessful: true
|
|
hasRecords: true
|
|
primaryKeysArePresent: true
|
|
primaryKeysAreUnique: true
|
|
ssls:
|
|
streamHash: bfed8da033bc94c8260dabd1a582cb34261fa0fc
|
|
hasResponse: true
|
|
responsesAreSuccessful: true
|
|
hasRecords: true
|
|
primaryKeysArePresent: true
|
|
primaryKeysAreUnique: true
|
|
assist:
|
|
docsUrl: https://developers.data.dc.gov/guide/overview
|
|
|
|
schemas:
|
|
locations:
|
|
type: object
|
|
$schema: http://json-schema.org/schema#
|
|
additionalProperties: true
|
|
properties:
|
|
address:
|
|
type:
|
|
- object
|
|
- "null"
|
|
properties:
|
|
type:
|
|
type:
|
|
- string
|
|
- "null"
|
|
geometry:
|
|
type:
|
|
- object
|
|
- "null"
|
|
properties:
|
|
type:
|
|
type:
|
|
- string
|
|
- "null"
|
|
coordinates:
|
|
type:
|
|
- array
|
|
- "null"
|
|
items:
|
|
type:
|
|
- number
|
|
- "null"
|
|
properties:
|
|
type:
|
|
- object
|
|
- "null"
|
|
properties:
|
|
AddrNum:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Alias:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Anc:
|
|
type:
|
|
- string
|
|
- "null"
|
|
BlockKey:
|
|
type:
|
|
- string
|
|
- "null"
|
|
CensusTract:
|
|
type:
|
|
- string
|
|
- "null"
|
|
FullAddress:
|
|
type:
|
|
- string
|
|
- "null"
|
|
HasCondoUnit:
|
|
type:
|
|
- string
|
|
- "null"
|
|
HasResUnit:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Latitude:
|
|
type:
|
|
- number
|
|
- "null"
|
|
Longitude:
|
|
type:
|
|
- number
|
|
- "null"
|
|
MarId:
|
|
type:
|
|
- string
|
|
- "null"
|
|
NationalGrid:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Quadrant:
|
|
type:
|
|
- string
|
|
- "null"
|
|
ResidenceType:
|
|
type:
|
|
- string
|
|
- "null"
|
|
SSL:
|
|
type:
|
|
- string
|
|
- "null"
|
|
StName:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Status:
|
|
type:
|
|
- string
|
|
- "null"
|
|
StreetType:
|
|
type:
|
|
- string
|
|
- "null"
|
|
SubBlockKey:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Ward:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Xcoord:
|
|
type:
|
|
- number
|
|
- "null"
|
|
Ycoord:
|
|
type:
|
|
- number
|
|
- "null"
|
|
Zipcode:
|
|
type:
|
|
- string
|
|
- "null"
|
|
_Score:
|
|
type:
|
|
- number
|
|
- "null"
|
|
distance:
|
|
type:
|
|
- number
|
|
- "null"
|
|
units:
|
|
type:
|
|
- string
|
|
- "null"
|
|
zones:
|
|
type:
|
|
- object
|
|
- "null"
|
|
units:
|
|
type: object
|
|
$schema: http://json-schema.org/schema#
|
|
properties:
|
|
FullAddress:
|
|
type:
|
|
- string
|
|
- "null"
|
|
MarId:
|
|
type: string
|
|
Status:
|
|
type:
|
|
- string
|
|
- "null"
|
|
UnitNum:
|
|
type:
|
|
- string
|
|
- "null"
|
|
UnitSSL:
|
|
type:
|
|
- string
|
|
- "null"
|
|
UnitType:
|
|
type:
|
|
- string
|
|
- "null"
|
|
required:
|
|
- MarId
|
|
additionalProperties: true
|
|
ssls:
|
|
type: object
|
|
$schema: http://json-schema.org/schema#
|
|
additionalProperties: true
|
|
properties:
|
|
Col:
|
|
type:
|
|
- string
|
|
- "null"
|
|
FullAddress:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Lot:
|
|
type:
|
|
- string
|
|
- "null"
|
|
Lot_type:
|
|
type:
|
|
- string
|
|
- "null"
|
|
MarId:
|
|
type:
|
|
- string
|
|
- "null"
|
|
SSL:
|
|
type: string
|
|
Square:
|
|
type:
|
|
- string
|
|
- "null"
|
|
required:
|
|
- SSL
|