7ca371b26d
* Connector files * Add test files * Add integration test config files * Multiple changes to make it on Airbyte standards * Cleaning up * More clean ups * More clean ups * Removed max pages * Remove unused variable * Correctly separating Full refresh and incremental * Removed unused variables * Fix full_refresh class * Better code for creating stream classes * Fixing review comments * Update docs and Enum class * Update type conversion function * Fix enum class and update docs * Update discover * Implemented some unit tests * Update discover * Update test_source * Increase discovery test timeout * Update configured_catalog * Fix default_cursor_field * Adding final unit tests * Update spec: set client_id and tenant_id as secrets * Update discover to deal with Lookup and Picklist types * Fix Lookup data type conversion * add microsoft dataverse to source def * run format * auto-bump connector version Co-authored-by: Marcelo Pio de Castro <marcelopiocastro@gmail.com> Co-authored-by: daniloss99 <danilosiqueira99@gmail.com> Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
103 lines
4.3 KiB
Python
103 lines
4.3 KiB
Python
#
|
|
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
|
|
#
|
|
|
|
import logging
|
|
from typing import Any, Iterator, List, Mapping, MutableMapping, Tuple, Union
|
|
|
|
from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode
|
|
from airbyte_cdk.sources import AbstractSource
|
|
from airbyte_cdk.sources.streams import Stream
|
|
|
|
from .dataverse import convert_dataverse_type, do_request, get_auth
|
|
from .streams import IncrementalMicrosoftDataverseStream, MicrosoftDataverseStream
|
|
|
|
|
|
class SourceMicrosoftDataverse(AbstractSource):
|
|
def __init__(self):
|
|
self.catalogs = None
|
|
|
|
def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
|
|
response = do_request(config, "EntityDefinitions?$expand=Attributes")
|
|
response_json = response.json()
|
|
streams = []
|
|
for entity in response_json["value"]:
|
|
schema = {"properties": {}}
|
|
for attribute in entity["Attributes"]:
|
|
dataverse_type = attribute["AttributeType"]
|
|
if dataverse_type == "Lookup":
|
|
attribute["LogicalName"] = "_" + attribute["LogicalName"] + "_value"
|
|
attribute_type = convert_dataverse_type(dataverse_type)
|
|
|
|
if not attribute_type:
|
|
continue
|
|
|
|
schema["properties"][attribute["LogicalName"]] = attribute_type
|
|
|
|
if entity["CanChangeTrackingBeEnabled"]["Value"] and entity["ChangeTrackingEnabled"]:
|
|
schema["properties"].update({"_ab_cdc_updated_at": {"type": "string"}, "_ab_cdc_deleted_at": {"type": ["null", "string"]}})
|
|
stream = AirbyteStream(
|
|
name=entity["LogicalName"], json_schema=schema, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]
|
|
)
|
|
stream.source_defined_cursor = True
|
|
if "modifiedon" in schema["properties"]:
|
|
stream.default_cursor_field = ["modifiedon"]
|
|
else:
|
|
stream = AirbyteStream(name=entity["LogicalName"], json_schema=schema, supported_sync_modes=[SyncMode.full_refresh])
|
|
|
|
stream.source_defined_primary_key = [[entity["PrimaryIdAttribute"]]]
|
|
streams.append(stream)
|
|
return AirbyteCatalog(streams=streams)
|
|
|
|
def check_connection(self, logger, config) -> Tuple[bool, any]:
|
|
"""
|
|
:param config: the user-input config object conforming to the connector's spec.yaml
|
|
:param logger: logger object
|
|
:return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
|
|
"""
|
|
try:
|
|
response = do_request(config, "")
|
|
# Raises an exception for error codes (4xx or 5xx)
|
|
response.raise_for_status()
|
|
return True, None
|
|
except Exception as e:
|
|
return False, e
|
|
|
|
def read(
|
|
self,
|
|
logger: logging.Logger,
|
|
config: Mapping[str, Any],
|
|
catalog: ConfiguredAirbyteCatalog,
|
|
state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None,
|
|
) -> Iterator[AirbyteMessage]:
|
|
self.catalogs = catalog
|
|
return super().read(logger, config, catalog, state)
|
|
|
|
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
|
"""
|
|
:param config: A Mapping of the user input configuration as defined in the connector spec.
|
|
"""
|
|
auth = get_auth(config)
|
|
|
|
streams = []
|
|
for catalog in self.catalogs.streams:
|
|
response = do_request(config, f"EntityDefinitions(LogicalName='{catalog.stream.name}')")
|
|
response_json = response.json()
|
|
|
|
args = {
|
|
"url": config["url"],
|
|
"stream_name": catalog.stream.name,
|
|
"stream_path": response_json["EntitySetName"],
|
|
"primary_key": catalog.primary_key,
|
|
"schema": catalog.stream.json_schema,
|
|
"odata_maxpagesize": config["odata_maxpagesize"],
|
|
"authenticator": auth,
|
|
}
|
|
|
|
if catalog.sync_mode == SyncMode.incremental:
|
|
streams.append(IncrementalMicrosoftDataverseStream(**args, config_cursor_field=catalog.cursor_field))
|
|
else:
|
|
streams.append(MicrosoftDataverseStream(**args))
|
|
|
|
return streams
|