221c9b1839
* Restore AvailabilityStrategy
* Add test for http availability strategy on empty stream from original bug fix 7c17351631
* fix flake errors
* Change CheckStream to use availability strategy
* Refactor test from bug fix
* fix flake errors
* Remove extra helper file from cherry-pick
* Merge tests for default http availability strategy
* turn off HttpAvailabilityStrategy as default (for now) (#21488)
* turn off HttpAvailabilityStrategy as default (for now)
* Update imports accordingly
51 lines
2.1 KiB
Python
51 lines
2.1 KiB
Python
#
|
|
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
|
|
#
|
|
|
|
import logging
|
|
import traceback
|
|
from dataclasses import InitVar, dataclass
|
|
from typing import Any, List, Mapping, Tuple
|
|
|
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
|
from airbyte_cdk.sources.source import Source
|
|
from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
|
|
from dataclasses_jsonschema import JsonSchemaMixin
|
|
|
|
|
|
@dataclass
|
|
class CheckStream(ConnectionChecker, JsonSchemaMixin):
|
|
"""
|
|
Checks the connections by checking availability of one or many streams selected by the developer
|
|
|
|
Attributes:
|
|
stream_name (List[str]): names of streams to check
|
|
"""
|
|
|
|
stream_names: List[str]
|
|
options: InitVar[Mapping[str, Any]]
|
|
|
|
def __post_init__(self, options: Mapping[str, Any]):
|
|
self._options = options
|
|
|
|
def check_connection(self, source: Source, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]:
|
|
streams = source.streams(config)
|
|
stream_name_to_stream = {s.name: s for s in streams}
|
|
if len(streams) == 0:
|
|
return False, f"No streams to connect to from source {source}"
|
|
for stream_name in self.stream_names:
|
|
if stream_name not in stream_name_to_stream.keys():
|
|
raise ValueError(f"{stream_name} is not part of the catalog. Expected one of {stream_name_to_stream.keys()}.")
|
|
|
|
stream = stream_name_to_stream[stream_name]
|
|
availability_strategy = stream.availability_strategy or HttpAvailabilityStrategy()
|
|
try:
|
|
stream_is_available, reason = availability_strategy.check_availability(stream, logger, source)
|
|
if stream_is_available:
|
|
return True, None
|
|
else:
|
|
return False, reason
|
|
except Exception as error:
|
|
logger.error(f"Encountered an error trying to connect to stream {stream_name}. Error: \n {traceback.format_exc()}")
|
|
return False, f"Unable to connect to stream {stream_name} - {error}"
|