jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2026-01-30 16:01:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Run MyPy on CDK/base-python and fix issues. (#3175)

Browse Source
This commit is contained in:
Davin Chia
2021-05-04 11:02:53 +08:00
committed by GitHub
parent dd45537239
commit 72e7fe35a3
11 changed files with 54 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
airbyte-cdk/python/airbyte_cdk/base_python/cdk/abstract_source.py
View File

@@ -166,9 +166,10 @@ class AbstractSource(Source, ABC):
yield self._checkpoint_state(stream_name, stream_state, connector_state, logger)
def _read_full_refresh(self, stream_instance: Stream, configured_stream: ConfiguredAirbyteStream) -> Iterator[AirbyteMessage]:
args = {"sync_mode": SyncMode.full_refresh, "cursor_field": configured_stream.cursor_field}
for slices in stream_instance.stream_slices(**args):
for record in stream_instance.read_records(stream_slice=slices, **args):
slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=configured_stream.cursor_field)
for slice in slices:
records = stream_instance.read_records(stream_slice=slice, sync_mode=SyncMode.full_refresh, cursor_field=configured_stream.cursor_field)
for record in records:
yield self._as_airbyte_record(configured_stream.stream.name, record)
def _checkpoint_state(self, stream_name, stream_state, connector_state, logger):

8
airbyte-cdk/python/airbyte_cdk/base_python/cdk/streams/auth/oauth.py
View File

@@ -21,7 +21,7 @@
# SOFTWARE.
from typing import Any, Mapping, Tuple
from typing import Any, Mapping, Tuple, List, Union, MutableMapping
import pendulum
import requests
@@ -34,7 +34,7 @@ class Oauth2Authenticator(HttpAuthenticator):
The generated access token is attached to each request via the Authorization header.
"""
def __init__(self, token_refresh_endpoint: str, client_id: str, client_secret: str, refresh_token: str, scopes: [str] = None):
def __init__(self, token_refresh_endpoint: str, client_id: str, client_secret: str, refresh_token: str, scopes: List[str] = None):
self.token_refresh_endpoint = token_refresh_endpoint
self.client_secret = client_secret
self.client_id = client_id
@@ -59,9 +59,9 @@ class Oauth2Authenticator(HttpAuthenticator):
def token_has_expired(self) -> bool:
return pendulum.now() > self._token_expiry_date
def get_refresh_request_body(self) -> Mapping[str, any]:
def get_refresh_request_body(self) -> Mapping[str, Any]:
""" Override to define additional parameters """
payload = {
payload: MutableMapping[str, Any] = {
"grant_type": "refresh_token",
"client_id": self.client_id,
"client_secret": self.client_secret,

8
airbyte-cdk/python/airbyte_cdk/base_python/cdk/streams/core.py
View File

@@ -33,7 +33,7 @@ from airbyte_cdk.base_python.schema_helpers import ResourceSchemaLoader
def package_name_from_class(cls: object) -> str:
"""Find the package name given a class name"""
module = inspect.getmodule(cls)
module: Any = inspect.getmodule(cls)
return module.__name__.split(".")[0]
@@ -57,7 +57,7 @@ class Stream(ABC):
self,
sync_mode: SyncMode,
cursor_field: List[str] = None,
stream_slice: Mapping[str, any] = None,
stream_slice: Mapping[str, Any] = None,
stream_state: Mapping[str, Any] = None,
) -> Iterable[Mapping[str, Any]]:
"""
@@ -79,7 +79,7 @@ class Stream(ABC):
if self.supports_incremental:
stream.source_defined_cursor = self.source_defined_cursor
stream.supported_sync_modes.append(SyncMode.incremental)
stream.supported_sync_modes.append(SyncMode.incremental) # type: ignore
stream.default_cursor_field = self._wrapped_cursor_field()
return stream
@@ -110,7 +110,7 @@ class Stream(ABC):
def stream_slices(
self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
) -> Iterable[Optional[Mapping[str, any]]]:
) -> Iterable[Optional[Mapping[str, Any]]]:
"""
Override to define the slices for this stream. See the stream slicing section of the docs for more information.

21
airbyte-cdk/python/airbyte_cdk/base_python/cdk/streams/http.py
View File

@@ -204,28 +204,27 @@ class HttpStream(Stream, ABC):
def read_records(
self,
sync_mode: SyncMode,
stream_slice: Optional[Mapping[str, Any]] = None,
stream_state: Optional[Mapping[str, Any]] = None,
cursor_field: List[str] = None,
stream_slice: Mapping[str, Any] = None,
stream_state: Mapping[str, Any] = None,
) -> Iterable[Mapping[str, Any]]:
stream_state = stream_state or {}
args = {"stream_state": stream_state, "stream_slice": stream_slice}
pagination_complete = False
while not pagination_complete:
next_page_token = None
request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token)
request = self._create_prepared_request(
path=self.path(**args),
headers=dict(self.request_headers(**args), **self.authenticator.get_auth_header()),
params=self.request_params(**args),
json=self.request_body_json(**args),
path=self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
headers=dict(request_headers, **self.authenticator.get_auth_header()),
params=self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
)
response = self._send_request(request)
yield from self.parse_response(response, **args)
yield from self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)
next_page_token = self.next_page_token(response)
if next_page_token:
args["next_page_token"] = next_page_token
else:
if not next_page_token:
pagination_complete = True
# Always return an empty generator just in case no records were ever yielded