cbf9ea76c1
* initial work to parse manifest objects into pydantic models * pr feedback and some other validations * rerun manifest schema generation * remove field constraint due to bug * initial work to construct most components from greenhouse * custom components parse subcomponent fields correctly and adding a few more component constructors * construct components from gnews * first pass at posthog.yaml * Handle nested custom components with list values. Also includes updates to posthog.yaml, including autoformatting changes. * adding constructors for slicers, filters, and transformations and a few bug fixes * make sed work across multiple OS * add NoAuth component * fix handling of custom components with nested list * Autogenerate `TYPE_NAME_TO_MODEL` mapping * Handle default kwargs not defined on model for custom components * Re-add `options` for CartesianProductStreamSlicer for backwards compat with custom stream slicers * add basic unit tests for the model component factory * add back defaults and extra parameters like options to retain compatibility with legacy flow and backwards compatibility * Remove `_get_defaults`; using actual default values on classes instead * Add backoff strategy component creation functions * add back defaults and extra parameters like options to retain compatibility with legacy flow and backwards compatibility * add lots of tests to construct components from the pydantic models and a few bug fixes * add a few tests for the model to component factory * add catch * fix a bug where propagated schema doesn't work with old factory * clean up a few files * add type inference for custom components, more tests and some refactoring of the model factory * template, docs, manifest updates, pr feedback and some cleanup * pr feedback and polish schema a bit * fix tests from the latest rebase of master * fix the last few bugs I found and adjust a few sources that weren't perfectly compatible with the new component flow * fix CheckStream bug cleanup and a few small tweaks and polish * add additional test to cover bug case * fix formatting * 🤖 Bump minor version of Airbyte CDK Co-authored-by: Catherine Noll <noll.catherine@gmail.com> Co-authored-by: brianjlai <brianjlai@users.noreply.github.com>
36 lines
1.2 KiB
Python
36 lines
1.2 KiB
Python
#
|
|
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
|
|
#
|
|
|
|
from dataclasses import dataclass
|
|
from typing import List, Optional
|
|
|
|
from airbyte_cdk.sources.declarative.extractors import DpathExtractor
|
|
from airbyte_cdk.sources.declarative.requesters import RequestOption
|
|
from airbyte_cdk.sources.declarative.requesters.error_handlers import DefaultErrorHandler
|
|
from airbyte_cdk.sources.declarative.requesters.paginators import PaginationStrategy
|
|
from airbyte_cdk.sources.declarative.stream_slicers import SubstreamSlicer
|
|
|
|
|
|
@dataclass
|
|
class TestingSomeComponent(DefaultErrorHandler):
|
|
"""
|
|
A basic test class with various field permutations used to test manifests with custom components
|
|
"""
|
|
|
|
subcomponent_field_with_hint: DpathExtractor = DpathExtractor(field_pointer=[], config={}, options={})
|
|
basic_field: str = ""
|
|
optional_subcomponent_field: Optional[RequestOption] = None
|
|
list_of_subcomponents: List[RequestOption] = None
|
|
without_hint = None
|
|
|
|
|
|
@dataclass
|
|
class TestingCustomSubstreamSlicer(SubstreamSlicer):
|
|
"""
|
|
A test class based on a SubstreamSlicer used for testing manifests that use custom components.
|
|
"""
|
|
|
|
custom_field: str
|
|
custom_pagination_strategy: PaginationStrategy
|