d9fa24ffff
* checkout from alex/cac * checkout from alex/cac * checkout from alex/cac * Add missing tests * Add missing files * Add missing tests * add missing file * missing file * missing file * rename * doc * doc * remove broken test * rename * jinja dependency * Add comment * comment * comment * pyjq dependency * rename file * delete unused file * Revert "delete unused file" This reverts commit758e939367. * fix * rename * abstract property * delete unused field * delete unused field * rename * pass kwargs directly * isort * Revert "isort" This reverts commit4a79223944. * isort * update state * fix imports * update dependency * format * rename file * decoder * Use decoder * Update comment * dict_state is actually backed by a dict * Add a comment * update state takes kwargs * move state out of offset paginator * update jq parameter order * update * remove incremental mixin * delete comment * update comments * update comments * remove no_state * rename package * checkout from alex/cac * Add missing tests * Add missing files * missing file * rename * jinja dependency * Add comment * comment * comment * Revert "delete unused file" This reverts commit758e939367. * delete unused field * delete unused field * rename * pass kwargs directly * isort * Revert "isort" This reverts commit4a79223944. * format * decoder * better error handling * remove nostate * isort * remove print * move test * delete duplicates * delete dead code * Update mapping type to [str, Any] * add comment * Add comment * pass parameters through kwargs * pass parameters through kwargs * update interface to pass source in interface * update interface to pass source in interface * rename to stream_slicer * Allow passing a string or an enum * Define StateType enum * convert state_type if not of type type * convert state_type if not of type type * Low code connectors: string interpolation with jinja (#12852) * checkout from alex/cac * Add missing tests * Add missing files * missing file * rename * jinja dependency * Add comment * comment * comment * Revert "delete unused file" This reverts commit758e939367. * delete unused field * delete unused field * rename * pass kwargs directly * isort * Revert "isort" This reverts commit4a79223944. * format * decoder * better error handling * remove nostate * isort * delete dead code * Update mapping type to [str, Any] * add comment * Add comment * pass parameters through kwargs * move test to right module * Add missing test * Use authbase instead of deprecated class * leverage generator * Delete dead code * rename methods * rename to declarative * rename the classes too * Try to install packages to build jq * isort * only automake * Revert "only automake" This reverts commitc8fe154ffc. * remove git * format * Add jq dependency * Use request header provider * rename * rename field * remove get_context method * rename
73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
#
|
|
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
|
|
#
|
|
|
|
from enum import Enum
|
|
from typing import Mapping, Union
|
|
|
|
from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
|
|
from airbyte_cdk.sources.declarative.states.state import State
|
|
|
|
|
|
def _get_max(*, name, val, other_state):
|
|
other_val = other_state.get(name)
|
|
if other_val:
|
|
return max(val, other_val)
|
|
else:
|
|
return val
|
|
|
|
|
|
class StateType(Enum):
|
|
STR = str
|
|
INT = int
|
|
|
|
|
|
class DictState(State):
|
|
stream_state_field = "stream_state"
|
|
|
|
def __init__(self, initial_mapping: Mapping[str, str] = None, state_type: Union[str, StateType, type] = "STR", config=None):
|
|
if initial_mapping is None:
|
|
initial_mapping = dict()
|
|
if config is None:
|
|
config = dict()
|
|
self._templates_to_evaluate = initial_mapping
|
|
if type(state_type) == str:
|
|
self._state_type = StateType[state_type].value
|
|
elif type(state_type) == StateType:
|
|
self._state_type = state_type.value
|
|
elif type(state_type) == type:
|
|
self._state_type = state_type
|
|
else:
|
|
raise Exception(f"Unexpected type for state_type. Got {state_type}")
|
|
self._interpolator = JinjaInterpolation()
|
|
self._context = dict()
|
|
self._config = config
|
|
|
|
def update_state(self, **kwargs):
|
|
stream_state = kwargs.get(self.stream_state_field)
|
|
prev_stream_state = self.get_stream_state() or stream_state
|
|
self._context.update(**kwargs)
|
|
|
|
self._context[self.stream_state_field] = self._compute_state(prev_stream_state)
|
|
|
|
def get_state(self, state_field):
|
|
return self._context.get(state_field, {})
|
|
|
|
def get_stream_state(self):
|
|
return self.get_state(self.stream_state_field)
|
|
|
|
def _compute_state(self, prev_state):
|
|
updated_state = {
|
|
self._interpolator.eval(name, self._config): self._interpolator.eval(value, self._config, **self._context)
|
|
for name, value in self._templates_to_evaluate.items()
|
|
}
|
|
updated_state = {name: self._state_type(value) for name, value in updated_state.items() if value}
|
|
|
|
if prev_state:
|
|
next_state = {name: _get_max(name=name, val=value, other_state=prev_state) for name, value in updated_state.items()}
|
|
else:
|
|
next_state = updated_state
|
|
|
|
self._context[self.stream_state_field] = next_state
|
|
return next_state
|