Low code connectors: implement components for sendgrid (#12853)

* checkout from alex/cac * checkout from alex/cac * checkout from alex/cac * Add missing tests * Add missing files * Add missing tests * add missing file * missing file * missing file * rename * doc * doc * remove broken test * rename * jinja dependency * Add comment * comment * comment * pyjq dependency * rename file * delete unused file * Revert "delete unused file" This reverts commit 758e939367. * fix * rename * abstract property * delete unused field * delete unused field * rename * pass kwargs directly * isort * Revert "isort" This reverts commit 4a79223944. * isort * update state * fix imports * update dependency * format * rename file * decoder * Use decoder * Update comment * dict_state is actually backed by a dict * Add a comment * update state takes kwargs * move state out of offset paginator * update jq parameter order * update * remove incremental mixin * delete comment * update comments * update comments * remove no_state * rename package * checkout from alex/cac * Add missing tests * Add missing files * missing file * rename * jinja dependency * Add comment * comment * comment * Revert "delete unused file" This reverts commit 758e939367. * delete unused field * delete unused field * rename * pass kwargs directly * isort * Revert "isort" This reverts commit 4a79223944. * format * decoder * better error handling * remove nostate * isort * remove print * move test * delete duplicates * delete dead code * Update mapping type to [str, Any] * add comment * Add comment * pass parameters through kwargs * pass parameters through kwargs * update interface to pass source in interface * update interface to pass source in interface * rename to stream_slicer * Allow passing a string or an enum * Define StateType enum * convert state_type if not of type type * convert state_type if not of type type * Low code connectors: string interpolation with jinja (#12852) * checkout from alex/cac * Add missing tests * Add missing files * missing file * rename * jinja dependency * Add comment * comment * comment * Revert "delete unused file" This reverts commit 758e939367. * delete unused field * delete unused field * rename * pass kwargs directly * isort * Revert "isort" This reverts commit 4a79223944. * format * decoder * better error handling * remove nostate * isort * delete dead code * Update mapping type to [str, Any] * add comment * Add comment * pass parameters through kwargs * move test to right module * Add missing test * Use authbase instead of deprecated class * leverage generator * Delete dead code * rename methods * rename to declarative * rename the classes too * Try to install packages to build jq * isort * only automake * Revert "only automake" This reverts commit c8fe154ffc. * remove git * format * Add jq dependency * Use request header provider * rename * rename field * remove get_context method * rename
2026-01-05 21:02:13 -05:00 · 2022-06-01 07:02:35 -07:00
parent 7616735bd8
commit d9fa24ffff
24 changed files with 1137 additions and 3 deletions
--- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_interpolated_mapping.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_interpolated_mapping.py
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
+
+
+def test():
+    d = {
+        "field": "value",
+        "field_to_interpolate_from_config": "{{ config['c'] }}",
+        "field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}",
+    }
+    config = {"c": "VALUE_FROM_CONFIG"}
+    kwargs = {"a": "VALUE_FROM_KWARGS"}
+    mapping = InterpolatedMapping(d)
+
+    interpolated = mapping.eval(config, **{"kwargs": kwargs})
+
+    assert interpolated["field"] == "value"
+    assert interpolated["field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG"
+    assert interpolated["field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS"
--- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_interpolated_string.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_interpolated_string.py
@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+
+config = {"field": "value"}
+
+
+def test_static_value():
+    static_value = "HELLO WORLD"
+    s = InterpolatedString(static_value)
+    assert s.eval(config) == "HELLO WORLD"
+
+
+def test_eval_from_config():
+    string = "{{ config['field'] }}"
+    s = InterpolatedString(string)
+    assert s.eval(config) == "value"
+
+
+def test_eval_from_kwargs():
+    string = "{{ kwargs['c'] }}"
+    kwargs = {"c": "airbyte"}
+    s = InterpolatedString(string)
+    assert s.eval(config, **{"kwargs": kwargs}) == "airbyte"
--- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_jinja.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_jinja.py
@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
+
+
+def test_get_value_from_config():
+    interpolation = JinjaInterpolation()
+    s = "{{ config['date'] }}"
+    config = {"date": "2022-01-01"}
+    val = interpolation.eval(s, config)
+    assert val == "2022-01-01"
+
+
+def test_get_value_from_stream_slice():
+    interpolation = JinjaInterpolation()
+    s = "{{ stream_slice['date'] }}"
+    config = {"date": "2022-01-01"}
+    stream_slice = {"date": "2020-09-09"}
+    val = interpolation.eval(s, config, **{"stream_slice": stream_slice})
+    assert val == "2020-09-09"
+
+
+def test_get_value_from_a_list_of_mappings():
+    interpolation = JinjaInterpolation()
+    s = "{{ records[0]['date'] }}"
+    config = {"date": "2022-01-01"}
+    records = [{"date": "2020-09-09"}]
+    val = interpolation.eval(s, config, **{"records": records})
+    assert val == "2020-09-09"
--- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_request_parameter_provider.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_request_parameter_provider.py
@@ -0,0 +1,78 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import (
+    InterpolatedRequestParameterProvider,
+)
+
+state = {"date": "2021-01-01"}
+stream_slice = {"start_date": "2020-01-01"}
+next_page_token = {"offset": "12345"}
+config = {"option": "OPTION"}
+
+
+def test():
+    request_parameters = {"a_static_request_param": "a_static_value"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_parameters == request_params
+
+
+def test_value_depends_on_state():
+    request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == state["date"]
+
+
+def test_value_depends_on_stream_slice():
+    request_parameters = {"a_static_request_param": "{{ stream_slice['start_date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == stream_slice["start_date"]
+
+
+def test_value_depends_on_next_page_token():
+    request_parameters = {"a_static_request_param": "{{ next_page_token['offset'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == next_page_token["offset"]
+
+
+def test_value_depends_on_config():
+    request_parameters = {"a_static_request_param": "{{ config['option'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == config["option"]
+
+
+def test_parameter_is_interpolated():
+    request_parameters = {
+        "{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC"
+    }
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params[f"{state['date']} - {stream_slice['start_date']} - {next_page_token['offset']} - {config['option']}"] == "ABC"
+
+
+def test_none_value():
+    request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params({}, stream_slice, next_page_token)
+
+    assert len(request_params) == 0
--- a/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_interpolated_request_parameter_provider.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_interpolated_request_parameter_provider.py
@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import (
+    InterpolatedRequestParameterProvider,
+)
+
+state = {"date": "2021-01-01"}
+stream_slice = {"start_date": "2020-01-01"}
+next_page_token = {"offset": "12345"}
+config = {"option": "OPTION"}
+
+
+def test():
+    request_parameters = {"a_static_request_param": "a_static_value"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_parameters == request_params
+
+
+def test_value_depends_on_state():
+    request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == state["date"]
+
+
+def test_value_depends_on_stream_slice():
+    request_parameters = {"a_static_request_param": "{{ stream_slice['start_date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == stream_slice["start_date"]
+
+
+def test_value_depends_on_next_page_token():
+    request_parameters = {"a_static_request_param": "{{ next_page_token['offset'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == next_page_token["offset"]
+
+
+def test_value_depends_on_config():
+    request_parameters = {"a_static_request_param": "{{ config['option'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == config["option"]
+
+
+def test_parameter_is_interpolated():
+    request_parameters = {
+        "{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC"
+    }
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params[f"{state['date']} - {stream_slice['start_date']} - {next_page_token['offset']} - {config['option']}"] == "ABC"
+
+
+def test_none_value():
+    request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params({}, stream_slice, next_page_token)
+
+    assert len(request_params) == 0
--- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_next_page_url_paginator.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_next_page_url_paginator.py
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import json
+
+import requests
+from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
+from airbyte_cdk.sources.declarative.requesters.paginators.interpolated_paginator import InterpolatedPaginator
+from airbyte_cdk.sources.declarative.requesters.paginators.next_page_url_paginator import NextPageUrlPaginator
+
+config = {"option": "OPTION"}
+response = requests.Response()
+response.headers = {"A_HEADER": "HEADER_VALUE"}
+response_body = {"_metadata": {"next": "https://airbyte.io/next_url"}}
+response._content = json.dumps(response_body).encode("utf-8")
+last_responses = [{"id": 0}]
+decoder = JsonDecoder()
+
+
+def test_value_depends_response_body():
+    next_page_tokens = {"next_page_url": "{{ decoded_response['_metadata']['next'] }}"}
+    paginator = create_paginator(next_page_tokens)
+
+    next_page_token = paginator.next_page_token(response, last_responses)
+
+    assert next_page_token == {"next_page_url": "next_url"}
+
+
+def test_no_next_page_found():
+    next_page_tokens = {"next_page_url": "{{ decoded_response['_metadata']['next'] }}"}
+    paginator = create_paginator(next_page_tokens)
+
+    r = requests.Response()
+    r._content = json.dumps({"data": []}).encode("utf-8")
+    next_page_token = paginator.next_page_token(r, last_responses)
+
+    assert next_page_token is None
+
+
+def create_paginator(template):
+    return NextPageUrlPaginator("https://airbyte.io/", InterpolatedPaginator(template, decoder, config))
--- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_paginator.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_paginator.py
@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import requests
+from airbyte_cdk.sources.declarative.requesters.paginators.offset_paginator import OffsetPaginator
+from airbyte_cdk.sources.declarative.states.dict_state import DictState
+
+response = requests.Response()
+
+tag = "cursor"
+last_responses = [{"id": 0}, {"id": 1}]
+state = DictState()
+
+
+def test_return_none_if_fewer_records_than_limit():
+    limit = 5
+    paginator = OffsetPaginator(limit, state, tag)
+
+    assert paginator._get_offset() == 0
+
+    next_page_token = paginator.next_page_token(response, last_responses)
+
+    assert next_page_token is None
+
+
+def test_return_next_offset_limit_1():
+    limit = 1
+    paginator = OffsetPaginator(limit, state, tag)
+
+    next_page_token = paginator.next_page_token(response, last_responses)
+
+    assert next_page_token == {tag: 1}
+    assert paginator._get_offset() == 1
+
+
+def test_return_next_offset_limit_2():
+    limit = 2
+    paginator = OffsetPaginator(limit, state, tag)
+
+    next_page_token = paginator.next_page_token(response, last_responses)
+
+    assert next_page_token == {tag: 2}
+    assert paginator._get_offset() == 2
+
+    next_page_token = paginator.next_page_token(response, [{"id": 2}])
+    assert next_page_token is None
--- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_dict_state.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_dict_state.py
@@ -0,0 +1,92 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.states.dict_state import DictState, StateType
+
+config = {"name": "date"}
+name = "{{ config['name'] }}"
+value = "{{ last_record['updated_at'] }}"
+dict_mapping = {
+    name: value,
+}
+
+
+def test_empty_state_is_none():
+    state = DictState(dict_mapping, "INT", config)
+    initial_state = state.get_stream_state()
+    expected_state = {}
+    assert expected_state == initial_state
+
+
+def test_state_type():
+    state_type_string = DictState(dict_mapping, "INT", config)
+    state_type_type = DictState(dict_mapping, int, config)
+    state_type_enum = DictState(dict_mapping, StateType.INT, config)
+    assert state_type_string._state_type == state_type_type._state_type == state_type_enum._state_type
+
+
+def test_update_initial_state():
+    state = DictState(dict_mapping, "STR", config)
+    stream_slice = None
+    stream_state = None
+    last_response = {"data": {"id": "1234", "updated_at": "2021-01-01"}, "last_refresh": "2020-01-01"}
+    last_record = {"id": "1234", "updated_at": "2021-01-01"}
+    state.update_state(stream_slice=stream_slice, stream_state=stream_state, last_response=last_response, last_record=last_record)
+    actual_state = state.get_stream_state()
+    expected_state = {"date": "2021-01-01"}
+    assert expected_state == actual_state
+
+
+def test_update_state_with_recent_cursor():
+    state = DictState(dict_mapping, "STR", config)
+    stream_slice = None
+    stream_state = {"date": "2020-12-31"}
+    last_response = {"data": {"id": "1234", "updated_at": "2021-01-01"}, "last_refresh": "2020-01-01"}
+    last_record = {"id": "1234", "updated_at": "2021-01-01"}
+    state.update_state(stream_slice=stream_slice, stream_state=stream_state, last_response=last_response, last_record=last_record)
+    actual_state = state.get_stream_state()
+    expected_state = {"date": "2021-01-01"}
+    assert expected_state == actual_state
+
+
+def test_update_state_with_old_cursor():
+    state = DictState(dict_mapping, "STR", config)
+    stream_slice = None
+    stream_state = {"date": "2021-01-02"}
+    last_response = {"data": {"id": "1234", "updated_at": "2021-01-01"}, "last_refresh": "2020-01-01"}
+    last_record = {"id": "1234", "updated_at": "2021-01-01"}
+    state.update_state(stream_slice=stream_slice, stream_state=stream_state, last_response=last_response, last_record=last_record)
+    actual_state = state.get_stream_state()
+    expected_state = {"date": "2021-01-02"}
+    assert expected_state == actual_state
+
+
+def test_update_state_with_older_state():
+    state = DictState(dict_mapping, "STR", config)
+    stream_slice = None
+    stream_state = {"date": "2021-01-02"}
+    last_response = {"data": {"id": "1234", "updated_at": "2021-01-02"}, "last_refresh": "2020-01-01"}
+    last_record = {"id": "1234", "updated_at": "2021-01-02"}
+    state.update_state(stream_slice=stream_slice, stream_state=stream_state, last_response=last_response, last_record=last_record)
+    actual_state = state.get_stream_state()
+    expected_state = {"date": "2021-01-02"}
+
+    out_of_order_response = {"data": {"id": "1234", "updated_at": "2021-01-02"}, "last_refresh": "2020-01-01"}
+    out_of_order_record = {"id": "1234", "updated_at": "2021-01-01"}
+    state.update_state(
+        stream_slice=stream_slice, stream_state=stream_state, last_response=out_of_order_response, last_record=out_of_order_record
+    )
+    assert expected_state == actual_state
+
+
+def test_state_is_a_timestamp():
+    state = DictState(dict_mapping, "INT", config)
+    stream_slice = None
+    stream_state = {"date": 12345}
+    last_response = {"data": {"id": "1234", "updated_at": 123456}, "last_refresh": "2020-01-01"}
+    last_record = {"id": "1234", "updated_at": 123456}
+    state.update_state(stream_slice=stream_slice, stream_state=stream_state, last_response=last_response, last_record=last_record)
+    actual_state = state.get_stream_state()
+    expected_state = {"date": 123456}
+    assert expected_state == actual_state
--- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py
@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from unittest.mock import MagicMock
+
+import requests
+from airbyte_cdk.sources.declarative.requesters.http_requester import HttpMethod, HttpRequester
+
+
+def test():
+    http_method = "GET"
+
+    request_parameters_provider = MagicMock()
+    request_params = {"param": "value"}
+    request_parameters_provider.request_params.return_value = request_params
+
+    request_headers_provider = MagicMock()
+    request_headers = {"header": "value"}
+    request_headers_provider.request_headers.return_value = request_headers
+
+    authenticator = MagicMock()
+
+    retrier = MagicMock()
+    max_retries = 10
+    should_retry = True
+    backoff_time = 1000
+    retrier.max_retries = max_retries
+    retrier.should_retry.return_value = should_retry
+    retrier.backoff_time.return_value = backoff_time
+
+    config = {"url": "https://airbyte.io"}
+    stream_slice = {"id": "1234"}
+
+    name = "stream_name"
+
+    requester = HttpRequester(
+        name=name,
+        url_base="{{ config['url'] }}",
+        path="v1/{{ stream_slice['id'] }}",
+        http_method=http_method,
+        request_parameters_provider=request_parameters_provider,
+        request_headers_provider=request_headers_provider,
+        authenticator=authenticator,
+        retrier=retrier,
+        config=config,
+    )
+
+    assert requester.get_url_base() == "https://airbyte.io"
+    assert requester.get_path(stream_state=None, stream_slice=stream_slice, next_page_token=None) == "v1/1234"
+    assert requester.get_authenticator() == authenticator
+    assert requester.get_method() == HttpMethod.GET
+    assert requester.request_params(stream_state=None, stream_slice=None, next_page_token=None) == request_params
+    assert requester.max_retries == max_retries
+    assert requester.should_retry(requests.Response()) == should_retry
+    assert requester.backoff_time(requests.Response()) == backoff_time
--- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_parameter_provider.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_parameter_provider.py
@@ -0,0 +1,78 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import (
+    InterpolatedRequestParameterProvider,
+)
+
+state = {"date": "2021-01-01"}
+stream_slice = {"start_date": "2020-01-01"}
+next_page_token = {"offset": "12345"}
+config = {"option": "OPTION"}
+
+
+def test():
+    request_parameters = {"a_static_request_param": "a_static_value"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_parameters == request_params
+
+
+def test_value_depends_on_state():
+    request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == state["date"]
+
+
+def test_value_depends_on_stream_slice():
+    request_parameters = {"a_static_request_param": "{{ stream_slice['start_date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == stream_slice["start_date"]
+
+
+def test_value_depends_on_next_page_token():
+    request_parameters = {"a_static_request_param": "{{ next_page_token['offset'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == next_page_token["offset"]
+
+
+def test_value_depends_on_config():
+    request_parameters = {"a_static_request_param": "{{ config['option'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params["a_static_request_param"] == config["option"]
+
+
+def test_parameter_is_interpolated():
+    request_parameters = {
+        "{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC"
+    }
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params(state, stream_slice, next_page_token)
+
+    assert request_params[f"{state['date']} - {stream_slice['start_date']} - {next_page_token['offset']} - {config['option']}"] == "ABC"
+
+
+def test_none_value():
+    request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"}
+    provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config)
+
+    request_params = provider.request_params({}, stream_slice, next_page_token)
+
+    assert len(request_params) == 0
--- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py
@@ -33,7 +33,7 @@ def test():

    state = MagicMock()
    underlying_state = {"date": "2021-01-01"}
-    state.get_state.return_value = underlying_state
+    state.get_stream_state.return_value = underlying_state

    url_base = "https://airbyte.io"
    requester.get_url_base.return_value = url_base
--- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/init.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/init.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
--- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py
@@ -0,0 +1,138 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import datetime
+import unittest
+
+import pytest
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer
+
+FAKE_NOW = datetime.datetime(2022, 1, 1, tzinfo=datetime.timezone.utc)
+
+config = {"start_date": "2021-01-01"}
+start_date = InterpolatedString("{{ stream_state['date'] }}", "{{ config['start_date'] }}")
+end_date_now = InterpolatedString(
+    "{{ today_utc() }}",
+)
+end_date = InterpolatedString("2021-01-10")
+cursor_value = InterpolatedString("{{ stream_state['date'] }}")
+timezone = datetime.timezone.utc
+
+datetime_format = "%Y-%m-%d"
+
+
+@pytest.fixture()
+def mock_datetime_now(monkeypatch):
+    datetime_mock = unittest.mock.MagicMock(wraps=datetime.datetime)
+    datetime_mock.now.return_value = FAKE_NOW
+    monkeypatch.setattr(datetime, "datetime", datetime_mock)
+
+
+def test_stream_slices_1_day(mock_datetime_now):
+    stream_state = None
+
+    expected_slices = [
+        {"start_date": "2021-01-01", "end_date": "2021-01-01"},
+        {"start_date": "2021-01-02", "end_date": "2021-01-02"},
+        {"start_date": "2021-01-03", "end_date": "2021-01-03"},
+        {"start_date": "2021-01-04", "end_date": "2021-01-04"},
+        {"start_date": "2021-01-05", "end_date": "2021-01-05"},
+        {"start_date": "2021-01-06", "end_date": "2021-01-06"},
+        {"start_date": "2021-01-07", "end_date": "2021-01-07"},
+        {"start_date": "2021-01-08", "end_date": "2021-01-08"},
+        {"start_date": "2021-01-09", "end_date": "2021-01-09"},
+        {"start_date": "2021-01-10", "end_date": "2021-01-10"},
+    ]
+    step = "1d"
+    slicer = DatetimeStreamSlicer(start_date, end_date, step, cursor_value, datetime_format, config)
+    stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state)
+
+    assert expected_slices == stream_slices
+
+
+def test_stream_slices_2_days(mock_datetime_now):
+    stream_state = None
+
+    expected_slices = [
+        {"start_date": "2021-01-01", "end_date": "2021-01-02"},
+        {"start_date": "2021-01-03", "end_date": "2021-01-04"},
+        {"start_date": "2021-01-05", "end_date": "2021-01-06"},
+        {"start_date": "2021-01-07", "end_date": "2021-01-08"},
+        {"start_date": "2021-01-09", "end_date": "2021-01-10"},
+    ]
+    step = "2d"
+    slicer = DatetimeStreamSlicer(start_date, end_date, step, cursor_value, datetime_format, config)
+    stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state)
+
+    assert expected_slices == stream_slices
+
+
+def test_stream_slices_from_stream_state(mock_datetime_now):
+    stream_state = {"date": "2021-01-05"}
+
+    expected_slices = [
+        # FIXME: should this include 2021-01-05?
+        {"start_date": "2021-01-05", "end_date": "2021-01-05"},
+        {"start_date": "2021-01-06", "end_date": "2021-01-06"},
+        {"start_date": "2021-01-07", "end_date": "2021-01-07"},
+        {"start_date": "2021-01-08", "end_date": "2021-01-08"},
+        {"start_date": "2021-01-09", "end_date": "2021-01-09"},
+        {"start_date": "2021-01-10", "end_date": "2021-01-10"},
+    ]
+    step = "1d"
+    slicer = DatetimeStreamSlicer(start_date, end_date, step, cursor_value, datetime_format, config)
+    stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state)
+
+    assert expected_slices == stream_slices
+
+
+def test_stream_slices_12_days(mock_datetime_now):
+    stream_state = None
+
+    expected_slices = [
+        {"start_date": "2021-01-01", "end_date": "2021-01-10"},
+    ]
+    step = "12d"
+    slicer = DatetimeStreamSlicer(start_date, end_date, step, cursor_value, datetime_format, config)
+    stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state)
+
+    assert expected_slices == stream_slices
+
+
+def test_init_from_config(mock_datetime_now):
+    step = "1d"
+
+    slicer = DatetimeStreamSlicer(start_date, end_date_now, step, cursor_value, datetime_format, config)
+    assert datetime.datetime(2021, 1, 1, tzinfo=timezone) == slicer._start_time
+    assert FAKE_NOW == slicer._end_time
+    assert datetime.timedelta(days=1) == slicer._step
+    assert datetime.timezone.utc == slicer._timezone
+    assert datetime_format == slicer._datetime_format
+
+
+def test_end_date_past_now(mock_datetime_now):
+    step = "1d"
+    invalid_end_date = InterpolatedString(
+        f"{(FAKE_NOW + datetime.timedelta(days=1)).strftime(datetime_format)}",
+    )
+    slicer = DatetimeStreamSlicer(start_date, invalid_end_date, step, cursor_value, datetime_format, config)
+
+    assert slicer._end_time != invalid_end_date
+    assert slicer._end_time == datetime.datetime.now()
+
+
+def test_start_date_after_end_date():
+    step = "1d"
+    invalid_start_date = InterpolatedString("2021-01-11")
+    slicer = DatetimeStreamSlicer(invalid_start_date, end_date, step, cursor_value, datetime_format, config)
+
+    assert slicer._start_time != invalid_start_date
+    assert slicer._start_time == slicer._end_time
+    assert slicer._start_time == datetime.datetime(2021, 1, 10, tzinfo=datetime.timezone.utc)
+
+
+if __name__ == "__main__":
+    unittest.main()
--- a/airbyte-cdk/python/unit_tests/sources/declarative/test_jq.py
+++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_jq.py
@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import json
+
+import requests
+from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
+from airbyte_cdk.sources.declarative.extractors.jq import JqExtractor
+
+config = {"field": "record_array"}
+decoder = JsonDecoder()
+
+
+def test():
+    transform = ".data[]"
+    extractor = JqExtractor(transform, decoder, config)
+
+    records = [{"id": 1}, {"id": 2}]
+    body = {"data": records}
+    response = create_response(body)
+    actual_records = extractor.extract_records(response)
+
+    assert actual_records == records
+
+
+def test_field_in_config():
+    transform = ".{{ config['field'] }}[]"
+    extractor = JqExtractor(transform, decoder, config)
+
+    records = [{"id": 1}, {"id": 2}]
+    body = {"record_array": records}
+    response = create_response(body)
+    actual_records = extractor.extract_records(response)
+
+    assert actual_records == records
+
+
+def test_field_in_kwargs():
+    transform = ".{{ kwargs['data_field'] }}[]"
+    kwargs = {"data_field": "records"}
+    extractor = JqExtractor(transform, decoder, config, kwargs=kwargs)
+
+    records = [{"id": 1}, {"id": 2}]
+    body = {"records": records}
+    response = create_response(body)
+    actual_records = extractor.extract_records(response)
+
+    assert actual_records == records
+
+
+def create_response(body):
+    response = requests.Response()
+    response._content = json.dumps(body).encode("utf-8")
+    return response
+
+
+def test_default():
+    transform = ".{{kwargs['field']}}[]"
+    extractor = JqExtractor(transform, decoder, config)
+
+    records = [{"id": 1}, {"id": 2}]
+    response = create_response(records)
+    actual_records = extractor.extract_records(response)
+
+    assert actual_records == records