🐛 Source Hubspot: fix then state has explicit empty str state value (#26719)
This commit is contained in:
Baz
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
|
||||
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
|
||||
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
|
||||
|
||||
LABEL io.airbyte.version=0.8.0
|
||||
LABEL io.airbyte.version=0.8.1
|
||||
LABEL io.airbyte.name=airbyte/source-hubspot
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -5,7 +5,7 @@ data:
|
||||
connectorSubtype: api
|
||||
connectorType: source
|
||||
definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
|
||||
dockerImageTag: 0.8.0
|
||||
dockerImageTag: 0.8.1
|
||||
dockerRepository: airbyte/source-hubspot
|
||||
githubIssueLabel: source-hubspot
|
||||
icon: hubspot.svg
|
||||
|
||||
@@ -31,5 +31,8 @@
|
||||
},
|
||||
"tickets": {
|
||||
"updatedAt": "2021-02-23T00:00:00Z"
|
||||
},
|
||||
"forms": {
|
||||
"updatedAt": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -753,18 +753,22 @@ class ClientSideIncrementalStream(Stream, IncrementalMixin):
|
||||
if isinstance(record.get(self.cursor_field), str)
|
||||
else pendulum.from_format(str(record.get(self.cursor_field)), self.cursor_field_datetime_format)
|
||||
)
|
||||
start_date = (
|
||||
pendulum.from_format(str(stream_state.get(self.cursor_field)), self.cursor_field_datetime_format)
|
||||
if stream_state
|
||||
else self._start_date
|
||||
)
|
||||
cursor_value = max(start_date, record_value)
|
||||
# default state value before all futher checks
|
||||
state_value = self._start_date
|
||||
# we should check the presence of `stream_state` to overcome `availability strategy` check issues
|
||||
if stream_state:
|
||||
state_value = stream_state.get(self.cursor_field)
|
||||
# sometimes the state is saved as `EMPTY STRING` explicitly
|
||||
state_value = (
|
||||
self._start_date if str(state_value) == "" else pendulum.from_format(str(state_value), self.cursor_field_datetime_format)
|
||||
)
|
||||
# compare the state with record values and get the max value between of two
|
||||
cursor_value = max(state_value, record_value)
|
||||
max_state = max(str(self.state.get(self.cursor_field)), cursor_value.format(self.cursor_field_datetime_format))
|
||||
# save the state
|
||||
self.state = {self.cursor_field: int(max_state) if int_field_type else max_state}
|
||||
return (
|
||||
not stream_state
|
||||
or pendulum.from_format(str(stream_state.get(self.cursor_field)), self.cursor_field_datetime_format) < record_value
|
||||
)
|
||||
# emmit record if it has bigger cursor value compare to the state (`True` only)
|
||||
return record_value > state_value
|
||||
|
||||
def read_records(
|
||||
self,
|
||||
|
||||
@@ -10,7 +10,6 @@ from unittest.mock import MagicMock
|
||||
import mock
|
||||
import pendulum
|
||||
import pytest
|
||||
|
||||
from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type
|
||||
from source_hubspot.errors import HubspotRateLimited, InvalidStartDateConfigError
|
||||
from source_hubspot.helpers import APIv3Property
|
||||
|
||||
@@ -255,6 +255,40 @@ def test_client_side_incremental_stream(requests_mock, common_params, fake_prope
|
||||
assert stream.state == {stream.cursor_field: pendulum.parse(latest_cursor_value).to_rfc3339_string()}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state, record, expected",
|
||||
[
|
||||
({"updatedAt": ""}, {"id": "test_id_1", "updatedAt": "2023-01-30T23:46:36.287Z"}, (True, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})),
|
||||
({"updatedAt": "2023-01-30T23:46:36.287000+00:00"}, {"id": "test_id_1", "updatedAt": "2023-01-29T01:02:03.123Z"}, (False, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})),
|
||||
],
|
||||
ids=[
|
||||
"Empty Sting in state + new record",
|
||||
"State + old record",
|
||||
]
|
||||
)
|
||||
def test_empty_string_in_state(state, record, expected, requests_mock, common_params, fake_properties_list):
|
||||
stream = Forms(**common_params)
|
||||
stream.state = state
|
||||
# overcome the availability strartegy issues by mocking the responses
|
||||
# A.K.A: not related to the test at all, but definetely required.
|
||||
properties_response = [
|
||||
{
|
||||
"json": [
|
||||
{"name": property_name, "type": "string", "CreatedAt": "2023-01-30T23:46:24.355Z", "updatedAt": "2023-01-30T23:46:36.287Z"}
|
||||
for property_name in fake_properties_list
|
||||
],
|
||||
"status_code": 200,
|
||||
}
|
||||
]
|
||||
requests_mock.register_uri("GET", stream.url, json=record)
|
||||
requests_mock.register_uri("GET", "/properties/v2/form/properties", properties_response)
|
||||
# end of mocking `availability strategy`
|
||||
|
||||
result = stream.filter_by_state(stream.state, record)
|
||||
assert result == expected[0]
|
||||
assert stream.state == expected[1]
|
||||
|
||||
|
||||
@pytest.fixture(name="custom_object_schema")
|
||||
def custom_object_schema_fixture():
|
||||
return {
|
||||
|
||||
@@ -146,6 +146,7 @@ Now that you have set up the Hubspot source connector, check out the following H
|
||||
## Changelog
|
||||
| Version | Date | Pull Request | Subject |
|
||||
|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| 0.8.1 | 2023-05-29 | [26719](https://github.com/airbytehq/airbyte/pull/26719) | Handle issue when `state` value is literaly `"" (empty str)`
|
||||
| 0.8.0 | 2023-04-10 | [16032](https://github.com/airbytehq/airbyte/pull/16032) | Add new stream `Custom Object` |
|
||||
| 0.7.0 | 2023-04-10 | [24450](https://github.com/airbytehq/airbyte/pull/24450) | Add new stream `Goals` |
|
||||
| 0.6.2 | 2023-04-28 | [25667](https://github.com/airbytehq/airbyte/pull/25667) | Fixed bug with `Invalid Date` like `2000-00-00T00:00:00Z` while settip up the connector |
|
||||
|
||||
Reference in New Issue
Block a user