jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-23 21:03:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

🐛 Source Sentry: fix None state_value + other bad state_values (#23619)

Browse Source
This commit is contained in:
Baz
2023-03-02 16:07:54 +02:00
committed by GitHub
parent e1f9905229
commit 93fc5c9c0b
8 changed files with 66 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View File

@@ -2220,7 +2220,7 @@
- sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781
name: Sentry
dockerRepository: airbyte/source-sentry
dockerImageTag: 0.1.11
dockerImageTag: 0.1.12
documentationUrl: https://docs.airbyte.com/integrations/sources/sentry
icon: sentry.svg
sourceType: api

2
airbyte-config/init/src/main/resources/seed/source_specs.yaml
View File

@@ -16831,7 +16831,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-sentry:0.1.11"
- dockerImage: "airbyte/source-sentry:0.1.12"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/sentry"
connectionSpecification:

2
airbyte-integrations/connectors/source-sentry/Dockerfile
View File

@@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.11
LABEL io.airbyte.version=0.1.12
LABEL io.airbyte.name=airbyte/source-sentry

3
airbyte-integrations/connectors/source-sentry/README.md
View File

@@ -102,7 +102,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Connector A
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
To run your integration tests with acceptance tests, from the connector root, run
```
python -m pytest integration_tests -p integration_tests.acceptance
docker build . --no-cache -t airbyte/source-sentry:dev \
&& python -m pytest -p connector_acceptance_test.plugin
```
To run your integration tests with docker

21
airbyte-integrations/connectors/source-sentry/source_sentry/streams.py
View File

@@ -20,6 +20,8 @@ class SentryStream(HttpStream, ABC):
def __init__(self, hostname: str, **kwargs):
super().__init__(**kwargs)
self._url_base = self.URL_TEMPLATE.format(hostname=hostname, api_version=self.API_VERSION)
# hardcode the start_date default value, since it's not present in spec.
self.start_date = "1900-01-01T00:00:00.0Z"
@property
def url_base(self) -> str:
@@ -69,14 +71,21 @@ class SentryIncremental(SentryStreamPagination, IncrementalMixin):
super(SentryIncremental, self).__init__(*args, **kwargs)
self._cursor_value = None
def validate_state_value(self, state_value: str = None) -> str:
none_or_empty = state_value == "None" if state_value else True
return self.start_date if none_or_empty else state_value
def get_state_value(self, stream_state: Mapping[str, Any] = None) -> str:
state_value = self.validate_state_value(stream_state.get(self.cursor_field, self.start_date) if stream_state else self.start_date)
return pendulum.parse(state_value)
def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable:
"""
Endpoint does not provide query filtering params, but they provide us
cursor field in most cases, so we used that as incremental filtering
during the parsing.
"""
start_date = "1900-01-01T00:00:00.0Z"
if pendulum.parse(record[self.cursor_field]) > pendulum.parse((stream_state or {}).get(self.cursor_field, start_date)):
if pendulum.parse(record[self.cursor_field]) > self.get_state_value(stream_state):
# Persist state.
# There is a bug in state setter: because of self._cursor_value is not defined it raises Attribute error
# which is ignored in airbyte_cdk/sources/abstract_source.py:320 and we have an empty state in return
@@ -92,7 +101,7 @@ class SentryIncremental(SentryStreamPagination, IncrementalMixin):
@property
def state(self) -> Mapping[str, Any]:
return {self.cursor_field: str(self._cursor_value)}
return {self.cursor_field: self._cursor_value}
@state.setter
def state(self, value: Mapping[str, Any]):
@@ -100,9 +109,11 @@ class SentryIncremental(SentryStreamPagination, IncrementalMixin):
Define state as a max between given value and current state
"""
if not self._cursor_value:
self._cursor_value = value[self.cursor_field]
self._cursor_value = value.get(self.cursor_field)
else:
self._cursor_value = max(value[self.cursor_field], self.state[self.cursor_field])
current_value = value.get(self.cursor_field) or self.start_date
current_state = str(self.get_state_value(self.state))
self._cursor_value = max(current_value, current_state)
class Events(SentryIncremental):

43
airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py
View File

@@ -4,6 +4,7 @@
from unittest.mock import MagicMock
import pendulum as pdm
import pytest
from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryStreamPagination
@@ -108,3 +109,45 @@ def test_project_detail_request_params():
stream = ProjectDetail(**INIT_ARGS)
expected = {}
assert stream.request_params(stream_state=None, next_page_token=None) == expected
@pytest.mark.parametrize(
"state, expected",
[
({}, "1900-01-01T00:00:00.0Z"),
({"dateCreated": ""}, "1900-01-01T00:00:00.0Z"),
({"dateCreated": "None"}, "1900-01-01T00:00:00.0Z"),
({"dateCreated": "2023-01-01T00:00:00.0Z"}, "2023-01-01T00:00:00.0Z"),
],
ids=[
"No State",
"State is Empty String",
"State is 'None'",
"State is present",
],
)
def test_validate_state_value(state, expected):
stream = Events(**INIT_ARGS)
state_value = state.get(stream.cursor_field)
assert stream.validate_state_value(state_value) == expected
@pytest.mark.parametrize(
"state, expected",
[
({}, "1900-01-01T00:00:00.0Z"),
({"dateCreated": ""}, "1900-01-01T00:00:00.0Z"),
({"dateCreated": "None"}, "1900-01-01T00:00:00.0Z"),
({"dateCreated": "2023-01-01T00:00:00.0Z"}, "2023-01-01T00:00:00.0Z"),
],
ids=[
"No State",
"State is Empty String",
"State is 'None'",
"State is present",
],
)
def test_get_state_value(state, expected):
stream = Events(**INIT_ARGS)
# we expect the datetime object out of get_state_value method.
assert stream.get_state_value(state) == pdm.parse(expected)

2
connectors.md
View File

@@ -208,7 +208,7 @@
| **Sendgrid** | <img alt="Sendgrid icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/sendgrid.svg" height="30" height="30"/> | Source | airbyte/source-sendgrid:0.3.1 | generally_available | [link](https://docs.airbyte.com/integrations/sources/sendgrid) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sendgrid) | <small>`fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87`</small> |
| **Sendinblue** | <img alt="Sendinblue icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/sendinblue.svg" height="30" height="30"/> | Source | airbyte/source-sendinblue:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/sendinblue) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sendinblue) | <small>`2e88fa20-a2f6-43cc-bba6-98a0a3f244fb`</small> |
| **Senseforce** | <img alt="Senseforce icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/senseforce.svg" height="30" height="30"/> | Source | airbyte/source-senseforce:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/senseforce) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-senseforce) | <small>`39de93cb-1511-473e-a673-5cbedb9436af`</small> |
| **Sentry** | <img alt="Sentry icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/sentry.svg" height="30" height="30"/> | Source | airbyte/source-sentry:0.1.11 | generally_available | [link](https://docs.airbyte.com/integrations/sources/sentry) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sentry) | <small>`cdaf146a-9b75-49fd-9dd2-9d64a0bb4781`</small> |
| **Sentry** | <img alt="Sentry icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/sentry.svg" height="30" height="30"/> | Source | airbyte/source-sentry:0.1.12 | generally_available | [link](https://docs.airbyte.com/integrations/sources/sentry) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sentry) | <small>`cdaf146a-9b75-49fd-9dd2-9d64a0bb4781`</small> |
| **Shopify** | <img alt="Shopify icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/shopify.svg" height="30" height="30"/> | Source | airbyte/source-shopify:0.3.2 | alpha | [link](https://docs.airbyte.com/integrations/sources/shopify) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-shopify) | <small>`9da77001-af33-4bcd-be46-6252bf9342b9`</small> |
| **Short.io** | <img alt="Short.io icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/short.svg" height="30" height="30"/> | Source | airbyte/source-shortio:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/sources/shortio) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-shortio) | <small>`2fed2292-5586-480c-af92-9944e39fe12d`</small> |
| **Slack** | <img alt="Slack icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/slack.svg" height="30" height="30"/> | Source | airbyte/source-slack:0.1.23 | generally_available | [link](https://docs.airbyte.com/integrations/sources/slack) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-slack) | <small>`c2281cee-86f9-4a86-bb48-d23286b4c7bd`</small> |

1
docs/integrations/sources/sentry.md
View File

@@ -45,6 +45,7 @@ The Sentry source connector supports the following [sync modes](https://docs.air
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------|
| 0.1.12 | 2023-03-01 | [23619](https://github.com/airbytehq/airbyte/pull/23619) | Fix bug when `stream state` is `None` or any other bad value occurs |
| 0.1.11 | 2023-02-02 | [22303](https://github.com/airbytehq/airbyte/pull/22303) | Turn ON default AvailabilityStrategy |
| 0.1.10 | 2023-01-27 | [22041](https://github.com/airbytehq/airbyte/pull/22041) | Set `AvailabilityStrategy` for streams explicitly to `None` |
| 0.1.9 | 2022-12-20 | [21864](https://github.com/airbytehq/airbyte/pull/21864) | Add state persistence to incremental sync |