jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-23 21:03:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

Source surveycto incremental sync fix (#25522)

Browse Source 
* changes

* changes

* bump version

* fix eof

* remove datetime import

* update documentation

* auto-bump connector version

---------

Co-authored-by: siddhant <ssiddhant3030@gmail.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
This commit is contained in:
Mal Hancock
2023-04-25 14:36:18 -07:00
committed by GitHub
parent 0d9b7b4f7b
commit 0963ea4438
9 changed files with 17 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json
View File

@@ -24845,7 +24845,7 @@
"sourceDefinitionId": "dd4632f4-15e0-4649-9b71-41719fb1fdee",
"name": "SurveyCTO",
"dockerRepository": "airbyte/source-surveycto",
"dockerImageTag": "0.1.0",
"dockerImageTag": "0.1.1",
"documentationUrl": "https://docs.airbyte.com/integrations/sources/surveycto",
"icon": "surveycto.svg",
"sourceType": "api",
@@ -24879,7 +24879,7 @@
"form_id": {
"type": "array",
"title": "Form's Id",
"description": "Unique identifier for one of the forms",
"description": "Unique identifier for one of your forms",
"order": 3
},
"start_date": {
@@ -24889,7 +24889,7 @@
"examples": [ "Jan 09, 2022 00:00:00 AM" ],
"default": "Jan 09, 2022 00:00:00 AM",
"order": 4,
"pattern": "^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dic) [0-9]{2}, [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} (AM|PM)$"
"pattern": "^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{2}, [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} (AM|PM)$"
}
}
},

2
airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml
View File

@@ -2036,7 +2036,7 @@
- name: SurveyCTO
sourceDefinitionId: dd4632f4-15e0-4649-9b71-41719fb1fdee
dockerRepository: airbyte/source-surveycto
dockerImageTag: 0.1.0
dockerImageTag: 0.1.1
documentationUrl: https://docs.airbyte.com/integrations/sources/surveycto
icon: surveycto.svg
sourceType: api

6
airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml
View File

@@ -15246,7 +15246,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-surveycto:0.1.0"
- dockerImage: "airbyte/source-surveycto:0.1.1"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/surveycto"
connectionSpecification:
@@ -15278,7 +15278,7 @@
form_id:
type: "array"
title: "Form's Id"
description: "Unique identifier for one of the forms"
description: "Unique identifier for one of your forms"
order: 3
start_date:
type: "string"
@@ -15288,7 +15288,7 @@
- "Jan 09, 2022 00:00:00 AM"
default: "Jan 09, 2022 00:00:00 AM"
order: 4
pattern: "^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dic) [0-9]{2}, [0-9]{4}\
pattern: "^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{2}, [0-9]{4}\
\ [0-9]{2}:[0-9]{2}:[0-9]{2} (AM|PM)$"
supportsNormalization: false
supportsDBT: false

5
airbyte-integrations/connectors/source-surveycto/Dockerfile
View File

@@ -37,5 +37,6 @@ COPY source_surveycto ./source_surveycto
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/source-surveycto
LABEL io.airbyte.version=0.1.1
LABEL io.airbyte.name=airbyte/source-surveycto

7
airbyte-integrations/connectors/source-surveycto/source_surveycto/helpers.py
View File

@@ -3,7 +3,6 @@
#
import base64
from datetime import datetime
import requests
from bigquery_schema_generator.generate_schema import SchemaGenerator
@@ -41,12 +40,6 @@ class Helpers(object):
for data in response_json:
try:
dateformat_in = "%b %d, %Y %I:%M:%S %p"
dateformat_out = "%Y-%m-%dT%H:%M:%S+00:00"
data["starttime"] = datetime.strptime(data["starttime"], dateformat_in).strftime(dateformat_out)
data["endtime"] = datetime.strptime(data["endtime"], dateformat_in).strftime(dateformat_out)
data["CompletionDate"] = datetime.strptime(data["CompletionDate"], dateformat_in).strftime(dateformat_out)
data["SubmissionDate"] = datetime.strptime(data["SubmissionDate"], dateformat_in).strftime(dateformat_out)
yield data
except Exception as e:
raise e

18
airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py
View File

@@ -5,7 +5,6 @@
import base64
from abc import ABC
from datetime import datetime
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
import requests
@@ -47,22 +46,19 @@ class SurveyStream(HttpStream, ABC):
class SurveyctoStream(SurveyStream, IncrementalMixin):
primary_key = "KEY"
date_format_scto = "%b %d, %Y %H:%M:%S %p"
dateformat = "%Y-%m-%dT%H:%M:%S+00:00"
cursor_field = "CompletionDate"
cursor_field = "SubmissionDate"
_cursor_value = None
@property
def state(self) -> Mapping[str, Any]:
initial_date = datetime.strptime(self.start_date, self.date_format_scto)
if self._cursor_value:
return {self.cursor_field: self._cursor_value}
else:
return {self.cursor_field: initial_date}
return {self.cursor_field: self.start_date}
@state.setter
def state(self, value: Mapping[str, Any]):
self._cursor_value = datetime.strptime(value[self.cursor_field], self.dateformat)
self._cursor_value = value[self.cursor_field]
@property
def name(self) -> str:
@@ -78,7 +74,7 @@ class SurveyctoStream(SurveyStream, IncrementalMixin):
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None
) -> MutableMapping[str, Any]:
ix = self.state[self.cursor_field]
return {"date": ix.strftime(self.date_format_scto)}
return {"date": ix}
def request_headers(
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
@@ -99,12 +95,6 @@ class SurveyctoStream(SurveyStream, IncrementalMixin):
for data in self.response_json:
try:
dateformat_in = "%b %d, %Y %I:%M:%S %p"
dateformat_out = "%Y-%m-%dT%H:%M:%S+00:00"
data["starttime"] = datetime.strptime(data["starttime"], dateformat_in).strftime(dateformat_out)
data["endtime"] = datetime.strptime(data["endtime"], dateformat_in).strftime(dateformat_out)
data["CompletionDate"] = datetime.strptime(data["CompletionDate"], dateformat_in).strftime(dateformat_out)
data["SubmissionDate"] = datetime.strptime(data["SubmissionDate"], dateformat_in).strftime(dateformat_out)
yield data
except Exception as e:
msg = "Encountered an exception parsing schema"

2
airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml
View File

@@ -38,4 +38,4 @@ connectionSpecification:
- "Jan 09, 2022 00:00:00 AM"
default: "Jan 09, 2022 00:00:00 AM"
order: 4
pattern: ^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dic) [0-9]{2}, [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} (AM|PM)$
pattern: ^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{2}, [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} (AM|PM)$

2
connectors.md
View File

@@ -232,7 +232,7 @@
| **Statuspage** | <img alt="Statuspage icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/statuspage.svg" height="30" height="30"/> | Source | airbyte/source-statuspage:0.1.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/statuspage) | [connectors/source/statuspage](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/statuspage) | [source-statuspage](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-statuspage) | <small>`74cbd708-46c3-4512-9c93-abd5c3e9a94d`</small> |
| **Strava** | <img alt="Strava icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/strava.svg" height="30" height="30"/> | Source | airbyte/source-strava:0.1.4 | beta | [docs](https://docs.airbyte.com/integrations/sources/strava) | [connectors/source/strava](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/strava) | [source-strava](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-strava) | <small>`7a4327c4-315a-11ec-8d3d-0242ac130003`</small> |
| **Stripe** | <img alt="Stripe icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/stripe.svg" height="30" height="30"/> | Source | airbyte/source-stripe:3.4.1 | generally_available | [docs](https://docs.airbyte.com/integrations/sources/stripe) | [connectors/source/stripe](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/stripe) | [source-stripe](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-stripe) | <small>`e094cb9a-26de-4645-8761-65c0c425d1de`</small> |
| **SurveyCTO** | <img alt="SurveyCTO icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveycto.svg" height="30" height="30"/> | Source | airbyte/source-surveycto:0.1.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/surveycto) | [connectors/source/surveycto](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/surveycto) | [source-surveycto](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-surveycto) | <small>`dd4632f4-15e0-4649-9b71-41719fb1fdee`</small> |
| **SurveyCTO** | <img alt="SurveyCTO icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveycto.svg" height="30" height="30"/> | Source | airbyte/source-surveycto:0.1.1 | alpha | [docs](https://docs.airbyte.com/integrations/sources/surveycto) | [connectors/source/surveycto](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/surveycto) | [source-surveycto](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-surveycto) | <small>`dd4632f4-15e0-4649-9b71-41719fb1fdee`</small> |
| **SurveyMonkey** | <img alt="SurveyMonkey icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveymonkey.svg" height="30" height="30"/> | Source | airbyte/source-surveymonkey:0.1.16 | generally_available | [docs](https://docs.airbyte.com/integrations/sources/surveymonkey) | [connectors/source/surveymonkey](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/surveymonkey) | [source-surveymonkey](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-surveymonkey) | <small>`badc5925-0485-42be-8caa-b34096cb71b5`</small> |
| **SurveySparrow** | <img alt="SurveySparrow icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveysparrow.svg" height="30" height="30"/> | Source | airbyte/source-survey-sparrow:0.2.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/survey-sparrow) | [connectors/source/survey-sparrow](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/survey-sparrow) | [source-survey-sparrow](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-survey-sparrow) | <small>`4a4d887b-0f2d-4b33-ab7f-9b01b9072804`</small> |
| **TMDb** | <img alt="TMDb icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/tmdb.svg" height="30" height="30"/> | Source | airbyte/source-tmdb:0.1.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/tmdb) | [connectors/source/tmdb](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/tmdb) | [source-tmdb](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tmdb) | <small>`6240848f-f795-45eb-8f5e-c7542822fc03`</small> |

1
docs/integrations/sources/surveycto.md
View File

@@ -46,4 +46,5 @@ The SurveyCTO source connector supports the following streams:
## Changelog
| Version | Date | Pull Request | Subject |
| 0.1.1 | 2023-04-25 | [24784](https://github.com/airbytehq/airbyte/pull/24784) | Fix incremental sync |
| 0.1.0 | 2022-11-16 | [19371](https://github.com/airbytehq/airbyte/pull/19371) | SurveyCTO Source Connector |