jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Source surveycto incremental sync fix (#25522)

Browse Source 
* changes

* changes

* bump version

* fix eof

* remove datetime import

* update documentation

* auto-bump connector version

---------

Co-authored-by: siddhant <ssiddhant3030@gmail.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
This commit is contained in:
Mal Hancock
2023-04-25 14:36:18 -07:00
committed by GitHub
parent 0d9b7b4f7b
commit 0963ea4438
9 changed files with 17 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
airbyte-integrations/connectors/source-surveycto/Dockerfile
View File

@@ -37,5 +37,6 @@ COPY source_surveycto ./source_surveycto
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/source-surveycto
LABEL io.airbyte.version=0.1.1
LABEL io.airbyte.name=airbyte/source-surveycto

7
airbyte-integrations/connectors/source-surveycto/source_surveycto/helpers.py
View File

@@ -3,7 +3,6 @@
#
import base64
from datetime import datetime
import requests
from bigquery_schema_generator.generate_schema import SchemaGenerator
@@ -41,12 +40,6 @@ class Helpers(object):
for data in response_json:
try:
dateformat_in = "%b %d, %Y %I:%M:%S %p"
dateformat_out = "%Y-%m-%dT%H:%M:%S+00:00"
data["starttime"] = datetime.strptime(data["starttime"], dateformat_in).strftime(dateformat_out)
data["endtime"] = datetime.strptime(data["endtime"], dateformat_in).strftime(dateformat_out)
data["CompletionDate"] = datetime.strptime(data["CompletionDate"], dateformat_in).strftime(dateformat_out)
data["SubmissionDate"] = datetime.strptime(data["SubmissionDate"], dateformat_in).strftime(dateformat_out)
yield data
except Exception as e:
raise e

18
airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py
View File

@@ -5,7 +5,6 @@
import base64
from abc import ABC
from datetime import datetime
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
import requests
@@ -47,22 +46,19 @@ class SurveyStream(HttpStream, ABC):
class SurveyctoStream(SurveyStream, IncrementalMixin):
primary_key = "KEY"
date_format_scto = "%b %d, %Y %H:%M:%S %p"
dateformat = "%Y-%m-%dT%H:%M:%S+00:00"
cursor_field = "CompletionDate"
cursor_field = "SubmissionDate"
_cursor_value = None
@property
def state(self) -> Mapping[str, Any]:
initial_date = datetime.strptime(self.start_date, self.date_format_scto)
if self._cursor_value:
return {self.cursor_field: self._cursor_value}
else:
return {self.cursor_field: initial_date}
return {self.cursor_field: self.start_date}
@state.setter
def state(self, value: Mapping[str, Any]):
self._cursor_value = datetime.strptime(value[self.cursor_field], self.dateformat)
self._cursor_value = value[self.cursor_field]
@property
def name(self) -> str:
@@ -78,7 +74,7 @@ class SurveyctoStream(SurveyStream, IncrementalMixin):
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None
) -> MutableMapping[str, Any]:
ix = self.state[self.cursor_field]
return {"date": ix.strftime(self.date_format_scto)}
return {"date": ix}
def request_headers(
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
@@ -99,12 +95,6 @@ class SurveyctoStream(SurveyStream, IncrementalMixin):
for data in self.response_json:
try:
dateformat_in = "%b %d, %Y %I:%M:%S %p"
dateformat_out = "%Y-%m-%dT%H:%M:%S+00:00"
data["starttime"] = datetime.strptime(data["starttime"], dateformat_in).strftime(dateformat_out)
data["endtime"] = datetime.strptime(data["endtime"], dateformat_in).strftime(dateformat_out)
data["CompletionDate"] = datetime.strptime(data["CompletionDate"], dateformat_in).strftime(dateformat_out)
data["SubmissionDate"] = datetime.strptime(data["SubmissionDate"], dateformat_in).strftime(dateformat_out)
yield data
except Exception as e:
msg = "Encountered an exception parsing schema"

2
airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml
View File

@@ -38,4 +38,4 @@ connectionSpecification:
- "Jan 09, 2022 00:00:00 AM"
default: "Jan 09, 2022 00:00:00 AM"
order: 4
pattern: ^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dic) [0-9]{2}, [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} (AM|PM)$
pattern: ^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{2}, [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} (AM|PM)$