jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-23 21:03:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

🐛 Source Github: Remove optional parameter Accept for reaction's streams to fix error with 502 HTTP status code (#9492)

Browse Source 
* Source Github: Remove optional parameter Accept for reaction's streams to fix error with 502 HTTP status code
This commit is contained in:
Yevhenii
2022-01-17 15:21:35 +02:00
committed by GitHub
parent 25fb7e7fd7
commit cb6d9abcab
6 changed files with 22 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-integrations/connectors/source-github/Dockerfile
View File

@@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.2.10
LABEL io.airbyte.version=0.2.11
LABEL io.airbyte.name=airbyte/source-github

10
airbyte-integrations/connectors/source-github/source_github/streams.py
View File

@@ -52,10 +52,15 @@ class GithubStream(HttpStream, ABC):
def should_retry(self, response: requests.Response) -> bool:
# We don't call `super()` here because we have custom error handling and GitHub API sometimes returns strange
# errors. So in `read_records()` we have custom error handling which don't require to call `super()` here.
return response.headers.get("X-RateLimit-Remaining") == "0" or response.status_code in (
retry_flag = response.headers.get("X-RateLimit-Remaining") == "0" or response.status_code in (
requests.codes.SERVER_ERROR,
requests.codes.BAD_GATEWAY,
)
if retry_flag:
self.logger.info(
f"Rate limit handling for the response with {response.status_code} status code with message: {response.json()}"
)
return retry_flag
def backoff_time(self, response: requests.Response) -> Union[int, float]:
# This method is called if we run into the rate limit. GitHub limits requests to 5000 per hour and provides
@@ -765,9 +770,6 @@ class ReactionStream(GithubStream, ABC):
for parent_record in self._parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
yield {self.parent_key: parent_record[self.parent_key], "repository": stream_slice["repository"]}
def request_headers(self, **kwargs) -> Mapping[str, Any]:
return {"Accept": "application/vnd.github.squirrel-girl-preview+json"}
class CommitCommentReactions(ReactionStream):
"""

14
airbyte-integrations/connectors/source-github/unit_tests/test_stream.py
View File

@@ -20,7 +20,12 @@ def test_bad_gateway_retry(time_mock):
stream = PullRequestCommentReactions(**args)
stream_slice = {"repository": "test_repo", "id": "id"}
responses.add("GET", "https://api.github.com/repos/test_repo/pulls/comments/id/reactions", status=HTTPStatus.BAD_GATEWAY)
responses.add(
"GET",
"https://api.github.com/repos/test_repo/pulls/comments/id/reactions",
status=HTTPStatus.BAD_GATEWAY,
json={"message": "Bad request"},
)
with pytest.raises(BaseBackoffException):
list(stream.read_records(sync_mode="full_refresh", stream_slice=stream_slice))
@@ -28,7 +33,12 @@ def test_bad_gateway_retry(time_mock):
assert sleep_delays == DEFAULT_BACKOFF_DELAYS
time_mock.reset_mock()
responses.add("GET", "https://api.github.com/repos/test_repo/pulls/comments/id/reactions", status=HTTPStatus.INTERNAL_SERVER_ERROR)
responses.add(
"GET",
"https://api.github.com/repos/test_repo/pulls/comments/id/reactions",
status=HTTPStatus.INTERNAL_SERVER_ERROR,
json={"message": "Server Error"},
)
with pytest.raises(BaseBackoffException):
list(stream.read_records(sync_mode="full_refresh", stream_slice=stream_slice))