🐛Source Github: remove BAD_GATEWAY code from backoff_time (#9999)

* remove BAD_GATEWAY code from backoff_time * bump version * bump version * add unit test * bump version
2025-12-25 02:09:19 -05:00 · 2022-02-09 11:10:16 +01:00
parent 05873803c9
commit 8749c7ab2c
6 changed files with 52 additions and 46 deletions
--- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
+++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -238,7 +238,7 @@
 - name: GitHub
  sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
  dockerRepository: airbyte/source-github
-  dockerImageTag: 0.2.16
+  dockerImageTag: 0.2.17
  documentationUrl: https://docs.airbyte.io/integrations/sources/github
  icon: github.svg
  sourceType: api
--- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml
+++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2144,7 +2144,7 @@
    supportsNormalization: false
    supportsDBT: false
    supported_destination_sync_modes: []
- dockerImage: "airbyte/source-github:0.2.16"
+- dockerImage: "airbyte/source-github:0.2.17"
  spec:
    documentationUrl: "https://docs.airbyte.io/integrations/sources/github"
    connectionSpecification:
--- a/airbyte-integrations/connectors/source-github/Dockerfile
+++ b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

-LABEL io.airbyte.version=0.2.16
+LABEL io.airbyte.version=0.2.17
 LABEL io.airbyte.name=airbyte/source-github
--- a/airbyte-integrations/connectors/source-github/source_github/streams.py
+++ b/airbyte-integrations/connectors/source-github/source_github/streams.py
@@ -72,7 +72,7 @@ class GithubStream(HttpStream, ABC):
        # `X-RateLimit-Reset` header which contains time when this hour will be finished and limits will be reset so
        # we again could have 5000 per another hour.

-        if response.status_code in [requests.codes.BAD_GATEWAY, requests.codes.SERVER_ERROR]:
+        if response.status_code == requests.codes.SERVER_ERROR:
            return None

        reset_time = response.headers.get("X-RateLimit-Reset")
--- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py
+++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py
@@ -3,6 +3,7 @@
 #

 from http import HTTPStatus
+from unittest.mock import MagicMock
 from unittest.mock import patch

 import pytest
@@ -15,23 +16,11 @@ DEFAULT_BACKOFF_DELAYS = [5, 10, 20, 40, 80]

@responses.activate
@patch("time.sleep")
-def test_bad_gateway_retry(time_mock):
+def test_internal_server_error_retry(time_mock):
    args = {"authenticator": None, "repositories": ["test_repo"], "start_date": "start_date", "page_size_for_large_streams": 30}
    stream = PullRequestCommentReactions(**args)
    stream_slice = {"repository": "test_repo", "id": "id"}

-    responses.add(
-        "GET",
-        "https://api.github.com/repos/test_repo/pulls/comments/id/reactions",
-        status=HTTPStatus.BAD_GATEWAY,
-        json={"message": "Bad request"},
-    )
-    with pytest.raises(BaseBackoffException):
-        list(stream.read_records(sync_mode="full_refresh", stream_slice=stream_slice))
-
-    sleep_delays = [delay[0][0] for delay in time_mock.call_args_list]
-    assert sleep_delays == DEFAULT_BACKOFF_DELAYS
-
    time_mock.reset_mock()
    responses.add(
        "GET",
@@ -44,3 +33,18 @@ def test_bad_gateway_retry(time_mock):

    sleep_delays = [delay[0][0] for delay in time_mock.call_args_list]
    assert sleep_delays == DEFAULT_BACKOFF_DELAYS
+
+
+@pytest.mark.parametrize(
+    ("http_status", "response_text", "expected_backoff_time"),
+    [
+        (HTTPStatus.BAD_GATEWAY, "", 60),
+    ],
+)
+def test_backoff_time(http_status, response_text, expected_backoff_time):
+    response_mock = MagicMock()
+    response_mock.status_code = http_status
+    response_mock.text = response_text
+    args = {"authenticator": None, "repositories": ["test_repo"], "start_date": "start_date", "page_size_for_large_streams": 30}
+    stream = PullRequestCommentReactions(**args)
+    assert stream.backoff_time(response_mock) == expected_backoff_time
--- a/docs/integrations/sources/github.md
+++ b/docs/integrations/sources/github.md
@@ -90,33 +90,35 @@ Your token should have at least the `repo` scope. Depending on which streams you

 ## Changelog

-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.2.15 | 2021-01-26 | [9802](https://github.com/airbytehq/airbyte/pull/9802) | Add missing fields for auto_merge in pull request stream |
-| 0.2.14 | 2021-01-21 | [9664](https://github.com/airbytehq/airbyte/pull/9664) | Add custom pagination size for large streams |
-| 0.2.13 | 2021-01-20 | [9619](https://github.com/airbytehq/airbyte/pull/9619) | Fix logging for function `should_retry` |
-| 0.2.11 | 2021-01-17 | [9492](https://github.com/airbytehq/airbyte/pull/9492) | Remove optional parameter `Accept` for reaction`s streams to fix error with 502 HTTP status code in response |
-| 0.2.10 | 2021-01-03 | [7250](https://github.com/airbytehq/airbyte/pull/7250) | Use CDK caching and convert PR-related streams to incremental |
-| 0.2.9 | 2021-12-29 | [9179](https://github.com/airbytehq/airbyte/pull/9179) | Use default retry delays on server error responses |
-| 0.2.8 | 2021-12-07 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description |
-| 0.2.7 | 2021-12-06 | [8518](https://github.com/airbytehq/airbyte/pull/8518) | Add connection retry with Github |
-| 0.2.6 | 2021-11-24 | [8030](https://github.com/airbytehq/airbyte/pull/8030) | Support start date property for PullRequestStats and Reviews streams |
-| 0.2.5 | 2021-11-21 | [8170](https://github.com/airbytehq/airbyte/pull/8170) | Fix slow check connection for organizations with a lot of repos |
-| 0.2.4 | 2021-11-11 | [7856](https://github.com/airbytehq/airbyte/pull/7856) | Resolve $ref fields in some stream schemas |
-| 0.2.3 | 2021-10-06 | [6833](https://github.com/airbytehq/airbyte/pull/6833) | Fix config backward compatability |
-| 0.2.2 | 2021-10-05 | [6761](https://github.com/airbytehq/airbyte/pull/6761) | Add oauth worflow specification |
-| 0.2.1 | 2021-09-22 | [6223](https://github.com/airbytehq/airbyte/pull/6223) | Add option to pull commits from user-specified branches |
-| 0.2.0 | 2021-09-19 | [5898](https://github.com/airbytehq/airbyte/pull/5898) and [6227](https://github.com/airbytehq/airbyte/pull/6227) | Don't minimize any output fields & add better error handling |
-| 0.1.11 | 2021-09-15 | [5949](https://github.com/airbytehq/airbyte/pull/5949) | Add caching for all streams |
-| 0.1.10 | 2021-09-09 | [5860](https://github.com/airbytehq/airbyte/pull/5860) | Add reaction streams |
-| 0.1.9 | 2021-09-02 | [5788](https://github.com/airbytehq/airbyte/pull/5788) | Handling empty repository, check method using RepositoryStats stream |
-| 0.1.8 | 2021-09-01 | [5757](https://github.com/airbytehq/airbyte/pull/5757) | Add more streams |
-| 0.1.7 | 2021-08-27 | [5696](https://github.com/airbytehq/airbyte/pull/5696) | Handle negative backoff values |
-| 0.1.6 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5223) | Add MultipleTokenAuthenticator |
-| 0.1.5 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5456) | Fix set up validation |
-| 0.1.4 | 2021-08-13 | [5136](https://github.com/airbytehq/airbyte/pull/5136) | Support syncing multiple repositories/organizations |
-| 0.1.3 | 2021-08-03 | [5156](https://github.com/airbytehq/airbyte/pull/5156) | Extended existing schemas with `users` property for certain streams |
-| 0.1.2 | 2021-07-13 | [4708](https://github.com/airbytehq/airbyte/pull/4708) | Fix bug with IssueEvents stream and add handling for rate limiting |
-| 0.1.1 | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream |
-| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub |
+| Version | Date       | Pull Request | Subject                                                                                                      |
+|:--------|:-----------| :--- |:-------------------------------------------------------------------------------------------------------------|
+| 0.2.17  | 2021-02-02 | [9999](https://github.com/airbytehq/airbyte/pull/9999) | Remove BAD_GATEWAY code from backoff_time                                                                    |
+| 0.2.16  | 2021-02-02 | [9868](https://github.com/airbytehq/airbyte/pull/9868) | Add log message for streams that are restricted for OAuth. Update oauth scopes.                              |
+| 0.2.15  | 2021-01-26 | [9802](https://github.com/airbytehq/airbyte/pull/9802) | Add missing fields for auto_merge in pull request stream                                                     |
+| 0.2.14  | 2021-01-21 | [9664](https://github.com/airbytehq/airbyte/pull/9664) | Add custom pagination size for large streams                                                                 |
+| 0.2.13  | 2021-01-20 | [9619](https://github.com/airbytehq/airbyte/pull/9619) | Fix logging for function `should_retry`                                                                      |
+| 0.2.11  | 2021-01-17 | [9492](https://github.com/airbytehq/airbyte/pull/9492) | Remove optional parameter `Accept` for reaction`s streams to fix error with 502 HTTP status code in response |
+| 0.2.10  | 2021-01-03 | [7250](https://github.com/airbytehq/airbyte/pull/7250) | Use CDK caching and convert PR-related streams to incremental                                                |
+| 0.2.9   | 2021-12-29 | [9179](https://github.com/airbytehq/airbyte/pull/9179) | Use default retry delays on server error responses                                                           |
+| 0.2.8   | 2021-12-07 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description                                                                    |
+| 0.2.7   | 2021-12-06 | [8518](https://github.com/airbytehq/airbyte/pull/8518) | Add connection retry with Github                                                                             |
+| 0.2.6   | 2021-11-24 | [8030](https://github.com/airbytehq/airbyte/pull/8030) | Support start date property for PullRequestStats and Reviews streams                                         |
+| 0.2.5   | 2021-11-21 | [8170](https://github.com/airbytehq/airbyte/pull/8170) | Fix slow check connection for organizations with a lot of repos                                              |
+| 0.2.4   | 2021-11-11 | [7856](https://github.com/airbytehq/airbyte/pull/7856) | Resolve $ref fields in some stream schemas                                                                   |
+| 0.2.3   | 2021-10-06 | [6833](https://github.com/airbytehq/airbyte/pull/6833) | Fix config backward compatability                                                                            |
+| 0.2.2   | 2021-10-05 | [6761](https://github.com/airbytehq/airbyte/pull/6761) | Add oauth worflow specification                                                                              |
+| 0.2.1   | 2021-09-22 | [6223](https://github.com/airbytehq/airbyte/pull/6223) | Add option to pull commits from user-specified branches                                                      |
+| 0.2.0   | 2021-09-19 | [5898](https://github.com/airbytehq/airbyte/pull/5898) and [6227](https://github.com/airbytehq/airbyte/pull/6227) | Don't minimize any output fields & add better error handling                                                 |
+| 0.1.11  | 2021-09-15 | [5949](https://github.com/airbytehq/airbyte/pull/5949) | Add caching for all streams                                                                                  |
+| 0.1.10  | 2021-09-09 | [5860](https://github.com/airbytehq/airbyte/pull/5860) | Add reaction streams                                                                                         |
+| 0.1.9   | 2021-09-02 | [5788](https://github.com/airbytehq/airbyte/pull/5788) | Handling empty repository, check method using RepositoryStats stream                                         |
+| 0.1.8   | 2021-09-01 | [5757](https://github.com/airbytehq/airbyte/pull/5757) | Add more streams                                                                                             |
+| 0.1.7   | 2021-08-27 | [5696](https://github.com/airbytehq/airbyte/pull/5696) | Handle negative backoff values                                                                               |
+| 0.1.6   | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5223) | Add MultipleTokenAuthenticator                                                                               |
+| 0.1.5   | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5456) | Fix set up validation                                                                                        |
+| 0.1.4   | 2021-08-13 | [5136](https://github.com/airbytehq/airbyte/pull/5136) | Support syncing multiple repositories/organizations                                                          |
+| 0.1.3   | 2021-08-03 | [5156](https://github.com/airbytehq/airbyte/pull/5156) | Extended existing schemas with `users` property for certain streams                                          |
+| 0.1.2   | 2021-07-13 | [4708](https://github.com/airbytehq/airbyte/pull/4708) | Fix bug with IssueEvents stream and add handling for rate limiting                                           |
+| 0.1.1   | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream                                                                      |
+| 0.1.0   | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub                                                                                           |