feat(source-github): add max_waiting_time to configuration (#38758)

Signed-off-by: Artem Inzhyyants <artem.inzhyyants@gmail.com>

airbyte-integrations/connectors/source-github/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
-  dockerImageTag: 1.7.4
+  dockerImageTag: 1.7.5
   dockerRepository: airbyte/source-github
   documentationUrl: https://docs.airbyte.com/integrations/sources/github
   githubIssueLabel: source-github

airbyte-integrations/connectors/source-github/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.7.4"
+version = "1.7.5"
 name = "source-github"
 description = "Source implementation for GitHub."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte-integrations/connectors/source-github/source_github/source.py

@@ -1,13 +1,12 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import logging
 from os import getenv
 from typing import Any, List, Mapping, MutableMapping, Optional, Tuple
 from urllib.parse import urlparse
 
-from airbyte_cdk import AirbyteLogger
-from airbyte_cdk.models import FailureType, SyncMode
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator
@@ -191,7 +190,7 @@ class SourceGithub(AbstractSource):
             )
         return user_message
 
-    def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
+    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
         config = self._validate_and_transform_config(config)
         try:
             authenticator = self._get_authenticator(config)
@@ -238,12 +237,13 @@ class SourceGithub(AbstractSource):
         # This parameter is deprecated and in future will be used sane default, page_size: 10
         page_size = config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM)
         access_token_type, _ = self.get_access_token(config)
-
+        max_waiting_time = config.get("max_waiting_time", 10) * 60
         organization_args = {
             "authenticator": authenticator,
             "organizations": organizations,
             "api_url": config.get("api_url"),
             "access_token_type": access_token_type,
+            "max_waiting_time": max_waiting_time,
         }
         start_date = config.get("start_date")
         organization_args_with_start_date = {**organization_args, "start_date": start_date}
@@ -254,6 +254,7 @@ class SourceGithub(AbstractSource):
             "repositories": repositories,
             "page_size_for_large_streams": page_size,
             "access_token_type": access_token_type,
+            "max_waiting_time": max_waiting_time,
         }
         repository_args_with_start_date = {**repository_args, "start_date": start_date}
 

airbyte-integrations/connectors/source-github/source_github/spec.json

@@ -131,6 +131,16 @@
         "description": "List of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.",
         "order": 4,
         "pattern_descriptor": "org/repo/branch1 org/repo/branch2"
+      },
+      "max_waiting_time": {
+        "type": "integer",
+        "title": "Max Waiting Time (in minutes)",
+        "examples": [10, 30, 60],
+        "default": 10,
+        "minimum": 1,
+        "maximum": 60,
+        "description": "Max Waiting Time for rate limit. Set higher value to wait till rate limits will be resetted to continue sync",
+        "order": 5
       }
     }
   },

airbyte-integrations/connectors/source-github/source_github/streams.py

@@ -42,7 +42,7 @@ class GithubStreamABC(HttpStream, ABC):
 
     def __init__(self, api_url: str = "https://api.github.com", access_token_type: str = "", **kwargs):
         if kwargs.get("authenticator"):
-            kwargs["authenticator"].max_time = self.max_time
+            kwargs["authenticator"].max_time = kwargs.pop("max_waiting_time", self.max_time)
         super().__init__(**kwargs)
 
         self.access_token_type = access_token_type