1
0
mirror of synced 2025-12-20 10:32:35 -05:00

Source Github: implement client-side throttling of requests (#25793)

Signed-off-by: Serhii Chvaliuk <grubberr@gmail.com>
This commit is contained in:
Serhii Chvaliuk
2023-05-17 22:18:37 +03:00
committed by GitHub
parent 5575d86f68
commit 30d8bb4133
9 changed files with 129 additions and 6 deletions

View File

@@ -11,6 +11,7 @@ from airbyte_cdk.sources import AbstractSource
from airbyte_cdk.sources.streams import Stream
from airbyte_cdk.sources.streams.http.auth import MultipleTokenAuthenticator
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter
from . import constants
from .streams import (
@@ -158,6 +159,13 @@ class SourceGithub(AbstractSource):
def _get_authenticator(self, config: Mapping[str, Any]):
_, token = self.get_access_token(config)
tokens = [t.strip() for t in token.split(constants.TOKEN_SEPARATOR)]
requests_per_hour = config.get("requests_per_hour")
if requests_per_hour:
return MultipleTokenAuthenticatorWithRateLimiter(
tokens=tokens,
auth_method="token",
requests_per_hour=requests_per_hour,
)
return MultipleTokenAuthenticator(tokens=tokens, auth_method="token")
@staticmethod