jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Airbyte CDK: handle private network exception as config error (#33751)

Browse Source
This commit is contained in:
Artem Inzhyyants
2024-01-10 15:20:40 +01:00
committed by GitHub
parent 212f960294
commit 9c6aea19cd
4 changed files with 16 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
airbyte-cdk/python/airbyte_cdk/entrypoint.py
View File

@@ -26,6 +26,7 @@ from airbyte_cdk.utils import is_cloud_environment
from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
from airbyte_protocol.models import FailureType
from requests import PreparedRequest, Response, Session
logger = init_logger("airbyte")
@@ -236,9 +237,10 @@ def _init_internal_request_filter() -> None:
try:
is_private = _is_private_url(parsed_url.hostname, parsed_url.port) # type: ignore [arg-type]
if is_private:
raise ValueError(
"Invalid URL endpoint: The endpoint that data is being requested from belongs to a private network. Source "
+ "connectors only support requesting data from public API endpoints."
raise AirbyteTracedException(
internal_message=f"Invalid URL endpoint: `{parsed_url.hostname!r}` belongs to a private network",
failure_type=FailureType.config_error,
message="Invalid URL endpoint: The endpoint that data is being requested from belongs to a private network. Source connectors only support requesting data from public API endpoints.",
)
except socket.gaierror as exception:
# This is a special case where the developer specifies an IP address string that is not formatted correctly like trailing

6
airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py
View File

@@ -769,8 +769,8 @@ def test_read_source_single_page_single_slice(mock_http_stream):
"deployment_mode, url_base, expected_error",
[
pytest.param("CLOUD", "https://airbyte.com/api/v1/characters", None, id="test_cloud_read_with_public_endpoint"),
pytest.param("CLOUD", "https://10.0.27.27", "ValueError", id="test_cloud_read_with_private_endpoint"),
pytest.param("CLOUD", "https://localhost:80/api/v1/cast", "ValueError", id="test_cloud_read_with_localhost"),
pytest.param("CLOUD", "https://10.0.27.27", "AirbyteTracedException", id="test_cloud_read_with_private_endpoint"),
pytest.param("CLOUD", "https://localhost:80/api/v1/cast", "AirbyteTracedException", id="test_cloud_read_with_localhost"),
pytest.param("CLOUD", "http://unsecured.protocol/api/v1", "InvalidSchema", id="test_cloud_read_with_unsecured_endpoint"),
pytest.param("CLOUD", "https://domainwithoutextension", "Invalid URL", id="test_cloud_read_with_invalid_url_endpoint"),
pytest.param("OSS", "https://airbyte.com/api/v1/", None, id="test_oss_read_with_public_endpoint"),
@@ -820,7 +820,7 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error
"deployment_mode, token_url, expected_error",
[
pytest.param("CLOUD", "https://airbyte.com/tokens/bearer", None, id="test_cloud_read_with_public_endpoint"),
pytest.param("CLOUD", "https://10.0.27.27/tokens/bearer", "ValueError", id="test_cloud_read_with_private_endpoint"),
pytest.param("CLOUD", "https://10.0.27.27/tokens/bearer", "AirbyteTracedException", id="test_cloud_read_with_private_endpoint"),
pytest.param("CLOUD", "http://unsecured.protocol/tokens/bearer", "InvalidSchema", id="test_cloud_read_with_unsecured_endpoint"),
pytest.param("CLOUD", "https://domainwithoutextension", "Invalid URL", id="test_cloud_read_with_invalid_url_endpoint"),
pytest.param("OSS", "https://airbyte.com/tokens/bearer", None, id="test_oss_read_with_public_endpoint"),

7
airbyte-cdk/python/unit_tests/sources/test_integration_source.py
View File

@@ -9,6 +9,7 @@ from unittest.mock import patch
import pytest
import requests
from airbyte_cdk.entrypoint import launch
from airbyte_cdk.utils import AirbyteTracedException
from unit_tests.sources.fixtures.source_test_fixture import (
HttpTestStream,
SourceFixtureOauthAuthenticator,
@@ -22,8 +23,8 @@ from unit_tests.sources.fixtures.source_test_fixture import (
[
pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"),
pytest.param("CLOUD", "https://localhost:80/api/v1/", [], ValueError, id="test_cloud_read_with_localhost"),
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
pytest.param("CLOUD", "https://localhost:80/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_localhost"),
pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
],
@@ -47,7 +48,7 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
[
pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"),
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
],

7
airbyte-cdk/python/unit_tests/test_entrypoint.py
View File

@@ -28,6 +28,7 @@ from airbyte_cdk.models import (
Type,
)
from airbyte_cdk.sources import Source
from airbyte_cdk.utils import AirbyteTracedException
class MockSource(Source):
@@ -276,12 +277,12 @@ def test_invalid_command(entrypoint: AirbyteEntrypoint, config_mock):
"deployment_mode, url, expected_error",
[
pytest.param("CLOUD", "https://airbyte.com", None, id="test_cloud_public_endpoint_is_successful"),
pytest.param("CLOUD", "https://192.168.27.30", ValueError, id="test_cloud_private_ip_address_is_rejected"),
pytest.param("CLOUD", "https://localhost:8080/api/v1/cast", ValueError, id="test_cloud_private_endpoint_is_rejected"),
pytest.param("CLOUD", "https://192.168.27.30", AirbyteTracedException, id="test_cloud_private_ip_address_is_rejected"),
pytest.param("CLOUD", "https://localhost:8080/api/v1/cast", AirbyteTracedException, id="test_cloud_private_endpoint_is_rejected"),
pytest.param("CLOUD", "http://past.lives.net/api/v1/inyun", ValueError, id="test_cloud_unsecured_endpoint_is_rejected"),
pytest.param("CLOUD", "https://not:very/cash:443.money", ValueError, id="test_cloud_invalid_url_format"),
pytest.param("CLOUD", "https://192.168.27.30 ", ValueError, id="test_cloud_incorrect_ip_format_is_rejected"),
pytest.param("cloud", "https://192.168.27.30", ValueError, id="test_case_insensitive_cloud_environment_variable"),
pytest.param("cloud", "https://192.168.27.30", AirbyteTracedException, id="test_case_insensitive_cloud_environment_variable"),
pytest.param("OSS", "https://airbyte.com", None, id="test_oss_public_endpoint_is_successful"),
pytest.param("OSS", "https://192.168.27.30", None, id="test_oss_private_endpoint_is_successful"),
pytest.param("OSS", "https://localhost:8080/api/v1/cast", None, id="test_oss_private_endpoint_is_successful"),