diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index 7b85d4ae6f0..8f5a9b3e433 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.4.1 +LABEL io.airbyte.version=1.4.2 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-gitlab/metadata.yaml b/airbyte-integrations/connectors/source-gitlab/metadata.yaml index 9dd6539299e..e198fc9b315 100644 --- a/airbyte-integrations/connectors/source-gitlab/metadata.yaml +++ b/airbyte-integrations/connectors/source-gitlab/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 - dockerImageTag: 1.4.1 + dockerImageTag: 1.4.2 dockerRepository: airbyte/source-gitlab githubIssueLabel: source-gitlab icon: gitlab.svg diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py index ff83316bbc2..d2528c2ffd1 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py @@ -3,6 +3,7 @@ # +import os import pendulum from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -42,6 +43,7 @@ from .streams import ( Tags, Users, ) +from .utils import parse_url class SingleUseRefreshTokenGitlabOAuth2Authenticator(SingleUseRefreshTokenOauth2Authenticator): @@ -129,7 +131,16 @@ class SourceGitlab(AbstractSource): for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): yield from stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + @staticmethod + def _is_http_allowed() -> bool: + return os.environ.get("DEPLOYMENT_MODE", "").upper() != "CLOUD" + def check_connection(self, logger, config) -> Tuple[bool, any]: + is_valid, scheme, _ = parse_url(config["api_url"]) + if not is_valid: + return False, "Invalid API resource locator." + if scheme == "http" and not self._is_http_allowed(): + return False, "Http scheme is not allowed in this environment. Please use `https` instead." try: projects = self._projects_stream(config) for stream_slice in projects.stream_slices(sync_mode=SyncMode.full_refresh): diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json index 0e5dd6ff8da..e3ec9fa01cd 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json @@ -75,7 +75,7 @@ }, "api_url": { "type": "string", - "examples": ["gitlab.com"], + "examples": ["gitlab.com", "https://gitlab.com", "https://gitlab.company.org"], "title": "API URL", "default": "gitlab.com", "description": "Please enter your basic URL from GitLab instance.", diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index 5ae40fe0ce4..de655d03e97 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -5,7 +5,6 @@ import datetime from abc import ABC from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import urlparse import pendulum import requests @@ -14,6 +13,8 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrate from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream +from .utils import parse_url + class GitlabStream(HttpStream, ABC): primary_key = "id" @@ -53,12 +54,7 @@ class GitlabStream(HttpStream, ABC): @property def url_base(self) -> str: - parse_result = urlparse(self.api_url) - # Default scheme to "https" if URL doesn't contain - scheme = parse_result.scheme if parse_result.scheme else "https" - # hostname without a scheme will result in `path` attribute - # Use path if netloc is not detected - host = parse_result.netloc if parse_result.netloc else parse_result.path + _, scheme, host = parse_url(self.api_url) return f"{scheme}://{host}/api/v4/" @property diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/utils.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/utils.py new file mode 100644 index 00000000000..0b5605260a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/utils.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Tuple + + +def parse_url(url: str) -> Tuple[bool, str, str]: + parts = url.split("://") + if len(parts) > 1: + scheme, url = parts + else: + scheme = "https" + if scheme not in ("http", "https"): + return False, "", "" + parts = url.split("/", 1) + if len(parts) > 1: + return False, "", "" + host, *_ = parts + return True, scheme, host diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py index 15c35e4764b..d250f548925 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py @@ -5,7 +5,7 @@ import pytest -@pytest.fixture(params=["gitlab.com", "https://gitlab.com", "https://gitlab.com/api/v4"]) +@pytest.fixture(params=["gitlab.com", "http://gitlab.com", "https://gitlab.com"]) def config(request): return { "start_date": "2021-01-01T00:00:00Z", diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py index e736b8044bc..1c2b2a54757 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -3,8 +3,8 @@ # import logging -import pytest +import pytest from source_gitlab import SourceGitlab from source_gitlab.streams import GitlabStream @@ -41,9 +41,31 @@ def test_connection_success(config, requests_mock, url_mocks): assert (status, msg) == (True, None) -def test_connection_fail(config, mocker, requests_mock): +def test_connection_fail_due_to_api_error(config, mocker, requests_mock): mocker.patch("time.sleep") requests_mock.get("/api/v4/groups", status_code=500) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config) assert status is False, msg.startswith('Unable to connect to Gitlab API with the provided credentials - "DefaultBackoffException"') + + +@pytest.mark.parametrize( + "api_url, deployment_env, expected_message", + ( + ("http://gitlab.my.company.org", "CLOUD", "Http scheme is not allowed in this environment. Please use `https` instead."), + ("https://gitlab.com/api/v4", "CLOUD", "Invalid API resource locator.") + ) +) +def test_connection_fail_due_to_config_error(mocker, api_url, deployment_env, expected_message): + mocker.patch("os.environ", {"DEPLOYMENT_MODE": deployment_env}) + source = SourceGitlab() + config = { + "start_date": "2021-01-01T00:00:00Z", + "api_url": api_url, + "credentials": { + "auth_type": "access_token", + "access_token": "token" + } + } + status, msg = source.check_connection(logging.getLogger(), config) + assert (status, msg) == (False, expected_message) diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index 4dc445b7054..99cbac18bd3 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -105,6 +105,7 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------| +| 1.4.2 | 2023-06-15 | [27346](https://github.com/airbytehq/airbyte/pull/27346) | Partially revert changes made in version 1.0.4, disallow http calls in cloud. | | 1.4.1 | 2023-06-13 | [27351](https://github.com/airbytehq/airbyte/pull/27351) | Fix OAuth token expiry date. | | 1.4.0 | 2023-06-12 | [27234](https://github.com/airbytehq/airbyte/pull/27234) | Skip stream slices on 403/404 errors, do not fail syncs. | | 1.3.1 | 2023-06-08 | [27147](https://github.com/airbytehq/airbyte/pull/27147) | Improve connectivity check for connections with no projects/groups |