jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Source Gitlab: fix EpicIssues stream (#20348)

Browse Source 
* #12900 source Gitlab: fix EpicIssues stream

* #12900 source gitlab: upd changelog

* auto-bump connector version

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
This commit is contained in:
Denys Davydov
2022-12-11 22:07:05 +02:00
committed by GitHub
parent 0192cb343d
commit 1fce890967
11 changed files with 186 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
airbyte-integrations/connectors/source-gitlab/Dockerfile
View File

@@ -6,12 +6,12 @@ RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
WORKDIR /airbyte/integration_code
COPY source_gitlab ./source_gitlab
COPY main.py ./
COPY setup.py ./
RUN pip install .
COPY source_gitlab ./source_gitlab
COPY main.py ./
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.8
LABEL io.airbyte.version=0.1.9
LABEL io.airbyte.name=airbyte/source-gitlab

6
airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml
View File

@@ -13,12 +13,14 @@ tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
empty_streams: ["group_issue_boards"]
expect_records:
path: "integration_tests/expected_records.txt"
# We cannot use these tests for testing Incremental, since for Gitlab the State is saved for each Project separately,
# and the Acceptance Tests at this stage do not support this functionality.
# Therefore, we hardcode the cursor_paths for our config.
incremental:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
- config_path: "secrets/config_with_ids.json"
configured_catalog_path: "integration_tests/incremental_catalog.json"
future_state_path: "integration_tests/abnormal_state.json"
cursor_paths:
commits: ["25157276", "created_at"]

60
airbyte-integrations/connectors/source-gitlab/integration_tests/abnormal_state.json
View File

@@ -1,22 +1,54 @@
{
"commits": {
"25157276": {
"created_at": "2121-03-18T12:51:05+00:00"
[
{
"type": "STREAM",
"stream": {
"stream_state": {
"25157276": {
"created_at": "2121-03-18T12:51:05+00:00"
}
},
"stream_descriptor": {
"name": "commits"
}
}
},
"issues": {
"25157276": {
"updated_at": "2121-03-15T16:08:06.041000+00:00"
{
"type": "STREAM",
"stream": {
"stream_state": {
"25157276": {
"updated_at": "2121-03-15T16:08:06.041000+00:00"
}
},
"stream_descriptor": {
"name": "issues"
}
}
},
"merge_requests": {
"25157276": {
"updated_at": "2121-03-18T12:51:06.319000+00:00"
{
"type": "STREAM",
"stream": {
"stream_state": {
"25157276": {
"updated_at": "2121-03-18T12:51:06.319000+00:00"
}
},
"stream_descriptor": {
"name": "merge_requests"
}
}
},
"pipelines": {
"25157276": {
"updated_at": "2121-03-18T12:51:52.007000+00:00"
{
"type": "STREAM",
"stream": {
"stream_state": {
"25157276": {
"updated_at": "2121-03-18T12:51:52.007000+00:00"
}
},
"stream_descriptor": {
"name": "pipelines"
}
}
}
}
]

71
airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json
View File

@@ -8,7 +8,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -18,7 +19,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -28,7 +30,8 @@
"source_defined_primary_key": [["name"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["name"]]
},
{
"stream": {
@@ -41,7 +44,8 @@
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["created_at"]
"cursor_field": ["created_at"],
"primary_key": [["id"]]
},
{
"stream": {
@@ -51,7 +55,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -64,7 +69,8 @@
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
"cursor_field": ["updated_at"],
"primary_key": [["id"]]
},
{
"stream": {
@@ -74,7 +80,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -84,7 +91,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -94,7 +102,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -104,7 +113,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -114,7 +124,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -124,7 +135,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -134,7 +146,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -147,7 +160,8 @@
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
"cursor_field": ["updated_at"],
"primary_key": [["id"]]
},
{
"stream": {
@@ -157,7 +171,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -167,7 +182,8 @@
"source_defined_primary_key": [["name"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["name"]]
},
{
"stream": {
@@ -177,7 +193,8 @@
"source_defined_primary_key": [["name"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["name"]]
},
{
"stream": {
@@ -190,7 +207,8 @@
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
"cursor_field": ["updated_at"],
"primary_key": [["id"]]
},
{
"stream": {
@@ -200,7 +218,8 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
@@ -210,7 +229,19 @@
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["id"]]
},
{
"stream": {
"name": "epic_issues",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["epic_issue_id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite",
"primary_key": [["epic_issue_id"]]
}
]
}

1
airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt Normal file
View File

@@ -0,0 +1 @@
{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "assignees": [], "type": "ISSUE", "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "<strong>19</strong> days remaining"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1670761695320}

60
airbyte-integrations/connectors/source-gitlab/integration_tests/incremental_catalog.json Normal file
View File

@@ -0,0 +1,60 @@
{
"streams": [
{
"stream": {
"name": "commits",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["created_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["created_at"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "issues",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "merge_requests",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "pipelines",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["updated_at"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append",
"cursor_field": ["updated_at"],
"primary_key": [["id"]]
}
]
}

4
airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_milestones.json
View File

@@ -30,11 +30,11 @@
},
"due_date": {
"type": ["null", "string"],
"format": "date-time"
"format": "date"
},
"start_date": {
"type": ["null", "string"],
"format": "date-time"
"format": "date"
},
"expired": {
"type": ["null", "boolean"]

20
airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py
View File

@@ -14,6 +14,7 @@ from airbyte_cdk.sources.streams.http import HttpStream
class GitlabStream(HttpStream, ABC):
primary_key = "id"
raise_on_http_errors = True
stream_base_params = {}
flatten_id_keys = []
flatten_list_keys = []
@@ -40,6 +41,17 @@ class GitlabStream(HttpStream, ABC):
def url_base(self) -> str:
return f"https://{self.api_url}/api/v4/"
def should_retry(self, response: requests.Response) -> bool:
# Gitlab API returns a 403 response in case a feature is disabled in a project (pipelines/jobs for instance).
if response.status_code == 403:
setattr(self, "raise_on_http_errors", False)
self.logger.warning(
f"Got 403 error when accessing URL {response.request.url}."
f" Very likely the feature is disabled for this project and/or group. Please double check it, or report a bug otherwise."
)
return False
return super().should_retry(response)
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
response_data = response.json()
if isinstance(response_data, dict):
@@ -49,6 +61,8 @@ class GitlabStream(HttpStream, ABC):
return {"page": self.page}
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
if response.status_code == 403:
return []
response_data = response.json()
if isinstance(response_data, list):
for record in response_data:
@@ -315,16 +329,14 @@ class Users(GitlabChildStream):
pass
# TODO: We need to upgrade the plan for these feature (epics) to be available
class Epics(GitlabChildStream):
primary_key = "iid"
flatten_id_keys = ["author"]
# TODO: We need to upgrade the plan for these feature (epics) to be available
class EpicIssues(GitlabChildStream):
primary_key = "epic_issue_id"
path_list = ["group_id", "id"]
path_list = ["group_id", "iid"]
flatten_id_keys = ["milestone", "assignee", "author"]
flatten_list_keys = ["assignees"]
path_template = "groups/{group_id}/epics/{id}/issues"
path_template = "groups/{group_id}/epics/{iid}/issues"