jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

🐛 Source Github: add new streams TeamMembers, TeamMemberships (#11893)

Browse Source 
Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
This commit is contained in:
Serhii Chvaliuk
2022-04-21 18:50:56 +03:00
committed by GitHub
parent 39b074ebb6
commit 8cf45693b1
12 changed files with 227 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View File

@@ -258,7 +258,7 @@
- name: GitHub
sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
dockerRepository: airbyte/source-github
dockerImageTag: 0.2.27
dockerImageTag: 0.2.28
documentationUrl: https://docs.airbyte.io/integrations/sources/github
icon: github.svg
sourceType: api

8
airbyte-config/init/src/main/resources/seed/source_specs.yaml
View File

@@ -2459,7 +2459,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-github:0.2.27"
- dockerImage: "airbyte/source-github:0.2.28"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
connectionSpecification:
@@ -2485,9 +2485,6 @@
option_title:
type: "string"
const: "OAuth Credentials"
enum:
- "OAuth Credentials"
default: "OAuth Credentials"
order: 0
access_token:
type: "string"
@@ -2502,9 +2499,6 @@
option_title:
type: "string"
const: "PAT Credentials"
enum:
- "PAT Credentials"
default: "PAT Credentials"
order: 0
personal_access_token:
type: "string"

2
airbyte-integrations/connectors/source-github/Dockerfile
View File

@@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.2.27
LABEL io.airbyte.version=0.2.28
LABEL io.airbyte.name=airbyte/source-github

1
airbyte-integrations/connectors/source-github/README.md
View File

@@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run:
```
source .venv/bin/activate
pip install -r requirements.txt
pip install '.[tests]'
```
If you are in an IDE, follow your IDE's instructions to activate the virtualenv.

20
airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json
View File

@@ -380,6 +380,26 @@
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "team_members",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["id"], ["team_slug"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "team_memberships",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["url"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
}
]
}

66
airbyte-integrations/connectors/source-github/source_github/schemas/team_members.json Normal file
View File

@@ -0,0 +1,66 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"login": {
"type": ["null", "string"]
},
"id": {
"type": "integer"
},
"node_id": {
"type": ["null", "string"]
},
"avatar_url": {
"type": ["null", "string"]
},
"gravatar_id": {
"type": ["null", "string"]
},
"url": {
"type": ["null", "string"]
},
"html_url": {
"type": ["null", "string"]
},
"followers_url": {
"type": ["null", "string"]
},
"following_url": {
"type": ["null", "string"]
},
"gists_url": {
"type": ["null", "string"]
},
"starred_url": {
"type": ["null", "string"]
},
"subscriptions_url": {
"type": ["null", "string"]
},
"organizations_url": {
"type": ["null", "string"]
},
"repos_url": {
"type": ["null", "string"]
},
"events_url": {
"type": ["null", "string"]
},
"received_events_url": {
"type": ["null", "string"]
},
"type": {
"type": ["null", "string"]
},
"site_admin": {
"type": ["null", "boolean"]
},
"organization": {
"type": "string"
},
"team_slug": {
"type": "string"
}
}
}

24
airbyte-integrations/connectors/source-github/source_github/schemas/team_memberships.json Normal file
View File

@@ -0,0 +1,24 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"state": {
"type": ["null", "string"]
},
"role": {
"type": ["null", "string"]
},
"url": {
"type": "string"
},
"organization": {
"type": "string"
},
"team_slug": {
"type": "string"
},
"username": {
"type": "string"
}
}
}

8
airbyte-integrations/connectors/source-github/source_github/source.py
View File

@@ -43,6 +43,8 @@ from .streams import (
Reviews,
Stargazers,
Tags,
TeamMembers,
TeamMemberships,
Teams,
Users,
WorkflowRuns,
@@ -184,6 +186,8 @@ class SourceGithub(AbstractSource):
pull_requests_stream = PullRequests(**repository_args_with_start_date)
projects_stream = Projects(**repository_args_with_start_date)
project_columns_stream = ProjectColumns(projects_stream, **repository_args_with_start_date)
teams_stream = Teams(**organization_args)
team_members_stream = TeamMembers(parent=teams_stream, **repository_args)
return [
Assignees(**repository_args),
@@ -215,8 +219,10 @@ class SourceGithub(AbstractSource):
Reviews(parent=pull_requests_stream, **repository_args_with_start_date),
Stargazers(**repository_args_with_start_date),
Tags(**repository_args),
Teams(**organization_args),
teams_stream,
team_members_stream,
Users(**organization_args),
Workflows(**repository_args),
WorkflowRuns(**repository_args),
TeamMemberships(parent=team_members_stream, **repository_args),
]

4
airbyte-integrations/connectors/source-github/source_github/spec.json
View File

@@ -21,8 +21,6 @@
"option_title": {
"type": "string",
"const": "OAuth Credentials",
"enum": ["OAuth Credentials"],
"default": "OAuth Credentials",
"order": 0
},
"access_token": {
@@ -41,8 +39,6 @@
"option_title": {
"type": "string",
"const": "PAT Credentials",
"enum": ["PAT Credentials"],
"default": "PAT Credentials",
"order": 0
},
"personal_access_token": {

70
airbyte-integrations/connectors/source-github/source_github/streams.py
View File

@@ -1003,3 +1003,73 @@ class WorkflowRuns(GithubStream):
response = response.json().get("workflow_runs")
for record in response:
yield record
class TeamMembers(GithubStream):
"""
API docs: https://docs.github.com/en/rest/reference/teams#list-team-members
"""
primary_key = ["id", "team_slug"]
def __init__(self, parent: Teams, **kwargs):
super().__init__(**kwargs)
self.parent = parent
def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
return f"orgs/{stream_slice['organization']}/teams/{stream_slice['team_slug']}/members"
def stream_slices(
self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
) -> Iterable[Optional[Mapping[str, Any]]]:
parent_stream_slices = self.parent.stream_slices(
sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state
)
for stream_slice in parent_stream_slices:
parent_records = self.parent.read_records(
sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
)
for record in parent_records:
yield {"organization": record["organization"], "team_slug": record["slug"]}
def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
record["organization"] = stream_slice["organization"]
record["team_slug"] = stream_slice["team_slug"]
return record
class TeamMemberships(GithubStream):
"""
API docs: https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user
"""
primary_key = ["url"]
def __init__(self, parent: TeamMembers, **kwargs):
super().__init__(**kwargs)
self.parent = parent
def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
return f"orgs/{stream_slice['organization']}/teams/{stream_slice['team_slug']}/memberships/{stream_slice['username']}"
def stream_slices(
self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
) -> Iterable[Optional[Mapping[str, Any]]]:
parent_stream_slices = self.parent.stream_slices(
sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state
)
for stream_slice in parent_stream_slices:
parent_records = self.parent.read_records(
sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state
)
for record in parent_records:
yield {"organization": record["organization"], "team_slug": record["team_slug"], "username": record["login"]}
def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
yield self.transform(response.json(), stream_slice=stream_slice)
def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
record["organization"] = stream_slice["organization"]
record["team_slug"] = stream_slice["team_slug"]
record["username"] = stream_slice["username"]
return record

33
airbyte-integrations/connectors/source-github/unit_tests/test_stream.py
View File

@@ -32,6 +32,8 @@ from source_github.streams import (
Reviews,
Stargazers,
Tags,
TeamMembers,
TeamMemberships,
Teams,
Users,
)
@@ -744,3 +746,34 @@ def test_stream_reviews_incremental_read():
assert responses.calls[0].request.params["direction"] == "asc"
assert urlbase(responses.calls[3].request.url) == url_pulls
assert responses.calls[3].request.params["direction"] == "asc"
@responses.activate
def test_stream_team_members_full_refresh():
organization_args = {"organizations": ["org1"]}
repository_args = {"repositories": [], "page_size_for_large_streams": 100}
responses.add("GET", "https://api.github.com/orgs/org1/teams", json=[{"slug": "team1"}, {"slug": "team2"}])
responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/members", json=[{"login": "login1"}, {"login": "login2"}])
responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login1", json={"username": "login1"})
responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login2", json={"username": "login2"})
responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/members", json=[{"login": "login2"}])
responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/memberships/login2", json={"username": "login2"})
stream = TeamMembers(parent=Teams(**organization_args), **repository_args)
records = read_full_refresh(stream)
assert records == [
{"login": "login1", "organization": "org1", "team_slug": "team1"},
{"login": "login2", "organization": "org1", "team_slug": "team1"},
{"login": "login2", "organization": "org1", "team_slug": "team2"},
]
stream = TeamMemberships(parent=stream, **repository_args)
records = read_full_refresh(stream)
assert records == [
{"username": "login1", "organization": "org1", "team_slug": "team1"},
{"username": "login2", "organization": "org1", "team_slug": "team1"},
{"username": "login2", "organization": "org1", "team_slug": "team2"},
]

3
docs/integrations/sources/github.md
View File

@@ -20,6 +20,8 @@ This connector outputs the following full refresh streams:
* [Pull request commits](https://docs.github.com/en/rest/reference/pulls#list-commits-on-a-pull-request)
* [Repositories](https://docs.github.com/en/rest/reference/repos#list-organization-repositories)
* [Tags](https://docs.github.com/en/rest/reference/repos#list-repository-tags)
* [TeamMembers](https://docs.github.com/en/rest/teams/members#list-team-members)
* [TeamMemberships](https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user)
* [Teams](https://docs.github.com/en/rest/reference/teams#list-teams)
* [Users](https://docs.github.com/en/rest/reference/orgs#list-organization-members)
* [Workflows](https://docs.github.com/en/rest/reference/actions#workflows)
@@ -111,6 +113,7 @@ Your token should have at least the `repo` scope. Depending on which streams you
| Version | Date | Pull Request | Subject |
|:--------|:-----------| :--- |:-------------------------------------------------------------------------------------------------------------|
| 0.2.28 | 2022-04-21 | [11893](https://github.com/airbytehq/airbyte/pull/11893) | Add new streams `TeamMembers`, `TeamMemberships` |
| 0.2.27 | 2022-04-02 | [11678](https://github.com/airbytehq/airbyte/pull/11678) | Fix "PAT Credentials" in spec |
| 0.2.26 | 2022-03-31 | [11623](https://github.com/airbytehq/airbyte/pull/11623) | Re-factored incremental sync for `Reviews` stream |
| 0.2.25 | 2022-03-31 | [11567](https://github.com/airbytehq/airbyte/pull/11567) | Improve code for better error handling |