jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

🎉 Source Github: add reaction streams (#5860)

Browse Source 
* Source Github: add reaction streams

Co-authored-by: ykurochkin <y.kurochkin@zazmic.com>
This commit is contained in:
Yevhenii
2021-09-09 13:13:29 +03:00
committed by GitHub
parent bb043b7f50
commit 9b71c28902
12 changed files with 169 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-integrations/connectors/source-github/Dockerfile
View File

@@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.9
LABEL io.airbyte.version=0.1.10
LABEL io.airbyte.name=airbyte/source-github

40
airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json
View File

@@ -43,6 +43,16 @@
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
"stream": {
"name": "commit_comment_reactions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "commit_comments",
@@ -82,6 +92,16 @@
"destination_sync_mode": "append",
"cursor_field": ["created_at"]
},
{
"stream": {
"name": "issue_comment_reactions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "issue_events",
@@ -118,6 +138,16 @@
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
"stream": {
"name": "issue_reactions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "issues",
@@ -154,6 +184,16 @@
"destination_sync_mode": "append",
"cursor_field": ["updated_at"]
},
{
"stream": {
"name": "pull_request_comment_reactions",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "pull_request_stats",

4
airbyte-integrations/connectors/source-github/source_github/schemas/commit_comment_reactions.json Normal file
View File

@@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "reaction.json"
}

4
airbyte-integrations/connectors/source-github/source_github/schemas/issue_comment_reactions.json Normal file
View File

@@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "reaction.json"
}

4
airbyte-integrations/connectors/source-github/source_github/schemas/issue_reactions.json Normal file
View File

@@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "reaction.json"
}

4
airbyte-integrations/connectors/source-github/source_github/schemas/pull_request_comment_reactions.json Normal file
View File

@@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "reaction.json"
}

24
airbyte-integrations/connectors/source-github/source_github/schemas/shared/reaction.json Normal file
View File

@@ -0,0 +1,24 @@
{
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "integer"]
},
"node_id": {
"type": ["null", "string"]
},
"content": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"],
"format": "date-time"
},
"user": {
"$ref": "user.json"
},
"repository": {
"type": ["null", "string"]
}
}
}

8
airbyte-integrations/connectors/source-github/source_github/source.py
View File

@@ -37,15 +37,19 @@ from .streams import (
Branches,
Collaborators,
Comments,
CommitCommentReactions,
CommitComments,
Commits,
Events,
IssueCommentReactions,
IssueEvents,
IssueLabels,
IssueMilestones,
IssueReactions,
Issues,
Organizations,
Projects,
PullRequestCommentReactions,
PullRequests,
PullRequestStats,
Releases,
@@ -112,15 +116,19 @@ class SourceGithub(AbstractSource):
Branches(**full_refresh_args),
Collaborators(**full_refresh_args),
Comments(**incremental_args),
CommitCommentReactions(**incremental_args),
CommitComments(**incremental_args),
Commits(**incremental_args),
Events(**incremental_args),
IssueCommentReactions(**incremental_args),
IssueEvents(**incremental_args),
IssueLabels(**full_refresh_args),
IssueMilestones(**incremental_args),
IssueReactions(**incremental_args),
Issues(**incremental_args),
Organizations(**organization_args),
Projects(**incremental_args),
PullRequestCommentReactions(**incremental_args),
PullRequestStats(**full_refresh_args),
PullRequests(**incremental_args),
Releases(**incremental_args),

73
airbyte-integrations/connectors/source-github/source_github/streams.py
View File

@@ -24,7 +24,8 @@
import os
import time
from abc import ABC
from abc import ABC, abstractmethod
from copy import deepcopy
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
from urllib import parse
@@ -276,7 +277,9 @@ class SemiIncrementalGithubStream(GithubStream):
class IncrementalGithubStream(SemiIncrementalGithubStream):
def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]:
params = super().request_params(stream_state=stream_state, **kwargs)
params["since"] = self.get_starting_point(stream_state=stream_state, repository=stream_slice["repository"])
since_params = self.get_starting_point(stream_state=stream_state, repository=stream_slice["repository"])
if since_params:
params["since"] = since_params
return params
@@ -686,3 +689,69 @@ class ReviewComments(IncrementalGithubStream):
def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
return f"repos/{stream_slice['repository']}/pulls/comments"
# Reactions streams
class ReactionStream(GithubStream, ABC):
parent_key = "id"
def __init__(self, **kwargs):
self._stream_kwargs = deepcopy(kwargs)
self._parent_stream = self.parent_entity(**kwargs)
kwargs.pop("start_date", None)
super().__init__(**kwargs)
@property
@abstractmethod
def parent_entity(self):
"""
Specify the class of the parent stream for which receive reactions
"""
def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
parent_path = self._parent_stream.path(stream_slice=stream_slice, **kwargs)
return f"{parent_path}/{stream_slice[self.parent_key]}/reactions"
def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
for stream_slice in super().stream_slices(**kwargs):
for parent_record in self._parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
yield {self.parent_key: parent_record[self.parent_key], "repository": stream_slice["repository"]}
def request_headers(self, **kwargs) -> Mapping[str, Any]:
return {"Accept": "application/vnd.github.squirrel-girl-preview+json"}
class CommitCommentReactions(ReactionStream):
"""
API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-commit-comment
"""
parent_entity = CommitComments
class IssueCommentReactions(ReactionStream):
"""
API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue-comment
"""
parent_entity = Comments
class IssueReactions(ReactionStream):
"""
API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue
"""
parent_entity = Issues
parent_key = "number"
class PullRequestCommentReactions(ReactionStream):
"""
API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-pull-request-review-comment
"""
parent_entity = ReviewComments