From 84ba3e79b3d223954fc2d997df02ff35c9d39840 Mon Sep 17 00:00:00 2001 From: "oleh.zorenko" <19872253+Zirochkaa@users.noreply.github.com> Date: Tue, 6 Jul 2021 17:06:34 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Native=20GitHub=20connector=20(#?= =?UTF-8?q?4174)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * GitHub source initial version All streams are in `full_refresh` mode only. * Run format command * Add handling for 404 error in Teams stream * Update creds naming for github * Update acceptance tests * Update check_connection() * Code review fixes Remove `/integration_tests/catalog.json` file. Use `Collaborators` class in `check_connection()` function. Remove excessive data from GithubStream class. * Remove unused import * Implement requested changes * Add incremental streams * Remove `supports_incremental` * Update docs * Implement requested changes * Implement requested changes * Implement requested changes * Bump connector version --- .github/workflows/publish-command.yml | 1 + .github/workflows/test-command.yml | 1 + .../ef69ef6e-aa7f-4af1-a01d-ef775033524e.json | 6 +- .../resources/seed/source_definitions.yaml | 6 +- airbyte-integrations/builds.md | 2 +- .../connectors/source-github/.dockerignore | 6 + .../connectors/source-github/CHANGELOG.md | 4 + .../connectors/source-github/Dockerfile | 15 + .../connectors/source-github/README.md | 131 ++++++ .../source-github/acceptance-test-config.yml | 39 ++ .../source-github/acceptance-test-docker.sh | 7 + .../connectors/source-github/build.gradle | 13 + .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 58 +++ .../integration_tests/acceptance.py | 36 ++ .../integration_tests/configured_catalog.json | 197 ++++++++ .../incremental_configured_catalog.json | 95 ++++ .../integration_tests/invalid_config.json | 5 + .../integration_tests/sample_config.json | 5 + .../integration_tests/sample_state.json | 57 +++ .../small_streams_configured_catalog.json | 96 ++++ .../too_large_streams_configured_catalog.json | 105 +++++ .../connectors/source-github/main.py | 33 ++ .../connectors/source-github/requirements.txt | 2 + .../connectors/source-github/setup.py | 49 ++ .../source-github/source_github/__init__.py | 27 ++ .../source_github/schemas/assignees.json | 63 +++ .../source_github/schemas/collaborators.json | 77 ++++ .../source_github/schemas/comments.json | 41 ++ .../schemas/commit_comments.json | 50 ++ .../source_github/schemas/commits.json | 123 +++++ .../source_github/schemas/events.json | 34 ++ .../source_github/schemas/issue_events.json | 37 ++ .../source_github/schemas/issue_labels.json | 30 ++ .../schemas/issue_milestones.json | 61 +++ .../source_github/schemas/issues.json | 107 +++++ .../source_github/schemas/projects.json | 50 ++ .../source_github/schemas/pull_requests.json | 233 ++++++++++ .../source_github/schemas/releases.json | 111 +++++ .../source_github/schemas/reviews.json | 61 +++ .../source_github/schemas/stargazers.json | 16 + .../source_github/schemas/teams.json | 45 ++ .../source-github/source_github/source.py | 85 ++++ .../source-github/source_github/spec.json | 28 ++ .../source-github/source_github/streams.py | 430 ++++++++++++++++++ .../source-github/unit_tests/unit_test.py | 27 ++ docs/integrations/sources/github.md | 51 ++- docs/project-overview/changelog/connectors.md | 51 ++- tools/bin/ci_credentials.sh | 4 +- 49 files changed, 2760 insertions(+), 51 deletions(-) create mode 100644 airbyte-integrations/connectors/source-github/.dockerignore create mode 100644 airbyte-integrations/connectors/source-github/CHANGELOG.md create mode 100644 airbyte-integrations/connectors/source-github/Dockerfile create mode 100644 airbyte-integrations/connectors/source-github/README.md create mode 100644 airbyte-integrations/connectors/source-github/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-github/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-github/build.gradle create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-github/main.py create mode 100644 airbyte-integrations/connectors/source-github/requirements.txt create mode 100644 airbyte-integrations/connectors/source-github/setup.py create mode 100644 airbyte-integrations/connectors/source-github/source_github/__init__.py create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/assignees.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/comments.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/commit_comments.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/commits.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/events.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/issue_labels.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/issues.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/projects.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/releases.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/reviews.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/stargazers.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/schemas/teams.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/source.py create mode 100644 airbyte-integrations/connectors/source-github/source_github/spec.json create mode 100644 airbyte-integrations/connectors/source-github/source_github/streams.py create mode 100644 airbyte-integrations/connectors/source-github/unit_tests/unit_test.py diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 7a18305ebea..d34de21c4d5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -86,6 +86,7 @@ jobs: FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} + GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }} GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index a2aafb46f2a..250093f37ca 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -84,6 +84,7 @@ jobs: FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} + GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }} GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json index 62b8bff5c50..959010245fd 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json @@ -1,8 +1,8 @@ { "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", "name": "GitHub", - "dockerRepository": "airbyte/source-github-singer", - "dockerImageTag": "0.2.8", - "documentationUrl": "https://hub.docker.com/r/airbyte/source-github-singer", + "dockerRepository": "airbyte/source-github", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-github", "icon": "github.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index af3f61fa953..0e5ffeebf91 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -33,9 +33,9 @@ icon: google-adwords.svg - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e name: GitHub - dockerRepository: airbyte/source-github-singer - dockerImageTag: 0.2.8 - documentationUrl: https://hub.docker.com/r/airbyte/source-github-singer + dockerRepository: airbyte/source-github + dockerImageTag: 0.1.0 + documentationUrl: https://hub.docker.com/r/airbyte/source-github icon: github.svg - sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 name: Microsoft SQL Server (MSSQL) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 4ac27623952..a5362365989 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -25,7 +25,7 @@ Freshdesk [![source-freshdesk](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-freshdesk%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-freshdesk) - GitHub [![source-github-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-github-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-github-singer) + GitHub [![source-github](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-github%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-github) GitLab [![source-gitlab](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-gitlab%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-gitlab) diff --git a/airbyte-integrations/connectors/source-github/.dockerignore b/airbyte-integrations/connectors/source-github/.dockerignore new file mode 100644 index 00000000000..7dd21408feb --- /dev/null +++ b/airbyte-integrations/connectors/source-github/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_github +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-github/CHANGELOG.md b/airbyte-integrations/connectors/source-github/CHANGELOG.md new file mode 100644 index 00000000000..2482683b688 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## 0.1.0 +Initial Release. diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile new file mode 100644 index 00000000000..31cea7e00fd --- /dev/null +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_github ./source_github +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/README.md b/airbyte-integrations/connectors/source-github/README.md new file mode 100644 index 00000000000..80d919873d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/README.md @@ -0,0 +1,131 @@ +# Github Source + +This is the repository for the Github source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/github). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-github:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/github) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_github/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source github test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-github:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-github:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-github:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-github:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-github:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-github:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-github:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-github:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml new file mode 100644 index 00000000000..d2c9f5784f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -0,0 +1,39 @@ +connector_image: airbyte/source-github:dev +tests: + spec: + - spec_path: "source_github/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/small_streams_configured_catalog.json" + validate_output_from_all_streams: yes + # Below streams have too much records and because of it we are getting timeouts and hitting rate limits. + # That's why we turned them off. +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/too_large_streams_configured_catalog.json" +# validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + issues: [ "airbytehq/airbyte", "updated_at" ] + comments: [ "airbytehq/airbyte", "updated_at" ] + commits: [ "airbytehq/airbyte", "created_at" ] + commit_comments: [ "airbytehq/airbyte", "updated_at" ] + issue_milestones: [ "airbytehq/airbyte", "updated_at" ] + projects: [ "airbytehq/airbyte", "updated_at" ] + releases: [ "airbytehq/airbyte", "created_at" ] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/small_streams_configured_catalog.json" + # Below streams have too much records and because of it we are getting timeouts and hitting rate limits. + # That's why we turned them off. +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/too_large_streams_configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-github/acceptance-test-docker.sh new file mode 100644 index 00000000000..1425ff74f15 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-github/build.gradle b/airbyte-integrations/connectors/source-github/build.gradle new file mode 100644 index 00000000000..7a8dac431e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_github' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/__init__.py b/airbyte-integrations/connectors/source-github/integration_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json new file mode 100644 index 00000000000..65eb95569a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json @@ -0,0 +1,58 @@ +{ + "commit_comments": { + "airbytehq/airbyte": { + "updated_at": "2121-04-30T20:36:17Z" + } + }, + "projects": { + "airbytehq/airbyte": { + "updated_at": "2121-06-28T17:24:51Z" + } + }, + "releases": { + "airbytehq/airbyte": { + "created_at": "2121-06-23T23:57:07Z" + } + }, + "issue_milestones": { + "airbytehq/airbyte": { + "updated_at": "2121-06-25T22:28:33Z" + } + }, + "issues": { + "airbytehq/airbyte": { + "updated_at": "2121-06-30T06:44:42Z" + } + }, + "comments": { + "airbytehq/airbyte": { + "updated_at": "2121-06-30T10:22:10Z" + } + }, + "commits": { + "airbytehq/airbyte": { + "created_at": "2121-06-30T10:04:41Z" + } + }, + + "stargazers": { + "airbytehq/airbyte": { + "starred_at": "2121-06-29T02:04:57Z" + } + }, + "events": { + "airbytehq/airbyte": { + "created_at": "2121-06-29T03:44:45Z" + } + }, + "issue_events": { + "airbytehq/airbyte": { + "created_at": "2121-06-29T01:49:42Z" + } + }, + "pull_requests": { + "airbytehq/airbyte": { + "updated_at": "2121-06-28T23:36:35Z" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py new file mode 100644 index 00000000000..eeb4a2d3e02 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json new file mode 100644 index 00000000000..4af5492c56a --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json @@ -0,0 +1,197 @@ +{ + "streams": [ + { + "stream": { + "name": "assignees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "reviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "pull_requests", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "commit_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_milestones", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "commits", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "collaborators", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stargazers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["starred_at"], + "source_defined_primary_key": [["user_id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["starred_at"] + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_labels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "issues", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "releases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json new file mode 100644 index 00000000000..37272ff421d --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json @@ -0,0 +1,95 @@ +{ + "streams": [ + { + "stream": { + "name": "issues", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "commits", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "commit_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_milestones", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "releases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json new file mode 100644 index 00000000000..81d882dcf2f --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "access_token": "WRONG KEY", + "repository": "airbytehq/airbyte", + "start_date": "2021-06-30T11:30:03Z" +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-github/integration_tests/sample_config.json new file mode 100644 index 00000000000..f4d6b543955 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "access_token": "", + "repository": "", + "start_date": "2021-06-29T03:46:25Z" +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json new file mode 100644 index 00000000000..27a0bdfed26 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json @@ -0,0 +1,57 @@ +{ + "commit_comments": { + "airbytehq/airbyte": { + "updated_at": "2021-04-30T20:36:17Z" + } + }, + "projects": { + "airbytehq/airbyte": { + "updated_at": "2021-06-28T17:24:51Z" + } + }, + "stargazers": { + "airbytehq/airbyte": { + "starred_at": "2021-06-29T02:04:57Z" + } + }, + "events": { + "airbytehq/airbyte": { + "created_at": "2021-06-29T03:44:45Z" + } + }, + "issue_events": { + "airbytehq/airbyte": { + "created_at": "2021-06-29T01:49:42Z" + } + }, + "releases": { + "airbytehq/airbyte": { + "created_at": "2021-06-23T23:57:07Z" + } + }, + "pull_requests": { + "airbytehq/airbyte": { + "updated_at": "2021-06-28T23:36:35Z" + } + }, + "issue_milestones": { + "airbytehq/airbyte": { + "updated_at": "2021-06-25T22:28:33Z" + } + }, + "issues": { + "airbytehq/airbyte": { + "updated_at": "2021-06-30T11:32:49Z" + } + }, + "comments": { + "airbytehq/airbyte": { + "updated_at": "2021-06-30T10:22:10Z" + } + }, + "commits": { + "airbytehq/airbyte": { + "created_at": "2021-06-30T10:04:41Z" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json new file mode 100644 index 00000000000..306b265dc0e --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json @@ -0,0 +1,96 @@ +{ + "streams": [ + { + "stream": { + "name": "assignees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "commit_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_milestones", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "collaborators", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_labels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "releases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json new file mode 100644 index 00000000000..22bbab21ec7 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json @@ -0,0 +1,105 @@ +{ + "streams": [ + { + "stream": { + "name": "reviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "pull_requests", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "commits", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "stargazers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["starred_at"], + "source_defined_primary_key": [["user_id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["starred_at"] + }, + { + "stream": { + "name": "issues", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-github/main.py b/airbyte-integrations/connectors/source-github/main.py new file mode 100644 index 00000000000..54457a26e18 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_github import SourceGithub + +if __name__ == "__main__": + source = SourceGithub() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-github/requirements.txt b/airbyte-integrations/connectors/source-github/requirements.txt new file mode 100644 index 00000000000..0411042aa09 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-github/setup.py b/airbyte-integrations/connectors/source-github/setup.py new file mode 100644 index 00000000000..f3537380657 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/setup.py @@ -0,0 +1,49 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "vcrpy==4.1.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_github", + description="Source implementation for Github.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-github/source_github/__init__.py b/airbyte-integrations/connectors/source-github/source_github/__init__.py new file mode 100644 index 00000000000..9345170d78f --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceGithub + +__all__ = ["SourceGithub"] diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/assignees.json b/airbyte-integrations/connectors/source-github/source_github/schemas/assignees.json new file mode 100644 index 00000000000..101ab1397be --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/assignees.json @@ -0,0 +1,63 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "login": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "gravatar_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "followers_url": { + "type": ["null", "string"] + }, + "following_url": { + "type": ["null", "string"] + }, + "gists_url": { + "type": ["null", "string"] + }, + "starred_url": { + "type": ["null", "string"] + }, + "subscriptions_url": { + "type": ["null", "string"] + }, + "organizations_url": { + "type": ["null", "string"] + }, + "repos_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "received_events_url": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "site_admin": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json b/airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json new file mode 100644 index 00000000000..0308a703380 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/collaborators.json @@ -0,0 +1,77 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "login": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "gravatar_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "followers_url": { + "type": ["null", "string"] + }, + "following_url": { + "type": ["null", "string"] + }, + "gists_url": { + "type": ["null", "string"] + }, + "starred_url": { + "type": ["null", "string"] + }, + "subscriptions_url": { + "type": ["null", "string"] + }, + "organizations_url": { + "type": ["null", "string"] + }, + "repos_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "received_events_url": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "site_admin": { + "type": ["null", "boolean"] + }, + "permissions": { + "type": ["null", "object"], + "properties": { + "pull": { + "type": ["null", "boolean"] + }, + "push": { + "type": ["null", "boolean"] + }, + "admin": { + "type": ["null", "boolean"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/comments.json b/airbyte-integrations/connectors/source-github/source_github/schemas/comments.json new file mode 100644 index 00000000000..0ffd511e423 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/comments.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "issue_url": { + "type": ["null", "string"] + }, + "author_association": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/commit_comments.json b/airbyte-integrations/connectors/source-github/source_github/schemas/commit_comments.json new file mode 100644 index 00000000000..9f8f2d32ab3 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/commit_comments.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "path": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "line": { + "type": ["null", "integer"] + }, + "commit_id": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "author_association": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/commits.json b/airbyte-integrations/connectors/source-github/source_github/schemas/commits.json new file mode 100644 index 00000000000..c39302ca4c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/commits.json @@ -0,0 +1,123 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + }, + "sha": { + "type": ["null", "string"] + }, + "node_id": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "comments_url": { + "type": ["null", "string"] + }, + "commit": { + "type": ["null", "object"], + "properties": { + "author": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "committer": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "message": { + "type": ["null", "string"] + }, + "tree": { + "type": ["null", "object"], + "properties": { + "sha": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "url": { + "type": ["null", "string"] + }, + "comment_count": { + "type": ["null", "integer"] + }, + "verification": { + "type": ["null", "object"], + "properties": { + "verified": { + "type": ["null", "boolean"] + }, + "reason": { + "type": ["null", "string"] + }, + "signature": { + "type": ["null", "string"] + }, + "payload": { + "type": ["null", "string"] + } + } + } + } + }, + "author_id": { + "type": ["null", "integer"] + }, + "committer_id": { + "type": ["null", "integer"] + }, + "parents": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "sha": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/events.json b/airbyte-integrations/connectors/source-github/source_github/schemas/events.json new file mode 100644 index 00000000000..e5ce1363d3b --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/events.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "public": { + "type": ["null", "boolean"] + }, + "payload": { + "type": ["null", "object"], + "properties": {} + }, + "repo_id": { + "type": ["null", "integer"] + }, + "actor_id": { + "type": ["null", "integer"] + }, + "org_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json new file mode 100644 index 00000000000..ceb8b127349 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "actor_id": { + "type": ["null", "integer"] + }, + "event": { + "type": ["null", "string"] + }, + "commit_id": { + "type": ["null", "string"] + }, + "commit_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "issue_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_labels.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_labels.json new file mode 100644 index 00000000000..7b1fd07c0ac --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_labels.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json new file mode 100644 index 00000000000..4264178725b --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "labels_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "integer"] + }, + "open_issues": { + "type": ["null", "integer"] + }, + "closed_issues": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "due_on": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issues.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issues.json new file mode 100644 index 00000000000..5c22c402a7e --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issues.json @@ -0,0 +1,107 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "repository_url": { + "type": ["null", "string"] + }, + "labels_url": { + "type": ["null", "string"] + }, + "comments_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "assignee_id": { + "type": ["null", "integer"] + }, + "assignees": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "milestone_id": { + "type": ["null", "integer"] + }, + "locked": { + "type": ["null", "boolean"] + }, + "active_lock_reason": { + "type": ["null", "string"] + }, + "comments": { + "type": ["null", "integer"] + }, + "pull_request": { + "properties": { + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "diff_url": { + "type": ["null", "string"] + }, + "patch_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "author_association": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/projects.json b/airbyte-integrations/connectors/source-github/source_github/schemas/projects.json new file mode 100644 index 00000000000..e74d1c22549 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/projects.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "owner_url": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "columns_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json new file mode 100644 index 00000000000..625f6589c64 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json @@ -0,0 +1,233 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "diff_url": { + "type": ["null", "string"] + }, + "patch_url": { + "type": ["null", "string"] + }, + "issue_url": { + "type": ["null", "string"] + }, + "commits_url": { + "type": ["null", "string"] + }, + "review_comments_url": { + "type": ["null", "string"] + }, + "review_comment_url": { + "type": ["null", "string"] + }, + "comments_url": { + "type": ["null", "string"] + }, + "statuses_url": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "locked": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "body": { + "type": ["null", "string"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "milestone_id": { + "type": ["null", "integer"] + }, + "active_lock_reason": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "merged_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "merge_commit_sha": { + "type": ["null", "string"] + }, + "assignee_id": { + "type": ["null", "integer"] + }, + "assignees": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "requested_reviewers": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "requested_teams": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "head": { + "type": ["null", "object"], + "properties": { + "label": { + "type": ["null", "string"] + }, + "ref": { + "type": ["null", "string"] + }, + "sha": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "repo_id": { + "type": ["null", "integer"] + } + } + }, + "base": { + "type": ["null", "object"], + "properties": { + "label": { + "type": ["null", "string"] + }, + "ref": { + "type": ["null", "string"] + }, + "sha": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "repo_id": { + "type": ["null", "integer"] + } + } + }, + "_links": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "html": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "issue": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "comments": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "review_comments": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "review_comment": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "commits": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "statuses": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + } + } + }, + "author_association": { + "type": ["null", "string"] + }, + "auto_merge": { + "type": ["null", "boolean"] + }, + "draft": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/releases.json b/airbyte-integrations/connectors/source-github/source_github/schemas/releases.json new file mode 100644 index 00000000000..227d6715219 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/releases.json @@ -0,0 +1,111 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "assets_url": { + "type": ["null", "string"] + }, + "upload_url": { + "type": ["null", "string"] + }, + "tarball_url": { + "type": ["null", "string"] + }, + "zipball_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "tag_name": { + "type": ["null", "string"] + }, + "target_commitish": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "draft": { + "type": ["null", "boolean"] + }, + "prerelease": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "author_id": { + "type": ["null", "integer"] + }, + "assets": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "browser_download_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "content_type": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "download_count": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "uploader_id": { + "type": ["null", "integer"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/reviews.json b/airbyte-integrations/connectors/source-github/source_github/schemas/reviews.json new file mode 100644 index 00000000000..60489ec6db1 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/reviews.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "body": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "pull_request_url": { + "type": ["null", "string"] + }, + "_links": { + "type": ["null", "object"], + "properties": { + "html": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "pull_request": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + } + } + }, + "submitted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "commit_id": { + "type": ["null", "string"] + }, + "author_association": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/stargazers.json b/airbyte-integrations/connectors/source-github/source_github/schemas/stargazers.json new file mode 100644 index 00000000000..15518b63cc1 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/stargazers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "starred_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/teams.json b/airbyte-integrations/connectors/source-github/source_github/schemas/teams.json new file mode 100644 index 00000000000..70dfb1314be --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/teams.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_ab_github_repository": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "slug": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "privacy": { + "type": ["null", "string"] + }, + "permission": { + "type": ["null", "string"] + }, + "members_url": { + "type": ["null", "string"] + }, + "repositories_url": { + "type": ["null", "string"] + }, + "parent": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py new file mode 100644 index 00000000000..ce0b36cbc1c --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -0,0 +1,85 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from typing import Any, List, Mapping, Tuple + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +from .streams import ( + Assignees, + Collaborators, + Comments, + CommitComments, + Commits, + Events, + IssueEvents, + IssueLabels, + IssueMilestones, + Issues, + Projects, + PullRequests, + Releases, + Reviews, + Stargazers, + Teams, +) + + +class SourceGithub(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + try: + authenticator = TokenAuthenticator(token=config["access_token"], auth_method="token") + commits_stream = Commits(authenticator=authenticator, repository=config["repository"], start_date=config["start_date"]) + next(commits_stream.read_records(sync_mode=SyncMode.full_refresh)) + return True, None + except Exception as e: + return False, repr(e) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + authenticator = TokenAuthenticator(token=config["access_token"], auth_method="token") + full_refresh_args = {"authenticator": authenticator, "repository": config["repository"]} + incremental_args = {"authenticator": authenticator, "repository": config["repository"], "start_date": config["start_date"]} + return [ + Assignees(**full_refresh_args), + Reviews(**full_refresh_args), + Collaborators(**full_refresh_args), + Teams(**full_refresh_args), + IssueLabels(**full_refresh_args), + Releases(**incremental_args), + Events(**incremental_args), + Comments(**incremental_args), + PullRequests(**incremental_args), + CommitComments(**incremental_args), + IssueMilestones(**incremental_args), + Commits(**incremental_args), + Stargazers(**incremental_args), + Projects(**incremental_args), + Issues(**incremental_args), + IssueEvents(**incremental_args), + ] diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json new file mode 100644 index 00000000000..1735b812c45 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -0,0 +1,28 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/github", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Github Source Spec", + "type": "object", + "required": ["access_token", "repository", "start_date"], + "additionalProperties": false, + "properties": { + "access_token": { + "type": "string", + "description": "Log into Github and then generate a personal access token.", + "airbyte_secret": true + }, + "repository": { + "type": "string", + "examples": ["airbytehq/airbyte"], + "description": "GitHub repository. e.g. 'airbytehq/airbyte' or 'singer-io/tap-github'" + }, + "start_date": { + "type": "string", + "description": "The date from which you'd like to replicate data for GitHub in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only in the following incremental streams: comments, commits and issues.", + "examples": ["2021-03-01T00:00:00Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py new file mode 100644 index 00000000000..306065401f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -0,0 +1,430 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import tempfile +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional + +import requests +import vcr +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream +from requests.exceptions import HTTPError + +cache_file = tempfile.NamedTemporaryFile() + + +class GithubStream(HttpStream, ABC): + url_base = "https://api.github.com/" + + primary_key = "id" + + # GitHub pagination could be from 1 to 100. + page_size = 100 + + # Default page value for pagination. + _page = 1 + + stream_base_params = {} + + # Fields in below variable will be used for data clearing. Put there keys which represent: + # - objects `{}`, like `user`, `actor` etc. + # - lists `[]`, like `labels`, `assignees` etc. + fields_to_minimize = () + + def __init__(self, repository: str, **kwargs): + super().__init__(**kwargs) + self.repository = repository + + def path(self, **kwargs) -> str: + return f"repos/{self.repository}/{self.name}" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + response_data = response.json() + if response_data and len(response_data) == self.page_size: + self._page += 1 + return {"page": self._page} + + def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + try: + yield from super().read_records(**kwargs) + except HTTPError as e: + error_msg = str(e) + + # This whole try/except situation in `read_records()` isn't good but right now in `self._send_request()` + # function we have `response.raise_for_status()` so we don't have much choice on how to handle errors. + # We added this try/except code because for private repositories `Teams` stream is not available and we get + # "404 Client Error: Not Found for url: https://api.github.com/orgs/sherifnada/teams?per_page=100" error. + # Blocked on https://github.com/airbytehq/airbyte/issues/3514. + if "/teams?" in error_msg: + error_msg = f"Syncing Team stream isn't available for repository {self.repository}" + + self.logger.warn(error_msg) + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = {"per_page": self.page_size} + + if next_page_token: + params.update(next_page_token) + + params.update(self.stream_base_params) + + return params + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + # Without sending `User-Agent` header we will be getting `403 Client Error: Forbidden for url` error. + return { + "User-Agent": "PostmanRuntime/7.28.0", + } + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + for record in response.json(): # GitHub puts records in an array. + yield self.transform(record=record) + + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Use this method to: + - remove excessive fields from record; + - minify subelements in the record. For example, if you have `reviews` record which looks like this: + { + "id": 671782869, + "node_id": "MDE3OlB1bGxSZXF1ZXN0UmV2aWV3NjcxNzgyODY5", + "user": { + "login": "keu", + "id": 1619536, + ... + }, + "body": "lgtm, just small comment", + ... + } + + `user` subelement contains almost all possible fields fo user and it's not optimal to store such data in + `reviews` record. We may leave only `user.id` field and save in to `user_id` field in the record. So if you + need to do something similar with your record you may use this method. + """ + for field in self.fields_to_minimize: + field_value = record.pop(field, None) + if field_value is None: + record[field] = field_value + elif isinstance(field_value, dict): + record[f"{field}_id"] = field_value.get("id") if field_value else None + elif isinstance(field_value, list): + record[field] = [value.get("id") for value in field_value] + + # TODO In the future when we will add support for multiple repositories we will need an additional field in + # each stream that will tell which repository the record belongs to. In singer it was `_sdc_repository` field. + # You may see an example here: + # https://github.com/airbytehq/tap-github/blob/a0530b80d299864a9a106398b74d87c31ecc8a7a/tap_github/__init__.py#L253 + # We decided to leave this field (and rename it) for future usage when support for multiple repositories + # will be added. + record["_ab_github_repository"] = self.repository + + return record + + +class SemiIncrementalGithubStream(GithubStream): + """ + Semi incremental streams are also incremental but with one difference, they: + - read all records; + - output only new records. + This means that semi incremental streams read all records (like full_refresh streams) but do filtering directly + in the code and output only latest records (like incremental streams). + """ + + cursor_field = "updated_at" + + # This flag is used to indicate that current stream supports `sort` and `direction` request parameters and that + # we should break processing records if possible. If `sort` is set to `updated` and `direction` is set to `desc` + # this means that latest records will be at the beginning of the response and after we processed those latest + # records we can just stop and not process other record. This will increase speed of each incremental stream + # which supports those 2 request parameters. + is_sorted_descending = False + + def __init__(self, start_date: str, **kwargs): + super().__init__(**kwargs) + self._start_date = start_date + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state + object and returning an updated state object. + """ + state_value = latest_cursor_value = latest_record.get(self.cursor_field) + + if current_stream_state.get(self.repository, {}).get(self.cursor_field): + state_value = max(latest_cursor_value, current_stream_state[self.repository][self.cursor_field]) + + return {self.repository: {self.cursor_field: state_value}} + + def get_starting_point(self, stream_state: Mapping[str, Any]): + start_point = self._start_date + + if stream_state and stream_state.get(self.repository, {}).get(self.cursor_field): + start_point = max(start_point, stream_state[self.repository][self.cursor_field]) + + return start_point + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + start_point = self.get_starting_point(stream_state=stream_state) + for record in super().read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ): + if record.get(self.cursor_field) > start_point: + yield record + elif self.is_sorted_descending and record.get(self.cursor_field) < start_point: + break + + +class IncrementalGithubStream(SemiIncrementalGithubStream): + def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, **kwargs) + params["since"] = self.get_starting_point(stream_state=stream_state) + return params + + +# Below are full refresh streams + + +class Assignees(GithubStream): + pass + + +class Reviews(GithubStream): + fields_to_minimize = ("user",) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + pull_request_number = stream_slice["pull_request_number"] + return f"repos/{self.repository}/pulls/{pull_request_number}/reviews" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + pull_requests_stream = PullRequests(authenticator=self.authenticator, repository=self.repository, start_date="") + for pull_request in pull_requests_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"pull_request_number": pull_request["number"]} + + +class Collaborators(GithubStream): + pass + + +class IssueLabels(GithubStream): + def path(self, **kwargs) -> str: + return f"repos/{self.repository}/labels" + + +class Teams(GithubStream): + def path(self, **kwargs) -> str: + owner, _ = self.repository.split("/") + return f"orgs/{owner}/teams" + + +# Below are semi incremental streams + + +class Releases(SemiIncrementalGithubStream): + cursor_field = "created_at" + fields_to_minimize = ("author",) + + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + record = super().transform(record=record) + + assets = record.get("assets", []) + for asset in assets: + uploader = asset.pop("uploader", None) + asset["uploader_id"] = uploader.get("id") if uploader else None + + return record + + +class Events(SemiIncrementalGithubStream): + cursor_field = "created_at" + fields_to_minimize = ( + "actor", + "repo", + "org", + ) + + +class PullRequests(SemiIncrementalGithubStream): + fields_to_minimize = ( + "user", + "milestone", + "assignee", + "labels", + "assignees", + "requested_reviewers", + "requested_teams", + ) + stream_base_params = { + "state": "all", + "sort": "updated", + "direction": "desc", + } + + def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + with vcr.use_cassette(cache_file.name, record_mode="new_episodes", serializer="json"): + yield from super().read_records(**kwargs) + + def path(self, **kwargs) -> str: + return f"repos/{self.repository}/pulls" + + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + record = super().transform(record=record) + + head = record.get("head", {}) + head_user = head.pop("user", None) + head["user_id"] = head_user.get("id") if head_user else None + head_repo = head.pop("repo", None) + head["repo_id"] = head_repo.get("id") if head_repo else None + + base = record.get("base", {}) + base_user = base.pop("user", None) + base["user_id"] = base_user.get("id") if base_user else None + base_repo = base.pop("repo", None) + base["repo_id"] = base_repo.get("id") if base_repo else None + + return record + + +class CommitComments(SemiIncrementalGithubStream): + fields_to_minimize = ("user",) + + def path(self, **kwargs) -> str: + return f"repos/{self.repository}/comments" + + +class IssueMilestones(SemiIncrementalGithubStream): + is_sorted_descending = True + fields_to_minimize = ("creator",) + stream_base_params = { + "state": "all", + "sort": "updated", + "direction": "desc", + } + + def path(self, **kwargs) -> str: + return f"repos/{self.repository}/milestones" + + +class Stargazers(SemiIncrementalGithubStream): + primary_key = "user_id" + cursor_field = "starred_at" + fields_to_minimize = ("user",) + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + headers = super().request_headers(**kwargs) + # We need to send below header if we want to get `starred_at` field. See docs (Alternative response with + # star creation timestamps) - https://docs.github.com/en/rest/reference/activity#list-stargazers. + headers["Accept"] = "application/vnd.github.v3.star+json" + return headers + + +class Projects(SemiIncrementalGithubStream): + fields_to_minimize = ("creator",) + stream_base_params = { + "state": "all", + } + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + headers = super().request_headers(**kwargs) + # Projects stream requires sending following `Accept` header. If we won't sent it + # we'll get `415 Client Error: Unsupported Media Type` error. + headers["Accept"] = "application/vnd.github.inertia-preview+json" + return headers + + +class IssueEvents(SemiIncrementalGithubStream): + fields_to_minimize = ( + "actor", + "issue", + ) + + def path(self, **kwargs) -> str: + return f"repos/{self.repository}/issues/events" + + +# Below are incremental streams + + +class Comments(IncrementalGithubStream): + fields_to_minimize = ("user",) + stream_base_params = { + "sort": "updated", + "direction": "desc", + } + + def path(self, **kwargs) -> str: + return f"repos/{self.repository}/issues/comments" + + +class Commits(IncrementalGithubStream): + primary_key = "sha" + cursor_field = "created_at" + fields_to_minimize = ( + "author", + "committer", + ) + + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + record = super().transform(record=record) + + # Record of the `commits` stream doesn't have an updated_at/created_at field at the top level (so we could + # just write `record["updated_at"]` or `record["created_at"]`). Instead each record has such value in + # `commit.author.date`. So the easiest way is to just enrich the record returned from API with top level + # field `created_at` and use it as cursor_field. + record["created_at"] = record["commit"]["author"]["date"] + + return record + + +class Issues(IncrementalGithubStream): + fields_to_minimize = ( + "user", + "assignee", + "milestone", + "labels", + "assignees", + ) + stream_base_params = { + "state": "all", + "sort": "updated", + "direction": "desc", + } diff --git a/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py new file mode 100644 index 00000000000..b8a8150b507 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/unit_test.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +def test_example_method(): + assert True diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index e6bff786834..88224deff6c 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -8,23 +8,38 @@ This Github source wraps the [Singer Github Tap](https://github.com/singer-io/ta ### Output schema -This connector outputs the following streams: +This connector outputs the following full refresh streams: -* [Assignees](https://developer.github.com/v3/issues/assignees/#list-assignees) -* [Collaborators](https://developer.github.com/v3/repos/collaborators/#list-collaborators) -* [Comments](https://developer.github.com/v3/issues/comments/#list-comments-in-a-repository) -* [Commits](https://developer.github.com/v3/repos/commits/#list-commits-on-a-repository) -* [Commit comments](https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#list-commit-comments-for-a-repository) -* [Events](https://docs.github.com/en/free-pro-team@latest/rest/reference/activity#list-repository-events) -* [Issues](https://developer.github.com/v3/issues/#list-issues-for-a-repository) -* [Issue events](https://docs.github.com/en/free-pro-team@latest/rest/reference/issues#list-issue-events-for-a-repository) +* [Assignees](https://docs.github.com/en/rest/reference/issues#list-assignees) +* [Reviews](https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request) +* [Collaborators](https://docs.github.com/en/rest/reference/repos#list-repository-collaborators) +* [Teams](https://docs.github.com/en/rest/reference/teams#list-teams) * [Issue labels](https://docs.github.com/en/free-pro-team@latest/rest/reference/issues#list-labels-for-a-repository) -* [Issue milestones](https://docs.github.com/en/free-pro-team@latest/rest/reference/issues#list-milestones) -* [Projects](https://docs.github.com/en/free-pro-team@latest/rest/reference/projects#list-repository-projects) -* [Pull requests](https://developer.github.com/v3/pulls/#list-pull-requests) -* [Releases](https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#list-releases) -* [Stargazers](https://developer.github.com/v3/activity/starring/#list-stargazers) -* [Teams](https://docs.github.com/en/free-pro-team@latest/rest/reference/teams#list-teams) + +This connector outputs the following incremental streams: + +* [Comments](https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository) +* [Commits](https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository) +* [Issues](https://docs.github.com/en/rest/reference/issues#list-repository-issues) +* [Commit comments](https://docs.github.com/en/rest/reference/repos#list-commit-comments-for-a-repository) +* [Events](https://docs.github.com/en/rest/reference/activity#list-repository-events) +* [Issue events](https://docs.github.com/en/rest/reference/issues#list-issue-events-for-a-repository) +* [Issue milestones](https://docs.github.com/en/rest/reference/issues#list-milestones) +* [Projects](https://docs.github.com/en/rest/reference/projects#list-repository-projects) +* [Pull requests](https://docs.github.com/en/rest/reference/pulls#list-pull-requests) +* [Releases](https://docs.github.com/en/rest/reference/repos#list-releases) +* [Stargazers](https://docs.github.com/en/rest/reference/activity#list-stargazers) + +**Note:** Only 3 streams from above 11 incremental streams (`comments`, `commits` and `issues`) are pure incremental +meaning that they: +- read only new records; +- output only new records. + +Other 8 incremental streams are also incremental but with one difference, they: +- read all records; +- output only new records. + +Please, consider this behaviour when using those 8 incremental streams because it may affect you API call limits. ### Features @@ -44,8 +59,10 @@ The Github connector should not run into Github API limitations under normal usa ### Requirements -* Github Account -* Github Personal Access Token wih the necessary permissions \(described below\) +* Github Account; +* `access_token` - Github Personal Access Token wih the necessary permissions \(described below\); +* `repository` - GitHub repository which looks like `/`; +* `start_date` - start date for 3 incremental streams: `comments`, `commits` and `issues`. ### Setup guide diff --git a/docs/project-overview/changelog/connectors.md b/docs/project-overview/changelog/connectors.md index 13d0cc2f83b..d6036ee0f71 100644 --- a/docs/project-overview/changelog/connectors.md +++ b/docs/project-overview/changelog/connectors.md @@ -11,27 +11,28 @@ Note: Airbyte is not built on top of Singer, but is compatible with Singer's pro Check out our [connector roadmap](https://github.com/airbytehq/airbyte/projects/3) to see what we're currently working on. ## 7/06/2021 -1 new source: +2 new sources: * [**Airbyte-native Gitlab**](https://docs.airbyte.io/integrations/sources/gitlab) +* [**Airbyte-native GitHub**](https://docs.airbyte.io/integrations/sources/github) -## 7/01/2021 +## 7/01/2021 Bugfixes: * **Looker** source: Now supports the Run Look stream. * **Google Adwords**: CI is fixed and new version is published. * **Slack** source: Now Airbyte native and supports channels, channel members, messages, users, and threads streams. -* **Freshdesk** source: Does not fail after 300 pages anymore. +* **Freshdesk** source: Does not fail after 300 pages anymore. * **MSSQL** source: Now has comprehensive data type tests. -## 6/24/2021 +## 6/24/2021 1 new source: * [**Db2**](https://docs.airbyte.io/integrations/sources/db2) New features: * **S3** destination: supports Avro and Jsonl output! -* **BigQuery** destination: now supports loading JSON data as structured data. -* **Looker** source: Now supports self-hosted instances. +* **BigQuery** destination: now supports loading JSON data as structured data. +* **Looker** source: Now supports self-hosted instances. * **Facebook** source: is now migrated to the CDK. ## 6/18/2021 @@ -49,7 +50,7 @@ New features: ## 6/10/2021 1 new destination: * [**S3**](https://docs.airbyte.io/integrations/destinations/s3) - + 3 new sources: * [**Harvest**](https://docs.airbyte.io/integrations/sources/harvest) * [**Amplitude**](https://docs.airbyte.io/integrations/sources/amplitude) @@ -93,7 +94,7 @@ Bugfixes: ## 05/25/2021 -4 new sources: +4 new sources: * [**Asana**](https://docs.airbyte.io/integrations/sources/asana) * [**Klaviyo**](https://docs.airbyte.io/integrations/sources/klaviyo) * [**Recharge**](https://docs.airbyte.io/integrations/sources/recharge) @@ -101,7 +102,7 @@ Bugfixes: Progress on connectors: * **CDC for MySQL** is now available! -* **Sendgrid** source: support incremental sync, as rewritten using HTTP CDK ([#3445](https://github.com/airbytehq/airbyte/pull/3445)) +* **Sendgrid** source: support incremental sync, as rewritten using HTTP CDK ([#3445](https://github.com/airbytehq/airbyte/pull/3445)) * **Github** source bugfix: exception when parsing null date values, use `created_at` as cursor value for issue_milestones ([#3314](https://github.com/airbytehq/airbyte/pull/3314)) * **Slack** source bugfix: don't overwrite thread_ts in threads stream ([#3483](https://github.com/airbytehq/airbyte/pull/3483)) * **Facebook Marketing** source: allow configuring insights lookback window ([#3396](https://github.com/airbytehq/airbyte/pull/3396)) @@ -127,7 +128,7 @@ Progress on connectors: * [**PokeAPI**](https://docs.airbyte.io/integrations/sources/pokeapi) \(talking about long tail and having fun ;\)\) Progress on connectors: -* **Zoom**: bugfix on declaring correct types to match data coming from API \([\#3159](https://github.com/airbytehq/airbyte/pull/3159)\), thanks to [vovavovavovavova](https://github.com/vovavovavovavova) +* **Zoom**: bugfix on declaring correct types to match data coming from API \([\#3159](https://github.com/airbytehq/airbyte/pull/3159)\), thanks to [vovavovavovavova](https://github.com/vovavovavovavova) * **Smartsheets**: bugfix on gracefully handling empty cell values \([\#3337](https://github.com/airbytehq/airbyte/pull/3337)\), thanks to [Nathan Nowack](https://github.com/zzstoatzz) * **Stripe**: fix date property name, only add connected account header when set, and set primary key \(\#3210\), thanks to [Nathan Yergler](https://github.com/nyergler) @@ -160,7 +161,7 @@ Progress on connectors: 3 new source connectors! * [**Zendesk Talk**](https://docs.airbyte.io/integrations/sources/zendesk-talk) -* [**Iterable**](https://docs.airbyte.io/integrations/sources/iterable) +* [**Iterable**](https://docs.airbyte.io/integrations/sources/iterable) * [**Quickbooks**](https://docs.airbyte.io/integrations/sources/quickbooks) Other progress on connectors: @@ -175,7 +176,7 @@ Other progress on connectors: ## 04/13/2021 -* New connector: [**Oracle DB**](https://docs.airbyte.io/integrations/sources/oracle), thanks to [Marcos Marx](https://github.com/marcosmarxm) +* New connector: [**Oracle DB**](https://docs.airbyte.io/integrations/sources/oracle), thanks to [Marcos Marx](https://github.com/marcosmarxm) ## 04/07/2021 @@ -188,8 +189,8 @@ Other progress on connectors: ## 03/29/2021 -* We started measuring throughput of connectors. This will help us improve that point for all connectors. -* **Redshift**: implemented Copy strategy to improve its throughput. +* We started measuring throughput of connectors. This will help us improve that point for all connectors. +* **Redshift**: implemented Copy strategy to improve its throughput. * **Instagram**: bugfix an issue which caused media and media\_insights streams to stop syncing prematurely. * Support NCHAR and NVCHAR types in SQL-based database sources. * Add the ability to specify a custom JDBC parameters for the MySQL source connector. @@ -197,7 +198,7 @@ Other progress on connectors: ## 03/22/2021 * 2 new source connectors: [**Gitlab**](https://docs.airbyte.io/integrations/sources/gitlab) and [**Airbyte-native Hubspot**](https://docs.airbyte.io/integrations/sources/hubspot) -* Developing connectors now requires almost no interaction with Gradle, Airbyte’s monorepo build tool. If you’re building a Python connector, you never have to worry about developing outside your typical flow. See [the updated documentation](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector). +* Developing connectors now requires almost no interaction with Gradle, Airbyte’s monorepo build tool. If you’re building a Python connector, you never have to worry about developing outside your typical flow. See [the updated documentation](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector). ## 03/15/2021 @@ -234,13 +235,13 @@ Other progress on connectors: * Other fixes: * Thanks to [@ns-admetrics](https://github.com/ns-admetrics) for contributing an upgrade to the **Shopify** source connector which now provides the landing\_site field containing UTM parameters in the Orders table. * **Sendgrid** source connector supports most available endpoints available in the API - * **Facebook** Source connector now supports syncing Ad Insights data + * **Facebook** Source connector now supports syncing Ad Insights data * **Freshdesk** source connector now supports syncing satisfaction ratings and conversations - * **Microsoft Teams** source connector now gracefully handles rate limiting - * Bug fix in **Slack** source where the last few records in a sync were sporadically dropped + * **Microsoft Teams** source connector now gracefully handles rate limiting + * Bug fix in **Slack** source where the last few records in a sync were sporadically dropped * Bug fix in **Google Analytics** source where the last few records in sync were sporadically dropped - * In **Redshift source**, support non alpha-numeric table names - * Bug fix in **Github Source** to fix instances where syncs didn’t always fail if there was an error while reading data from the API + * In **Redshift source**, support non alpha-numeric table names + * Bug fix in **Github Source** to fix instances where syncs didn’t always fail if there was an error while reading data from the API ## 02/02/2021 @@ -249,7 +250,7 @@ Other progress on connectors: * Enhanced continuous testing for Tempo and Looker sources * Other fixes / features: * Correctly handle boolean types in the File Source - * Add docs for [App Store](https://docs.airbyte.io/integrations/sources/appstore) source + * Add docs for [App Store](https://docs.airbyte.io/integrations/sources/appstore) source * Fix a bug in Snowflake destination where the connector didn’t check for all needed write permissions, causing some syncs to fail ## 01/26/2021 @@ -264,7 +265,7 @@ Other progress on connectors: * **1 new source:** App Store \(thanks to [@Muriloo](https://github.com/Muriloo)\) * Fixes on connectors: * Bug fix writing boolean columns to Redshift - * Bug fix where getting a connector’s input configuration hung indefinitely + * Bug fix where getting a connector’s input configuration hung indefinitely * Stripe connector now gracefully handles rate limiting from the Stripe API ## 01/12/2021 @@ -301,7 +302,7 @@ Other progress on connectors: ## 12/04/2020 -**New sources:** [Redshift](../../integrations/sources/redshift.md), [Greenhouse](../../integrations/sources/greenhouse.md) +**New sources:** [Redshift](../../integrations/sources/redshift.md), [Greenhouse](../../integrations/sources/greenhouse.md) **New destination:** [Redshift](../../integrations/destinations/redshift.md) ## 11/30/2020 @@ -334,7 +335,7 @@ Other progress on connectors: ## 11/04/2020 -**New sources:** [Facebook Ads](connectors.md), [Google Ads](../../integrations/sources/google-adwords.md), [Marketo](../../integrations/sources/marketo.md) +**New sources:** [Facebook Ads](connectors.md), [Google Ads](../../integrations/sources/google-adwords.md), [Marketo](../../integrations/sources/marketo.md) **New destination:** [Snowflake](../../integrations/destinations/snowflake.md) ## 10/30/2020 @@ -347,6 +348,6 @@ Other progress on connectors: ## 09/23/2020 -**New sources:** [Stripe](../../integrations/sources/stripe.md), [Postgres](../../integrations/sources/postgres.md) +**New sources:** [Stripe](../../integrations/sources/stripe.md), [Postgres](../../integrations/sources/postgres.md) **New destinations:** [BigQuery](../../integrations/destinations/bigquery.md), [Postgres](../../integrations/destinations/postgres.md), [local CSV](../../integrations/destinations/local-csv.md) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index b938472c499..8f7c76d6b6c 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -45,8 +45,8 @@ write_standard_creds source-freshdesk "$FRESHDESK_TEST_CREDS" write_standard_creds source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" write_standard_creds source-gitlab "$GITLAB_INTEGRATION_TEST_CREDS" write_standard_creds source-gitlab-singer "$GITLAB_INTEGRATION_TEST_CREDS" -write_standard_creds source-github-singer "$GH_INTEGRATION_TEST_CREDS" -write_standard_creds source-google-ads "$GOOGLE_ADS_TEST_CREDS" +write_standard_creds source-github "$GH_NATIVE_INTEGRATION_TEST_CREDS" +write_standard_creds source-google-ads "$GOOGLE_ADS_TEST_CREDS" write_standard_creds source-google-adwords-singer "$ADWORDS_INTEGRATION_TEST_CREDS" write_standard_creds source-googleanalytics-singer "$GOOGLE_ANALYTICS_TEST_CREDS" write_standard_creds source-googleanalytics-singer "$GOOGLE_ANALYTICS_TEST_TRACKING_ID" "tracker.txt"