diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 76e05454c0e..2d5bc7f6a0f 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -58,6 +58,7 @@ jobs: GSHEETS_INTEGRATION_TESTS_CREDS: ${{ secrets.GSHEETS_INTEGRATION_TESTS_CREDS }} HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }} + JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }} SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json new file mode 100644 index 00000000000..873efef803c --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "68e63de2-bb83-4c7e-93fa-a8a9051e3993", + "name": "Jira", + "dockerRepository": "airbyte/source-jira", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-jira" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a4df990b04f..d0b8e6daec1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -133,6 +133,11 @@ dockerRepository: airbyte/source-intercom-singer dockerImageTag: 0.1.0 documentationUrl: https://hub.docker.com/r/airbyte/source-intercom-singer +- sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 + name: Jira + dockerRepository: airbyte/source-jira + dockerImageTag: 0.1.0 + documentationUrl: https://hub.docker.com/r/airbyte/source-jira - sourceDefinitionId: 859e501d-2b67-471f-91bb-1c801414d28f name: Mixpanel dockerRepository: airbyte/source-mixpanel-singer diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile b/airbyte-integrations/connectors/source-jira/Dockerfile new file mode 100644 index 00000000000..856e64729f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/Dockerfile @@ -0,0 +1,16 @@ +FROM airbyte/integration-base-python:dev + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="source_jira" +ENV AIRBYTE_IMPL_MODULE="source_jira" +ENV AIRBYTE_IMPL_PATH="SourceJira" + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-jira diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile.test b/airbyte-integrations/connectors/source-jira/Dockerfile.test new file mode 100644 index 00000000000..3d028a5afc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/Dockerfile.test @@ -0,0 +1,22 @@ +FROM airbyte/base-python-test:dev + +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="integration_tests" +ENV AIRBYTE_TEST_MODULE="integration_tests" +ENV AIRBYTE_TEST_PATH="SourceJiraStandardTest" + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-jira-standard-test + +WORKDIR /airbyte/integration_code +COPY source_jira source_jira +COPY $CODE_PATH $CODE_PATH +COPY sample_files/*.json $CODE_PATH/ +COPY secrets/* $CODE_PATH +COPY source_jira/*.json $CODE_PATH +COPY setup.py ./ + +RUN pip install ".[tests]" + +WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-jira/README.md b/airbyte-integrations/connectors/source-jira/README.md new file mode 100644 index 00000000000..10a3817d90f --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/README.md @@ -0,0 +1,64 @@ +# Jira Source + +This is the repository for the Jira source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/jira). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment +First, build the module by running the following from the `airbyte` project root directory: +``` +./gradlew :airbyte-integrations:connectors:source-jira:build +``` + +This will generate a virtualenv for this module in `source-jira/.venv`. Make sure this venv is active in your +development environment of choice. To activate the venv from the terminal, run: +``` +cd airbyte-integrations/connectors/source-jira # cd into the connector directory +source .venv/bin/activate +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/jira) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_jira/spec.json` file. +See `sample_files/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in RPass under the secret name `source-jira-integration-test-config` +and place them into `secrets/config.json`. + + +### Locally running the connector +``` +python main_dev.py spec +python main_dev.py check --config secrets/config.json +python main_dev.py discover --config secrets/config.json +python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +pytest unit_tests +``` + +### Locally running the connector docker image +``` +# in airbyte root directory +./gradlew :airbyte-integrations:connectors:source-jira:airbyteDocker +docker run --rm airbyte/source-jira:dev spec +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-jira/secrets:/secrets airbyte/source-jira:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-jira/secrets:/secrets airbyte/source-jira:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-jira/secrets:/secrets -v $(pwd)/airbyte-integrations/connectors/source-jira/sample_files:/sample_files airbyte/source-jira:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +``` + +### Integration Tests +1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-jira:standardSourceTestPython` to run the standard integration test suite. +1. To run additional integration tests, place your integration tests in the `integration_tests` directory and run them with `pytest integration_tests`. + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connectors/source-jira/build.gradle b/airbyte-integrations/connectors/source-jira/build.gradle new file mode 100644 index 00000000000..4ab6aad783a --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' + id 'airbyte-integration-test-java' +} + +airbytePython { + moduleDirectory 'source_jira' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/__init__.py b/airbyte-integrations/connectors/source-jira/integration_tests/__init__.py new file mode 100644 index 00000000000..cc8cf1f763a --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/integration_tests/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .standard_source_test import SourceJiraStandardTest + +__all__ = ["SourceJiraStandardTest"] diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-jira/integration_tests/standard_source_test.py new file mode 100644 index 00000000000..4c4ac033b5f --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/integration_tests/standard_source_test.py @@ -0,0 +1,29 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from base_python_test import DefaultStandardSourceTest + + +class SourceJiraStandardTest(DefaultStandardSourceTest): + pass diff --git a/airbyte-integrations/connectors/source-jira/main_dev.py b/airbyte-integrations/connectors/source-jira/main_dev.py new file mode 100644 index 00000000000..5995a15927a --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/main_dev.py @@ -0,0 +1,32 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import sys + +from base_python.entrypoint import launch +from source_jira import SourceJira + +if __name__ == "__main__": + source = SourceJira() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-jira/requirements.txt b/airbyte-integrations/connectors/source-jira/requirements.txt new file mode 100644 index 00000000000..76af767f375 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/requirements.txt @@ -0,0 +1,4 @@ +-e ../../bases/airbyte-protocol +-e ../../bases/base-python +-e ../../bases/base-python-test +-e . diff --git a/airbyte-integrations/connectors/source-jira/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-jira/sample_files/configured_catalog.json new file mode 100644 index 00000000000..3801a70a561 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/sample_files/configured_catalog.json @@ -0,0 +1,473 @@ +{ + "streams": [ + { + "stream": { + "name": "projects", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expand": { + "type": "string", + "description": "Expand options that include additional project details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + }, + "self": { + "type": "string", + "description": "The URL of the project details.", + "format": "uri", + "readOnly": true + }, + "id": { + "type": "string", + "description": "The ID of the project." + }, + "key": { + "type": "string", + "description": "The key of the project.", + "readOnly": true + }, + "description": { + "type": "string", + "description": "A brief description of the project.", + "readOnly": true + }, + "lead": { + "description": "The username of the project lead.", + "readOnly": true + }, + "components": { + "type": "array", + "description": "List of the components contained in the project.", + "readOnly": true + }, + "issueTypes": { + "type": "array", + "description": "List of the issue types available in the project.", + "readOnly": true + }, + "url": { + "type": "string", + "description": "A link to information about this project, such as project documentation.", + "readOnly": true + }, + "email": { + "type": "string", + "description": "An email address associated with the project." + }, + "assigneeType": { + "type": "string", + "description": "The default assignee when creating issues for this project.", + "readOnly": true, + "enum": ["PROJECT_LEAD", "UNASSIGNED"] + }, + "versions": { + "type": "array", + "description": "The versions defined in the project. For more information, see [Create version](#api-rest-api-3-version-post).", + "readOnly": true + }, + "name": { + "type": "string", + "description": "The name of the project.", + "readOnly": true + }, + "roles": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "readOnly": true + }, + "description": "The name and self URL for each role defined in the project. For more information, see [Create project role](#api-rest-api-3-role-post).", + "readOnly": true + }, + "avatarUrls": { + "description": "The URLs of the project's avatars.", + "readOnly": true + }, + "projectCategory": { + "description": "The category the project belongs to.", + "readOnly": true + }, + "projectTypeKey": { + "type": "string", + "description": "The [project type](https://confluence.atlassian.com/x/GwiiLQ#Jiraapplicationsoverview-Productfeaturesandprojecttypes) of the project.", + "readOnly": true, + "enum": ["software", "service_desk", "business"] + }, + "simplified": { + "type": "boolean", + "description": "Whether the project is simplified.", + "readOnly": true + }, + "style": { + "type": "string", + "description": "The type of the project.", + "readOnly": true, + "enum": ["classic", "next-gen"] + }, + "favourite": { + "type": "boolean", + "description": "Whether the project is selected as a favorite." + }, + "isPrivate": { + "type": "boolean", + "description": "Whether the project is private.", + "readOnly": true + }, + "issueTypeHierarchy": { + "description": "The issue type hierarchy for the project", + "readOnly": true + }, + "permissions": { + "description": "User permissions on the project", + "readOnly": true + }, + "properties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "Map of project properties", + "readOnly": true + }, + "uuid": { + "type": "string", + "description": "Unique ID for next-gen projects.", + "format": "uuid", + "readOnly": true + }, + "insight": { + "description": "Insights about the project.", + "readOnly": true + }, + "deleted": { + "type": "boolean", + "description": "Whether the project is marked as deleted.", + "readOnly": true + }, + "retentionTillDate": { + "type": "string", + "description": "The date when the project is deleted permanently.", + "format": "date-time", + "readOnly": true + }, + "deletedDate": { + "type": "string", + "description": "The date when the project was marked as deleted.", + "format": "date-time", + "readOnly": true + }, + "deletedBy": { + "description": "The user who marked the project as deleted.", + "readOnly": true + }, + "archived": { + "type": "boolean", + "description": "Whether the project is archived.", + "readOnly": true + }, + "archivedDate": { + "type": "string", + "description": "The date when the project was archived.", + "format": "date-time", + "readOnly": true + }, + "archivedBy": { + "description": "The user who archived the project.", + "readOnly": true + } + }, + "additionalProperties": false, + "description": "Details about a project." + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "resolutions", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the issue resolution.", + "format": "uri" + }, + "id": { + "type": "string", + "description": "The ID of the issue resolution." + }, + "description": { + "type": "string", + "description": "The description of the issue resolution." + }, + "name": { + "type": "string", + "description": "The name of the issue resolution." + } + }, + "additionalProperties": false, + "description": "Details of an issue resolution." + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the user.", + "format": "uri", + "readOnly": true + }, + "key": { + "type": "string", + "description": "This property is no longer available and will be removed from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/) for details." + }, + "accountId": { + "maxLength": 128, + "type": "string", + "description": "The account ID of the user, which uniquely identifies the user across all Atlassian products. For example, *5b10ac8d82e05b22cc7d4ef5*. Required in requests." + }, + "accountType": { + "type": "string", + "description": "The user account type. Can take the following values:\n\n * `atlassian` regular Atlassian user account\n * `app` system account used for Connect applications and OAuth to represent external systems\n * `customer` Jira Service Desk account representing an external service desk", + "readOnly": true, + "enum": ["atlassian", "app", "customer", "unknown"] + }, + "name": { + "type": "string", + "description": "This property is no longer available and will be removed from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/) for details." + }, + "emailAddress": { + "type": "string", + "description": "The email address of the user. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "avatarUrls": { + "description": "The avatars of the user.", + "readOnly": true + }, + "displayName": { + "type": "string", + "description": "The display name of the user. Depending on the user’s privacy setting, this may return an alternative value.", + "readOnly": true + }, + "active": { + "type": "boolean", + "description": "Whether the user is active.", + "readOnly": true + }, + "timeZone": { + "type": "string", + "description": "The time zone specified in the user's profile. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "locale": { + "type": "string", + "description": "The locale of the user. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "groups": { + "description": "The groups that the user belongs to.", + "readOnly": true + }, + "applicationRoles": { + "description": "The application roles the user is assigned to.", + "readOnly": true + }, + "expand": { + "type": "string", + "description": "Expand options that include additional user details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + } + }, + "additionalProperties": false, + "description": "A user with details as permitted by the user's Atlassian Account privacy settings. However, be aware of these exceptions:\n\n * User record deleted from Atlassian: This occurs as the result of a right to be forgotten request. In this case, `displayName` provides an indication and other parameters have default values or are blank (for example, email is blank).\n * User record corrupted: This occurs as a results of events such as a server import and can only happen to deleted users. In this case, `accountId` returns *unknown* and all other parameters have fallback values.\n * User record unavailable: This usually occurs due to an internal service outage. In this case, all parameters have fallback values." + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "issues", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expand": { + "type": "string", + "description": "Expand options that include additional issue details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + }, + "id": { + "type": "string", + "description": "The ID of the issue.", + "readOnly": true + }, + "self": { + "type": "string", + "description": "The URL of the issue details.", + "format": "uri", + "readOnly": true + }, + "key": { + "type": "string", + "description": "The key of the issue.", + "readOnly": true + }, + "renderedFields": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "The rendered value of each field present on the issue.", + "readOnly": true + }, + "properties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "Details of the issue properties identified in the request.", + "readOnly": true + }, + "names": { + "type": "object", + "additionalProperties": { + "type": "string", + "readOnly": true + }, + "description": "The ID and name of each field present on the issue.", + "readOnly": true + }, + "schema": { + "type": "object", + "description": "The schema describing each field present on the issue.", + "readOnly": true + }, + "transitions": { + "type": "array", + "description": "The transitions that can be performed on the issue.", + "readOnly": true + }, + "operations": { + "description": "The operations that can be performed on the issue.", + "readOnly": true + }, + "editmeta": { + "description": "The metadata for the fields on the issue that can be amended.", + "readOnly": true + }, + "changelog": { + "description": "Details of changelogs associated with the issue.", + "readOnly": true + }, + "versionedRepresentations": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "readOnly": true + }, + "description": "The versions of each field on the issue.", + "readOnly": true + }, + "fieldsToInclude": { + "type": "object" + }, + "fields": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": false + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "issue_comments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the comment.", + "readOnly": true + }, + "id": { + "type": "string", + "description": "The ID of the comment.", + "readOnly": true + }, + "author": { + "description": "The ID of the user who created the comment.", + "readOnly": true + }, + "body": { + "description": "The comment text in [Atlassian Document Format](https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/)." + }, + "renderedBody": { + "type": "string", + "description": "The rendered version of the comment.", + "readOnly": true + }, + "updateAuthor": { + "description": "The ID of the user who updated the comment last.", + "readOnly": true + }, + "created": { + "type": "string", + "description": "The date and time at which the comment was created.", + "format": "date-time", + "readOnly": true + }, + "updated": { + "type": "string", + "description": "The date and time at which the comment was updated last.", + "format": "date-time", + "readOnly": true + }, + "visibility": { + "description": "The group or role to which this comment is visible. Optional on create and update." + }, + "jsdPublic": { + "type": "boolean", + "description": "Whether the comment is visible in Jira Service Desk. Defaults to true when comments are created in the Jira Cloud Platform. This includes when the site doesn't use Jira Service Desk or the project isn't a Jira Service Desk project and, therefore, there is no Jira Service Desk for the issue to be visible on. To create a comment with its visibility in Jira Service Desk set to false, use the Jira Service Desk REST API [Create request comment](https://developer.atlassian.com/cloud/jira/service-desk/rest/#api-rest-servicedeskapi-request-issueIdOrKey-comment-post) operation.", + "readOnly": true + }, + "properties": { + "type": "array", + "description": "A list of comment properties. Optional on create and update." + } + }, + "additionalProperties": true, + "description": "A comment." + }, + "supported_sync_modes": ["full_refresh"] + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-jira/sample_files/full_configured_catalog.json b/airbyte-integrations/connectors/source-jira/sample_files/full_configured_catalog.json new file mode 100644 index 00000000000..3801a70a561 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/sample_files/full_configured_catalog.json @@ -0,0 +1,473 @@ +{ + "streams": [ + { + "stream": { + "name": "projects", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expand": { + "type": "string", + "description": "Expand options that include additional project details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + }, + "self": { + "type": "string", + "description": "The URL of the project details.", + "format": "uri", + "readOnly": true + }, + "id": { + "type": "string", + "description": "The ID of the project." + }, + "key": { + "type": "string", + "description": "The key of the project.", + "readOnly": true + }, + "description": { + "type": "string", + "description": "A brief description of the project.", + "readOnly": true + }, + "lead": { + "description": "The username of the project lead.", + "readOnly": true + }, + "components": { + "type": "array", + "description": "List of the components contained in the project.", + "readOnly": true + }, + "issueTypes": { + "type": "array", + "description": "List of the issue types available in the project.", + "readOnly": true + }, + "url": { + "type": "string", + "description": "A link to information about this project, such as project documentation.", + "readOnly": true + }, + "email": { + "type": "string", + "description": "An email address associated with the project." + }, + "assigneeType": { + "type": "string", + "description": "The default assignee when creating issues for this project.", + "readOnly": true, + "enum": ["PROJECT_LEAD", "UNASSIGNED"] + }, + "versions": { + "type": "array", + "description": "The versions defined in the project. For more information, see [Create version](#api-rest-api-3-version-post).", + "readOnly": true + }, + "name": { + "type": "string", + "description": "The name of the project.", + "readOnly": true + }, + "roles": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "readOnly": true + }, + "description": "The name and self URL for each role defined in the project. For more information, see [Create project role](#api-rest-api-3-role-post).", + "readOnly": true + }, + "avatarUrls": { + "description": "The URLs of the project's avatars.", + "readOnly": true + }, + "projectCategory": { + "description": "The category the project belongs to.", + "readOnly": true + }, + "projectTypeKey": { + "type": "string", + "description": "The [project type](https://confluence.atlassian.com/x/GwiiLQ#Jiraapplicationsoverview-Productfeaturesandprojecttypes) of the project.", + "readOnly": true, + "enum": ["software", "service_desk", "business"] + }, + "simplified": { + "type": "boolean", + "description": "Whether the project is simplified.", + "readOnly": true + }, + "style": { + "type": "string", + "description": "The type of the project.", + "readOnly": true, + "enum": ["classic", "next-gen"] + }, + "favourite": { + "type": "boolean", + "description": "Whether the project is selected as a favorite." + }, + "isPrivate": { + "type": "boolean", + "description": "Whether the project is private.", + "readOnly": true + }, + "issueTypeHierarchy": { + "description": "The issue type hierarchy for the project", + "readOnly": true + }, + "permissions": { + "description": "User permissions on the project", + "readOnly": true + }, + "properties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "Map of project properties", + "readOnly": true + }, + "uuid": { + "type": "string", + "description": "Unique ID for next-gen projects.", + "format": "uuid", + "readOnly": true + }, + "insight": { + "description": "Insights about the project.", + "readOnly": true + }, + "deleted": { + "type": "boolean", + "description": "Whether the project is marked as deleted.", + "readOnly": true + }, + "retentionTillDate": { + "type": "string", + "description": "The date when the project is deleted permanently.", + "format": "date-time", + "readOnly": true + }, + "deletedDate": { + "type": "string", + "description": "The date when the project was marked as deleted.", + "format": "date-time", + "readOnly": true + }, + "deletedBy": { + "description": "The user who marked the project as deleted.", + "readOnly": true + }, + "archived": { + "type": "boolean", + "description": "Whether the project is archived.", + "readOnly": true + }, + "archivedDate": { + "type": "string", + "description": "The date when the project was archived.", + "format": "date-time", + "readOnly": true + }, + "archivedBy": { + "description": "The user who archived the project.", + "readOnly": true + } + }, + "additionalProperties": false, + "description": "Details about a project." + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "resolutions", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the issue resolution.", + "format": "uri" + }, + "id": { + "type": "string", + "description": "The ID of the issue resolution." + }, + "description": { + "type": "string", + "description": "The description of the issue resolution." + }, + "name": { + "type": "string", + "description": "The name of the issue resolution." + } + }, + "additionalProperties": false, + "description": "Details of an issue resolution." + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the user.", + "format": "uri", + "readOnly": true + }, + "key": { + "type": "string", + "description": "This property is no longer available and will be removed from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/) for details." + }, + "accountId": { + "maxLength": 128, + "type": "string", + "description": "The account ID of the user, which uniquely identifies the user across all Atlassian products. For example, *5b10ac8d82e05b22cc7d4ef5*. Required in requests." + }, + "accountType": { + "type": "string", + "description": "The user account type. Can take the following values:\n\n * `atlassian` regular Atlassian user account\n * `app` system account used for Connect applications and OAuth to represent external systems\n * `customer` Jira Service Desk account representing an external service desk", + "readOnly": true, + "enum": ["atlassian", "app", "customer", "unknown"] + }, + "name": { + "type": "string", + "description": "This property is no longer available and will be removed from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/) for details." + }, + "emailAddress": { + "type": "string", + "description": "The email address of the user. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "avatarUrls": { + "description": "The avatars of the user.", + "readOnly": true + }, + "displayName": { + "type": "string", + "description": "The display name of the user. Depending on the user’s privacy setting, this may return an alternative value.", + "readOnly": true + }, + "active": { + "type": "boolean", + "description": "Whether the user is active.", + "readOnly": true + }, + "timeZone": { + "type": "string", + "description": "The time zone specified in the user's profile. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "locale": { + "type": "string", + "description": "The locale of the user. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "groups": { + "description": "The groups that the user belongs to.", + "readOnly": true + }, + "applicationRoles": { + "description": "The application roles the user is assigned to.", + "readOnly": true + }, + "expand": { + "type": "string", + "description": "Expand options that include additional user details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + } + }, + "additionalProperties": false, + "description": "A user with details as permitted by the user's Atlassian Account privacy settings. However, be aware of these exceptions:\n\n * User record deleted from Atlassian: This occurs as the result of a right to be forgotten request. In this case, `displayName` provides an indication and other parameters have default values or are blank (for example, email is blank).\n * User record corrupted: This occurs as a results of events such as a server import and can only happen to deleted users. In this case, `accountId` returns *unknown* and all other parameters have fallback values.\n * User record unavailable: This usually occurs due to an internal service outage. In this case, all parameters have fallback values." + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "issues", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expand": { + "type": "string", + "description": "Expand options that include additional issue details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + }, + "id": { + "type": "string", + "description": "The ID of the issue.", + "readOnly": true + }, + "self": { + "type": "string", + "description": "The URL of the issue details.", + "format": "uri", + "readOnly": true + }, + "key": { + "type": "string", + "description": "The key of the issue.", + "readOnly": true + }, + "renderedFields": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "The rendered value of each field present on the issue.", + "readOnly": true + }, + "properties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "Details of the issue properties identified in the request.", + "readOnly": true + }, + "names": { + "type": "object", + "additionalProperties": { + "type": "string", + "readOnly": true + }, + "description": "The ID and name of each field present on the issue.", + "readOnly": true + }, + "schema": { + "type": "object", + "description": "The schema describing each field present on the issue.", + "readOnly": true + }, + "transitions": { + "type": "array", + "description": "The transitions that can be performed on the issue.", + "readOnly": true + }, + "operations": { + "description": "The operations that can be performed on the issue.", + "readOnly": true + }, + "editmeta": { + "description": "The metadata for the fields on the issue that can be amended.", + "readOnly": true + }, + "changelog": { + "description": "Details of changelogs associated with the issue.", + "readOnly": true + }, + "versionedRepresentations": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "readOnly": true + }, + "description": "The versions of each field on the issue.", + "readOnly": true + }, + "fieldsToInclude": { + "type": "object" + }, + "fields": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": false + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "issue_comments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the comment.", + "readOnly": true + }, + "id": { + "type": "string", + "description": "The ID of the comment.", + "readOnly": true + }, + "author": { + "description": "The ID of the user who created the comment.", + "readOnly": true + }, + "body": { + "description": "The comment text in [Atlassian Document Format](https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/)." + }, + "renderedBody": { + "type": "string", + "description": "The rendered version of the comment.", + "readOnly": true + }, + "updateAuthor": { + "description": "The ID of the user who updated the comment last.", + "readOnly": true + }, + "created": { + "type": "string", + "description": "The date and time at which the comment was created.", + "format": "date-time", + "readOnly": true + }, + "updated": { + "type": "string", + "description": "The date and time at which the comment was updated last.", + "format": "date-time", + "readOnly": true + }, + "visibility": { + "description": "The group or role to which this comment is visible. Optional on create and update." + }, + "jsdPublic": { + "type": "boolean", + "description": "Whether the comment is visible in Jira Service Desk. Defaults to true when comments are created in the Jira Cloud Platform. This includes when the site doesn't use Jira Service Desk or the project isn't a Jira Service Desk project and, therefore, there is no Jira Service Desk for the issue to be visible on. To create a comment with its visibility in Jira Service Desk set to false, use the Jira Service Desk REST API [Create request comment](https://developer.atlassian.com/cloud/jira/service-desk/rest/#api-rest-servicedeskapi-request-issueIdOrKey-comment-post) operation.", + "readOnly": true + }, + "properties": { + "type": "array", + "description": "A list of comment properties. Optional on create and update." + } + }, + "additionalProperties": true, + "description": "A comment." + }, + "supported_sync_modes": ["full_refresh"] + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-jira/sample_files/sample_config.json b/airbyte-integrations/connectors/source-jira/sample_files/sample_config.json new file mode 100644 index 00000000000..e1de201bee1 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/sample_files/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_token": "", + "domain": "", + "email": "" +} diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py new file mode 100644 index 00000000000..b154c3c08f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/setup.py @@ -0,0 +1,44 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from setuptools import find_packages, setup + +setup( + name="source_jira", + description="Source implementation for Jira.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=[ + "airbyte-protocol", + "base-python", + "requests", + ], + package_data={"": ["*.json", "schemas/*.json"]}, + setup_requires=["pytest-runner"], + tests_require=["pytest"], + extras_require={ + "tests": ["airbyte_python_test", "pytest"], + }, +) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/__init__.py b/airbyte-integrations/connectors/source-jira/source_jira/__init__.py new file mode 100644 index 00000000000..1034c6db113 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceJira + +__all__ = ["SourceJira"] diff --git a/airbyte-integrations/connectors/source-jira/source_jira/client.py b/airbyte-integrations/connectors/source-jira/source_jira/client.py new file mode 100644 index 00000000000..1a31649bd3d --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/client.py @@ -0,0 +1,90 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import operator +from functools import partial, reduce +from json.decoder import JSONDecodeError +from typing import Mapping, Tuple + +import requests +from base_python import BaseClient +from requests.auth import HTTPBasicAuth +from requests.exceptions import ConnectionError + + +class Client(BaseClient): + """ + Jira API Reference: https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/ + """ + + API_VERSION = 3 + DEFAULT_ITEMS_PER_PAGE = 100 + + PARAMS = {"maxResults": DEFAULT_ITEMS_PER_PAGE, "startAt": 0} + ENTITIES_MAP = { + "projects": {"url": "/project/search", "func": lambda v: v["values"], "params": PARAMS}, + "issues": {"url": "/search", "func": lambda v: v["issues"], "params": PARAMS}, + "issue_comments": { + "url": "/search", + "func": lambda v: reduce(operator.iadd, [obj["fields"]["comment"]["comments"] for obj in v["issues"]], []), + "params": {**PARAMS, **{"fields": ["comment"]}}, + }, + "users": {"url": "/users/search", "func": lambda v: v, "params": PARAMS}, + "resolutions": {"url": "/resolution", "func": lambda v: v, "params": {}}, + } + + def __init__(self, api_token, domain, email): + self.auth = HTTPBasicAuth(email, api_token) + self.base_api_url = f"https://{domain}/rest/api/{self.API_VERSION}" + super().__init__() + + def lists(self, name, url, params, func, **kwargs): + while True: + response = requests.get(f"{self.base_api_url}{url}", params=params, auth=self.auth) + data = func(response.json()) + yield from data + if name == "resolutions" or len(data) < self.DEFAULT_ITEMS_PER_PAGE: + break + params["startAt"] += self.DEFAULT_ITEMS_PER_PAGE + + def _enumerate_methods(self) -> Mapping[str, callable]: + return {entity: partial(self.lists, name=entity, **value) for entity, value in self.ENTITIES_MAP.items()} + + def health_check(self) -> Tuple[bool, str]: + alive = True + error_msg = None + + try: + next(self.lists(name="resolutions", **self.ENTITIES_MAP["resolutions"])) + + except ConnectionError as error: + alive, error_msg = False, str(error) + # If the input domain is incorrect or doesn't exist, then the response would be empty, resulting in a JSONDecodeError + except JSONDecodeError: + alive, error_msg = ( + False, + "Unable to connect to the Jira API with the provided credentials. Please make sure the input credentials and environment are correct.", + ) + + return alive, error_msg diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json new file mode 100644 index 00000000000..da07f901462 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json @@ -0,0 +1,58 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the comment.", + "readOnly": true + }, + "id": { + "type": "string", + "description": "The ID of the comment.", + "readOnly": true + }, + "author": { + "description": "The ID of the user who created the comment.", + "readOnly": true + }, + "body": { + "description": "The comment text in [Atlassian Document Format](https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/)." + }, + "renderedBody": { + "type": "string", + "description": "The rendered version of the comment.", + "readOnly": true + }, + "updateAuthor": { + "description": "The ID of the user who updated the comment last.", + "readOnly": true + }, + "created": { + "type": "string", + "description": "The date and time at which the comment was created.", + "format": "date-time", + "readOnly": true + }, + "updated": { + "type": "string", + "description": "The date and time at which the comment was updated last.", + "format": "date-time", + "readOnly": true + }, + "visibility": { + "description": "The group or role to which this comment is visible. Optional on create and update." + }, + "jsdPublic": { + "type": "boolean", + "description": "Whether the comment is visible in Jira Service Desk. Defaults to true when comments are created in the Jira Cloud Platform. This includes when the site doesn't use Jira Service Desk or the project isn't a Jira Service Desk project and, therefore, there is no Jira Service Desk for the issue to be visible on. To create a comment with its visibility in Jira Service Desk set to false, use the Jira Service Desk REST API [Create request comment](https://developer.atlassian.com/cloud/jira/service-desk/rest/#api-rest-servicedeskapi-request-issueIdOrKey-comment-post) operation.", + "readOnly": true + }, + "properties": { + "type": "array", + "description": "A list of comment properties. Optional on create and update." + } + }, + "additionalProperties": true, + "description": "A comment." +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json new file mode 100644 index 00000000000..ad552792b8a --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json @@ -0,0 +1,97 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expand": { + "type": "string", + "description": "Expand options that include additional issue details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + }, + "id": { + "type": "string", + "description": "The ID of the issue.", + "readOnly": true + }, + "self": { + "type": "string", + "description": "The URL of the issue details.", + "format": "uri", + "readOnly": true + }, + "key": { + "type": "string", + "description": "The key of the issue.", + "readOnly": true + }, + "renderedFields": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "The rendered value of each field present on the issue.", + "readOnly": true + }, + "properties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "Details of the issue properties identified in the request.", + "readOnly": true + }, + "names": { + "type": "object", + "additionalProperties": { + "type": "string", + "readOnly": true + }, + "description": "The ID and name of each field present on the issue.", + "readOnly": true + }, + "schema": { + "type": "object", + "description": "The schema describing each field present on the issue.", + "readOnly": true + }, + "transitions": { + "type": "array", + "description": "The transitions that can be performed on the issue.", + "readOnly": true + }, + "operations": { + "description": "The operations that can be performed on the issue.", + "readOnly": true + }, + "editmeta": { + "description": "The metadata for the fields on the issue that can be amended.", + "readOnly": true + }, + "changelog": { + "description": "Details of changelogs associated with the issue.", + "readOnly": true + }, + "versionedRepresentations": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "readOnly": true + }, + "description": "The versions of each field on the issue.", + "readOnly": true + }, + "fieldsToInclude": { + "type": "object" + }, + "fields": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": false +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/projects.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/projects.json new file mode 100644 index 00000000000..de55c0f165a --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/projects.json @@ -0,0 +1,181 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expand": { + "type": "string", + "description": "Expand options that include additional project details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + }, + "self": { + "type": "string", + "description": "The URL of the project details.", + "format": "uri", + "readOnly": true + }, + "id": { + "type": "string", + "description": "The ID of the project." + }, + "key": { + "type": "string", + "description": "The key of the project.", + "readOnly": true + }, + "description": { + "type": "string", + "description": "A brief description of the project.", + "readOnly": true + }, + "lead": { + "description": "The username of the project lead.", + "readOnly": true + }, + "components": { + "type": "array", + "description": "List of the components contained in the project.", + "readOnly": true + }, + "issueTypes": { + "type": "array", + "description": "List of the issue types available in the project.", + "readOnly": true + }, + "url": { + "type": "string", + "description": "A link to information about this project, such as project documentation.", + "readOnly": true + }, + "email": { + "type": "string", + "description": "An email address associated with the project." + }, + "assigneeType": { + "type": "string", + "description": "The default assignee when creating issues for this project.", + "readOnly": true, + "enum": ["PROJECT_LEAD", "UNASSIGNED"] + }, + "versions": { + "type": "array", + "description": "The versions defined in the project. For more information, see [Create version](#api-rest-api-3-version-post).", + "readOnly": true + }, + "name": { + "type": "string", + "description": "The name of the project.", + "readOnly": true + }, + "roles": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "readOnly": true + }, + "description": "The name and self URL for each role defined in the project. For more information, see [Create project role](#api-rest-api-3-role-post).", + "readOnly": true + }, + "avatarUrls": { + "description": "The URLs of the project's avatars.", + "readOnly": true + }, + "projectCategory": { + "description": "The category the project belongs to.", + "readOnly": true + }, + "projectTypeKey": { + "type": "string", + "description": "The [project type](https://confluence.atlassian.com/x/GwiiLQ#Jiraapplicationsoverview-Productfeaturesandprojecttypes) of the project.", + "readOnly": true, + "enum": ["software", "service_desk", "business"] + }, + "simplified": { + "type": "boolean", + "description": "Whether the project is simplified.", + "readOnly": true + }, + "style": { + "type": "string", + "description": "The type of the project.", + "readOnly": true, + "enum": ["classic", "next-gen"] + }, + "favourite": { + "type": "boolean", + "description": "Whether the project is selected as a favorite." + }, + "isPrivate": { + "type": "boolean", + "description": "Whether the project is private.", + "readOnly": true + }, + "issueTypeHierarchy": { + "description": "The issue type hierarchy for the project", + "readOnly": true + }, + "permissions": { + "description": "User permissions on the project", + "readOnly": true + }, + "properties": { + "type": "object", + "additionalProperties": { + "readOnly": true + }, + "description": "Map of project properties", + "readOnly": true + }, + "uuid": { + "type": "string", + "description": "Unique ID for next-gen projects.", + "format": "uuid", + "readOnly": true + }, + "insight": { + "description": "Insights about the project.", + "readOnly": true + }, + "deleted": { + "type": "boolean", + "description": "Whether the project is marked as deleted.", + "readOnly": true + }, + "retentionTillDate": { + "type": "string", + "description": "The date when the project is deleted permanently.", + "format": "date-time", + "readOnly": true + }, + "deletedDate": { + "type": "string", + "description": "The date when the project was marked as deleted.", + "format": "date-time", + "readOnly": true + }, + "deletedBy": { + "description": "The user who marked the project as deleted.", + "readOnly": true + }, + "archived": { + "type": "boolean", + "description": "Whether the project is archived.", + "readOnly": true + }, + "archivedDate": { + "type": "string", + "description": "The date when the project was archived.", + "format": "date-time", + "readOnly": true + }, + "archivedBy": { + "description": "The user who archived the project.", + "readOnly": true + } + }, + "additionalProperties": false, + "description": "Details about a project." +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/resolutions.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/resolutions.json new file mode 100644 index 00000000000..3c08918f588 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/resolutions.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the issue resolution.", + "format": "uri" + }, + "id": { + "type": "string", + "description": "The ID of the issue resolution." + }, + "description": { + "type": "string", + "description": "The description of the issue resolution." + }, + "name": { + "type": "string", + "description": "The name of the issue resolution." + } + }, + "additionalProperties": false, + "description": "Details of an issue resolution." +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/users.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/users.json new file mode 100644 index 00000000000..5cfbee3863e --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/users.json @@ -0,0 +1,78 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "self": { + "type": "string", + "description": "The URL of the user.", + "format": "uri", + "readOnly": true + }, + "key": { + "type": "string", + "description": "This property is no longer available and will be removed from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/) for details." + }, + "accountId": { + "maxLength": 128, + "type": "string", + "description": "The account ID of the user, which uniquely identifies the user across all Atlassian products. For example, *5b10ac8d82e05b22cc7d4ef5*. Required in requests." + }, + "accountType": { + "type": "string", + "description": "The user account type. Can take the following values:\n\n * `atlassian` regular Atlassian user account\n * `app` system account used for Connect applications and OAuth to represent external systems\n * `customer` Jira Service Desk account representing an external service desk", + "readOnly": true, + "enum": ["atlassian", "app", "customer", "unknown"] + }, + "name": { + "type": "string", + "description": "This property is no longer available and will be removed from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/) for details." + }, + "emailAddress": { + "type": "string", + "description": "The email address of the user. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "avatarUrls": { + "description": "The avatars of the user.", + "readOnly": true + }, + "displayName": { + "type": "string", + "description": "The display name of the user. Depending on the user’s privacy setting, this may return an alternative value.", + "readOnly": true + }, + "active": { + "type": "boolean", + "description": "Whether the user is active.", + "readOnly": true + }, + "timeZone": { + "type": "string", + "description": "The time zone specified in the user's profile. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "locale": { + "type": "string", + "description": "The locale of the user. Depending on the user’s privacy setting, this may be returned as null.", + "readOnly": true + }, + "groups": { + "description": "The groups that the user belongs to.", + "readOnly": true + }, + "applicationRoles": { + "description": "The application roles the user is assigned to.", + "readOnly": true + }, + "expand": { + "type": "string", + "description": "Expand options that include additional user details in the response.", + "readOnly": true, + "xml": { + "attribute": true + } + } + }, + "additionalProperties": false, + "description": "A user with details as permitted by the user's Atlassian Account privacy settings. However, be aware of these exceptions:\n\n * User record deleted from Atlassian: This occurs as the result of a right to be forgotten request. In this case, `displayName` provides an indication and other parameters have default values or are blank (for example, email is blank).\n * User record corrupted: This occurs as a results of events such as a server import and can only happen to deleted users. In this case, `accountId` returns *unknown* and all other parameters have fallback values.\n * User record unavailable: This usually occurs due to an internal service outage. In this case, all parameters have fallback values." +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py new file mode 100644 index 00000000000..dd5c37974c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -0,0 +1,31 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from base_python import BaseSource + +from .client import Client + + +class SourceJira(BaseSource): + client_class = Client diff --git a/airbyte-integrations/connectors/source-jira/source_jira/spec.json b/airbyte-integrations/connectors/source-jira/source_jira/spec.json new file mode 100644 index 00000000000..769c4db3446 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/spec.json @@ -0,0 +1,27 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/jira", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Jira Spec", + "type": "object", + "required": ["api_token", "domain", "email"], + "additionalProperties": false, + "properties": { + "api_token": { + "type": "string", + "description": "Jira API Token. See the docs for more information on how to generate this key.", + "airbyte_secret": true + }, + "domain": { + "type": "string", + "examples": ["domainname.atlassian.net"], + "pattern": ["^[a-zA-Z0-9._-]*\\.atlassian\\.net$"], + "description": "Domain for your Jira account, e.g. airbyteio.atlassian.net" + }, + "email": { + "type": "string", + "description": "The user email for your Jira account" + } + } + } +} diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 433d4fc7a8e..70d1803625e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -23,6 +23,7 @@ * [Greenhouse](integrations/sources/greenhouse.md) * [Hubspot](integrations/sources/hubspot.md) * [HTTP Request](integrations/sources/http-request.md) + * [Jira](integrations/sources/jira.md) * [Mailchimp](integrations/sources/mailchimp.md) * [Marketo](integrations/sources/marketo.md) * [Mixpanel](integrations/sources/mixpanel.md) diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md new file mode 100644 index 00000000000..51fe804327d --- /dev/null +++ b/docs/integrations/sources/jira.md @@ -0,0 +1,42 @@ +# Jira + +## Overview + +The Jira source supports Full Refresh syncs. That is, every time a sync is run, Airbyte will copy all rows in the tables and columns you set up for replication into the destination in a new table. + +### Output schema + +Several output streams are available from this source: + +* [Projects](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-projects/#api-group-projects) +* [Issues](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issues/#api-group-issues) +* [Issue comments](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-comments/#api-group-issue-comments) +* [Resolutions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-resolutions/#api-group-issue-resolutions) +* [Users](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-users/#api-group-users) + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | + +### Performance considerations + +The Jira connector should not run into Jira API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Jira API Token +* Jira Email +* Jira Domain + +### Setup guide + +Please follow the [Jira confluence for generating an API token](https://confluence.atlassian.com/cloud/api-tokens-938839638.html). diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 2f33bf2fcca..457ba7996dd 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -36,6 +36,7 @@ write_standard_creds source-google-sheets "$GSHEETS_INTEGRATION_TESTS_CREDS" "cr write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS" write_standard_creds source-hubspot-singer "$HUBSPOT_INTEGRATION_TESTS_CREDS" write_standard_creds source-intercom-singer "$INTERCOM_INTEGRATION_TEST_CREDS" +write_standard_creds source-jira "$JIRA_INTEGRATION_TEST_CREDS" write_standard_creds source-mailchimp "$MAILCHIMP_TEST_CREDS" write_standard_creds source-marketo-singer "$SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG" write_standard_creds source-mixpanel-singer "$MIXPANEL_INTEGRATION_TEST_CREDS"