1
0
mirror of synced 2025-12-25 02:09:19 -05:00

🎉 Source Zendesk Sunshine: support oauth (#7976)

* add oauth support

* bump version

* add java part

* change date format

* change spec

* upd invalid_configs

* bump version and format
This commit is contained in:
Anna Lvova
2022-01-04 22:14:57 +01:00
committed by GitHub
parent ee26499d7d
commit 7776dbec86
13 changed files with 492 additions and 36 deletions

View File

@@ -753,7 +753,7 @@
- name: Zendesk Sunshine
sourceDefinitionId: 325e0640-e7b3-4e24-b823-3361008f603f
dockerRepository: airbyte/source-zendesk-sunshine
dockerImageTag: 0.1.0
dockerImageTag: 0.1.1
documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-sunshine
icon: zendesk.svg
sourceType: api

View File

@@ -7553,7 +7553,7 @@
path_in_connector_config:
- "credentials"
- "client_secret"
- dockerImage: "airbyte/source-zendesk-sunshine:0.1.0"
- dockerImage: "airbyte/source-zendesk-sunshine:0.1.1"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk_sunshine"
connectionSpecification:
@@ -7561,32 +7561,129 @@
title: "Zendesk Sunshine Spec"
type: "object"
required:
- "api_token"
- "email"
- "start_date"
- "subdomain"
additionalProperties: false
additionalProperties: true
properties:
api_token:
type: "string"
airbyte_secret: true
description: "API Token. See the <a href=\"https://docs.airbyte.io/integrations/sources/zendesk_sunshine\"\
>docs</a> for information on how to generate this key."
email:
type: "string"
description: "The user email for your Zendesk account"
subdomain:
title: "Subdomain"
type: "string"
description: "The subdomain for your Zendesk Account"
description: "The subdomain for your Zendesk Account."
start_date:
title: "Start Date"
type: "string"
description: "The date from which you'd like to replicate the data"
description: "The date from which you'd like to replicate data for Zendesk\
\ Sunshine API, in the format YYYY-MM-DDT00:00:00Z."
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
examples: "2021-01-01T00:00:00.000000Z"
examples:
- "2021-01-01T00:00:00Z"
credentials:
title: "Authorization Method"
type: "object"
oneOf:
- type: "object"
title: "OAuth2.0"
required:
- "auth_method"
- "client_id"
- "client_secret"
- "access_token"
properties:
auth_method:
type: "string"
const: "oauth2.0"
enum:
- "oauth2.0"
default: "oauth2.0"
order: 0
client_id:
type: "string"
title: "Client ID"
description: "The Client ID of your OAuth application."
airbyte_secret: true
client_secret:
type: "string"
title: "Client Secret"
description: "The Client Secret of your OAuth application."
airbyte_secret: true
access_token:
type: "string"
title: "Access Token"
description: "Long-term access Token for making authenticated requests."
airbyte_secret: true
- type: "object"
title: "API Token"
required:
- "auth_method"
- "api_token"
- "email"
properties:
auth_method:
type: "string"
const: "api_token"
enum:
- "api_token"
default: "api_token"
order: 1
api_token:
type: "string"
title: "API Token"
description: "API Token. See the <a href=\"https://docs.airbyte.io/integrations/sources/zendesk_sunshine\"\
>docs</a> for information on how to generate this key."
airbyte_secret: true
email:
type: "string"
title: "Email"
description: "The user email for your Zendesk account"
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
advanced_auth:
auth_flow_type: "oauth2.0"
predicate_key:
- "credentials"
- "auth_method"
predicate_value: "oauth2.0"
oauth_config_specification:
oauth_user_input_from_connector_config_specification:
type: "object"
additionalProperties: false
properties:
subdomain:
type: "string"
path_in_connector_config:
- "subdomain"
complete_oauth_output_specification:
type: "object"
additionalProperties: false
properties:
access_token:
type: "string"
path_in_connector_config:
- "credentials"
- "access_token"
complete_oauth_server_input_specification:
type: "object"
additionalProperties: false
properties:
client_id:
type: "string"
client_secret:
type: "string"
complete_oauth_server_output_specification:
type: "object"
additionalProperties: false
properties:
client_id:
type: "string"
path_in_connector_config:
- "credentials"
- "client_id"
client_secret:
type: "string"
path_in_connector_config:
- "credentials"
- "client_secret"
- dockerImage: "airbyte/source-zendesk-support:0.1.11"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support"

View File

@@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.version=0.1.1
LABEL io.airbyte.name=airbyte/source-zendesk-sunshine

View File

@@ -7,13 +7,25 @@ tests:
connection:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "secrets/config_oauth.json"
status: "succeed"
- config_path: "secrets/config_api_token.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
- config_path: "integration_tests/invalid_config_api_token.json"
status: "failed"
- config_path: "integration_tests/invalid_config_oauth.json"
status: "failed"
discovery:
- config_path: "secrets/config.json"
basic_read:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
- config_path: "secrets/config_api_token.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
- config_path: "secrets/config_oauth.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
# incremental: # complex state ( {parent_id: {cur_field: value}} still not supported )
# - config_path: "secrets/config.json"
# configured_catalog_path: "integration_tests/configured_catalog.json"

View File

@@ -0,0 +1,9 @@
{
"credentials": {
"auth_method": "api_token",
"email": "test@ayhghghte.io",
"api_token": "fgfgvf ghnbvg hnghbvnhbvnvbn"
},
"subdomain": "d3v-airbyte",
"start_date": "2020-01-01T00:00:00Z"
}

View File

@@ -0,0 +1,10 @@
{
"credentials": {
"auth_method": "oauth2.0",
"client_id": "some_client_id",
"client_secret": "some_client_secret",
"access_token": "some_access_token"
},
"subdomain": "d3v-airbyte",
"start_date": "2020-01-01T00:00:00Z"
}

View File

@@ -4,7 +4,7 @@
import base64
from typing import Any, List, Mapping, Tuple
from typing import Any, List, Mapping, Tuple, Union
import pendulum
from airbyte_cdk.logger import AirbyteLogger
@@ -23,11 +23,24 @@ class Base64HttpAuthenticator(TokenAuthenticator):
super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs)
class ZendeskSunshineAuthenticator:
"""Provides the authentication capabilities for both old and new methods."""
@staticmethod
def get_auth(config: Mapping[str, Any]) -> Union[Base64HttpAuthenticator, TokenAuthenticator]:
credentials = config.get("credentials", {})
token = config.get("api_token") or credentials.get("api_token")
email = config.get("email") or credentials.get("email")
if email and token:
return Base64HttpAuthenticator(auth=(f"{email}/token", token))
return TokenAuthenticator(token=credentials["access_token"])
class SourceZendeskSunshine(AbstractSource):
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
try:
pendulum.parse(config["start_date"], strict=True)
authenticator = Base64HttpAuthenticator(auth=(f'{config["email"]}/token', config["api_token"]))
authenticator = ZendeskSunshineAuthenticator.get_auth(config)
stream = Limits(authenticator=authenticator, subdomain=config["subdomain"], start_date=pendulum.parse(config["start_date"]))
records = stream.read_records(sync_mode=SyncMode.full_refresh)
next(records)
@@ -47,7 +60,7 @@ class SourceZendeskSunshine(AbstractSource):
After this time is passed we have no data. It will require permanent population, to pass
the test criteria `stream should contain at least 1 record)
"""
authenticator = Base64HttpAuthenticator(auth=(f'{config["email"]}/token', config["api_token"]))
authenticator = ZendeskSunshineAuthenticator.get_auth(config)
args = {"authenticator": authenticator, "subdomain": config["subdomain"], "start_date": config["start_date"]}
return [
ObjectTypes(**args),

View File

@@ -4,28 +4,141 @@
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Zendesk Sunshine Spec",
"type": "object",
"required": ["api_token", "email", "start_date", "subdomain"],
"additionalProperties": false,
"required": ["start_date", "subdomain"],
"additionalProperties": true,
"properties": {
"api_token": {
"type": "string",
"airbyte_secret": true,
"description": "API Token. See the <a href=\"https://docs.airbyte.io/integrations/sources/zendesk_sunshine\">docs</a> for information on how to generate this key."
},
"email": {
"type": "string",
"description": "The user email for your Zendesk account"
},
"subdomain": {
"title": "Subdomain",
"type": "string",
"description": "The subdomain for your Zendesk Account"
"description": "The subdomain for your Zendesk Account."
},
"start_date": {
"title": "Start Date",
"type": "string",
"description": "The date from which you'd like to replicate the data",
"description": "The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z.",
"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
"examples": "2021-01-01T00:00:00.000000Z"
"examples": ["2021-01-01T00:00:00Z"]
},
"credentials": {
"title": "Authorization Method",
"type": "object",
"oneOf": [
{
"type": "object",
"title": "OAuth2.0",
"required": [
"auth_method",
"client_id",
"client_secret",
"access_token"
],
"properties": {
"auth_method": {
"type": "string",
"const": "oauth2.0",
"enum": ["oauth2.0"],
"default": "oauth2.0",
"order": 0
},
"client_id": {
"type": "string",
"title": "Client ID",
"description": "The Client ID of your OAuth application.",
"airbyte_secret": true
},
"client_secret": {
"type": "string",
"title": "Client Secret",
"description": "The Client Secret of your OAuth application.",
"airbyte_secret": true
},
"access_token": {
"type": "string",
"title": "Access Token",
"description": "Long-term access Token for making authenticated requests.",
"airbyte_secret": true
}
}
},
{
"type": "object",
"title": "API Token",
"required": ["auth_method", "api_token", "email"],
"properties": {
"auth_method": {
"type": "string",
"const": "api_token",
"enum": ["api_token"],
"default": "api_token",
"order": 1
},
"api_token": {
"type": "string",
"title": "API Token",
"description": "API Token. See the <a href=\"https://docs.airbyte.io/integrations/sources/zendesk_sunshine\">docs</a> for information on how to generate this key.",
"airbyte_secret": true
},
"email": {
"type": "string",
"title": "Email",
"description": "The user email for your Zendesk account"
}
}
}
]
}
}
},
"advanced_auth": {
"auth_flow_type": "oauth2.0",
"predicate_key": ["credentials", "auth_method"],
"predicate_value": "oauth2.0",
"oauth_config_specification": {
"complete_oauth_output_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"access_token": {
"type": "string",
"path_in_connector_config": ["credentials", "access_token"]
}
}
},
"complete_oauth_server_input_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"client_id": {
"type": "string"
},
"client_secret": {
"type": "string"
}
}
},
"complete_oauth_server_output_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"client_id": {
"type": "string",
"path_in_connector_config": ["credentials", "client_id"]
},
"client_secret": {
"type": "string",
"path_in_connector_config": ["credentials", "client_secret"]
}
}
},
"oauth_user_input_from_connector_config_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"subdomain": {
"type": "string",
"path_in_connector_config": ["subdomain"]
}
}
}
}
}

View File

@@ -77,6 +77,8 @@ class IncrementalSunshineStream(SunshineStream, ABC):
class ObjectTypes(SunshineStream):
primary_key = "key"
def path(self, **kwargs) -> str:
return "objects/types"
@@ -134,6 +136,8 @@ class ObjectRecords(IncrementalSunshineStream):
class RelationshipTypes(SunshineStream):
primary_key = "key"
def path(self, **kwargs) -> str:
return "relationships/types"
@@ -170,6 +174,8 @@ class CustomObjectEvents(SunshineStream):
class ObjectTypePolicies(SunshineStream):
primary_key = None
def stream_slices(self, **kwargs):
parent_stream = ObjectTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date)
for obj_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh):
@@ -200,5 +206,7 @@ class Jobs(SunshineStream):
class Limits(SunshineStream):
primary_key = "key"
def path(self, **kwargs) -> str:
return "limits"

View File

@@ -54,6 +54,7 @@ public class OAuthImplementationFactory {
.put("airbyte/source-drift", new DriftOAuthFlow(configRepository, httpClient))
.put("airbyte/source-zendesk-chat", new ZendeskChatOAuthFlow(configRepository, httpClient))
.put("airbyte/source-monday", new MondayOAuthFlow(configRepository, httpClient))
.put("airbyte/source-zendesk-sunshine", new ZendeskSunshineOAuthFlow(configRepository, httpClient))
.build();
}

View File

@@ -0,0 +1,100 @@
/*
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
*/
package io.airbyte.oauth.flows;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.oauth.BaseOAuth2Flow;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.apache.http.client.utils.URIBuilder;
/**
* Following docs from
* https://developer.zendesk.com/api-reference/custom-data/introduction/#authentication
*/
public class ZendeskSunshineOAuthFlow extends BaseOAuth2Flow {
public ZendeskSunshineOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) {
super(configRepository, httpClient);
}
@VisibleForTesting
public ZendeskSunshineOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier<String> stateSupplier) {
super(configRepository, httpClient, stateSupplier);
}
@Override
protected String formatConsentUrl(final UUID definitionId,
final String clientId,
final String redirectUrl,
final JsonNode inputOAuthConfiguration)
throws IOException {
// getting subdomain value from user's config
final String subdomain = getConfigValueUnsafe(inputOAuthConfiguration, "subdomain");
final URIBuilder builder = new URIBuilder()
.setScheme("https")
.setHost(String.format("%s.zendesk.com", subdomain))
.setPath("oauth/authorizations/new")
// required
.addParameter("response_type", "code")
.addParameter("redirect_uri", redirectUrl)
.addParameter("client_id", clientId)
.addParameter("scope", "read")
.addParameter("state", getState());
try {
return builder.build().toString();
} catch (final URISyntaxException e) {
throw new IOException("Failed to format Consent URL for OAuth flow", e);
}
}
@Override
protected Map<String, String> getAccessTokenQueryParameters(String clientId,
String clientSecret,
String authCode,
String redirectUrl) {
return ImmutableMap.<String, String>builder()
// required
.put("grant_type", "authorization_code")
.put("code", authCode)
.put("client_id", clientId)
.put("client_secret", clientSecret)
.put("redirect_uri", redirectUrl)
.put("scope", "read")
.build();
}
@Override
protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) {
// getting subdomain value from user's config
final String subdomain = getConfigValueUnsafe(inputOAuthConfiguration, "subdomain");
return String.format("https://%s.zendesk.com/oauth/tokens", subdomain);
}
@Override
protected Map<String, Object> extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException {
final Map<String, Object> result = new HashMap<>();
// getting out access_token
if (data.has("access_token")) {
result.put("access_token", data.get("access_token").asText());
} else {
throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl));
}
return result;
}
}

View File

@@ -0,0 +1,89 @@
/*
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
*/
package io.airbyte.oauth.flows;
import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.commons.json.Jsons;
import io.airbyte.oauth.BaseOAuthFlow;
import io.airbyte.oauth.MoreOAuthParameters;
import java.util.Map;
import org.junit.jupiter.api.Test;
public class ZendeskSunshineOAuthFlowTest extends BaseOAuthFlowTest {
@Override
protected BaseOAuthFlow getOAuthFlow() {
return new ZendeskSunshineOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState);
}
@Override
protected String getExpectedConsentUrl() {
return "https://test_subdomain.zendesk.com/oauth/authorizations/new?response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&client_id=test_client_id&scope=read&state=state";
}
@Override
protected JsonNode getInputOAuthConfiguration() {
return Jsons.jsonNode(Map.of("subdomain", "test_subdomain"));
}
@Override
protected JsonNode getUserInputFromConnectorConfigSpecification() {
return getJsonSchema(Map.of("subdomain", Map.of("type", "string")));
}
@Test
public void testEmptyOutputCompleteSourceOAuth() {}
@Test
public void testGetSourceConsentUrlEmptyOAuthSpec() {}
@Test
public void testValidateOAuthOutputFailure() {}
@Test
public void testCompleteSourceOAuth() {}
@Test
public void testEmptyInputCompleteDestinationOAuth() {}
@Test
public void testDeprecatedCompleteDestinationOAuth() {}
@Test
public void testDeprecatedCompleteSourceOAuth() {}
@Test
public void testEmptyOutputCompleteDestinationOAuth() {}
@Test
public void testCompleteDestinationOAuth() {}
@Test
public void testGetDestinationConsentUrlEmptyOAuthSpec() {}
@Test
public void testEmptyInputCompleteSourceOAuth() {}
@Override
protected Map<String, String> getExpectedOutput() {
return Map.of(
"access_token", "access_token_response",
"client_id", MoreOAuthParameters.SECRET_MASK,
"client_secret", MoreOAuthParameters.SECRET_MASK);
}
@Override
protected JsonNode getCompleteOAuthOutputSpecification() {
return getJsonSchema(Map.of("access_token", Map.of("type", "string")));
}
@Override
protected Map<String, String> getExpectedFilteredOutput() {
return Map.of(
"access_token", "access_token_response",
"client_id", MoreOAuthParameters.SECRET_MASK);
}
}

View File

@@ -47,13 +47,16 @@ The Zendesk connector should not run into Zendesk API limitations under normal u
### Requirements
* Zendesk Sunshine Access Token
* Zendesk Sunshine API Token
OR
* Zendesk Sunshine oauth2.0 application (client_id, client_secret, access_token)
### Setup guide
Please follow this [guide](https://developer.zendesk.com/documentation/custom-data/custom-objects/getting-started-with-custom-objects/#enabling-custom-objects)
Generate a Access Token as described in [here](https://developer.zendesk.com/api-reference/ticketing/introduction/#security-and-authentication)
Generate an API Token or oauth2.0 Access token as described in [here](https://developer.zendesk.com/api-reference/ticketing/introduction/#security-and-authentication)
We recommend creating a restricted, read-only key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access.
@@ -61,5 +64,6 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces
| Version | Date | Pull Request | Subject |
| :--- | :--- | :--- | :--- |
| 0.1.1 | 2021-11-15 | [7976](https://github.com/airbytehq/airbyte/pull/7976) | Add oauth2.0 support |
| 0.1.0 | 2021-07-08 | [4359](https://github.com/airbytehq/airbyte/pull/4359) | Initial Release |