diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f54766b8ec2..67ffc467703 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1434,11 +1434,14 @@ - name: Paystack sourceDefinitionId: 193bdcb8-1dd9-48d1-aade-91cadfd74f9b dockerRepository: airbyte/source-paystack - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.com/integrations/sources/paystack icon: paystack.svg sourceType: api releaseStage: alpha + allowedHosts: + hosts: + - "api.paystack.co" - name: PersistIq sourceDefinitionId: 3052c77e-8b91-47e2-97a0-a29a22794b4b dockerRepository: airbyte/source-persistiq diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5fae93876dc..b3a432a9935 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11176,7 +11176,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-paystack:0.1.1" +- dockerImage: "airbyte/source-paystack:0.1.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/paystack" connectionSpecification: @@ -11186,7 +11186,7 @@ required: - "secret_key" - "start_date" - additionalProperties: false + additionalProperties: true properties: secret_key: type: "string" diff --git a/airbyte-integrations/connectors/source-paystack/Dockerfile b/airbyte-integrations/connectors/source-paystack/Dockerfile index f832721b0ef..7c2a1c849a8 100644 --- a/airbyte-integrations/connectors/source-paystack/Dockerfile +++ b/airbyte-integrations/connectors/source-paystack/Dockerfile @@ -34,5 +34,5 @@ COPY source_paystack ./source_paystack ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-paystack diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml index 78a5be12a22..3060b65087b 100644 --- a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml @@ -1,22 +1,35 @@ connector_image: airbyte/source-paystack:dev -tests: +acceptance_tests: spec: - - spec_path: "source_paystack/spec.json" + tests: + - spec_path: "source_paystack/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["disputes", "transfers", "settlements", "invoices"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: disputes + bypass_reason: "not filled yet" + - name: settlements + bypass_reason: "not filled yet" + - name: transfers + bypass_reason: "not filled yet" incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json b/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json index da8bb19f9ff..de31dba3d4e 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json @@ -5,7 +5,7 @@ "title": "Paystack Source Spec", "type": "object", "required": ["secret_key", "start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "secret_key": { "type": "string", diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py index 65ad299e721..0709340c5f6 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -9,6 +9,7 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import pendulum import requests from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer class PaystackStream(HttpStream, ABC): @@ -93,7 +94,7 @@ class IncrementalPaystackStream(PaystackStream, ABC): class Customers(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#customer-list + API docs: https://paystack.com/docs/api/customer#list """ cursor_field = "createdAt" @@ -104,7 +105,7 @@ class Customers(IncrementalPaystackStream): class Disputes(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#dispute-list + API docs: https://paystack.com/docs/api/dispute#list """ cursor_field = "createdAt" @@ -115,7 +116,7 @@ class Disputes(IncrementalPaystackStream): class Invoices(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#invoice-list + API docs: https://paystack.com/docs/api/payment-request """ cursor_field = "created_at" @@ -126,10 +127,11 @@ class Invoices(IncrementalPaystackStream): class Refunds(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#refund-list + API docs: https://paystack.com/docs/api/refund """ cursor_field = "createdAt" + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) def path(self, **kwargs) -> str: return "refund" @@ -137,7 +139,7 @@ class Refunds(IncrementalPaystackStream): class Settlements(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#settlement + API docs: https://paystack.com/docs/api/settlement """ cursor_field = "createdAt" @@ -148,7 +150,7 @@ class Settlements(IncrementalPaystackStream): class Subscriptions(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#subscription-list + API docs: https://paystack.com/docs/api/subscription """ cursor_field = "createdAt" @@ -159,7 +161,7 @@ class Subscriptions(IncrementalPaystackStream): class Transactions(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#transaction-list + API docs: https://paystack.com/docs/api/transaction """ cursor_field = "createdAt" @@ -170,7 +172,7 @@ class Transactions(IncrementalPaystackStream): class Transfers(IncrementalPaystackStream): """ - API docs: https://paystack.com/docs/api/#transfer-list + API docs: https://paystack.com/docs/api/transfer """ cursor_field = "createdAt" diff --git a/connectors.md b/connectors.md index ae8d76c4495..05a5a9f02cd 100644 --- a/connectors.md +++ b/connectors.md @@ -164,7 +164,7 @@ | **Pardot** | Pardot icon | Source | airbyte/source-pardot:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/pardot) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-pardot) | `ad15c7ba-72a7-440b-af15-b9a963dc1a8a` | | **PartnerStack** | PartnerStack icon | Source | airbyte/source-partnerstack:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/partnerstack) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-partnerstack) | `d30fb809-6456-484d-8e2c-ee12e0f6888d` | | **Paypal Transaction** | Paypal Transaction icon | Source | airbyte/source-paypal-transaction:0.1.12 | generally_available | [link](https://docs.airbyte.com/integrations/sources/paypal-transaction) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-paypal-transaction) | `d913b0f2-cc51-4e55-a44c-8ba1697b9239` | -| **Paystack** | Paystack icon | Source | airbyte/source-paystack:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/paystack) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-paystack) | `193bdcb8-1dd9-48d1-aade-91cadfd74f9b` | +| **Paystack** | Paystack icon | Source | airbyte/source-paystack:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/sources/paystack) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-paystack) | `193bdcb8-1dd9-48d1-aade-91cadfd74f9b` | | **PersistIq** | PersistIq icon | Source | airbyte/source-persistiq:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/persistiq) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-persistiq) | `3052c77e-8b91-47e2-97a0-a29a22794b4b` | | **Pexels API** | Pexels API icon | Source | airbyte/source-pexels-api:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/pexels-api) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-pexels-api) | `69d9eb65-8026-47dc-baf1-e4bf67901fd6` | | **Pinterest** | Pinterest icon | Source | airbyte/source-pinterest:0.2.3 | generally_available | [link](https://docs.airbyte.com/integrations/sources/pinterest) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-pinterest) | `5cb7e5fe-38c2-11ec-8d3d-0242ac130003` | diff --git a/docs/integrations/sources/paystack.md b/docs/integrations/sources/paystack.md index 858f5a0ddc9..92c6da2a335 100644 --- a/docs/integrations/sources/paystack.md +++ b/docs/integrations/sources/paystack.md @@ -1,21 +1,36 @@ # Paystack +This page contains the setup guide and reference information for the Paystack source connector. -## Overview +## Prerequisites +* Secret Key +* Start Day +* Lookback Window -The Paystack source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. +## Setup guide +### Step 1: Set up Paystack connector +1. Log into your [Airbyte Cloud](https://cloud.airbyte.io/workspaces) or Airbyte Open Source account. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Paystack** from the Source type dropdown. +4. Enter a name for your source. +5. For **Secret Key** enter your secret key. The Paystack API key usually starts with **'sk_live_'**. You can find yours secret key [here](https://dashboard.paystack.com/#/settings/developer). +6. For **Start Date** enter UTC date and time in the format **2017-01-25T00:00:00Z**. Any data before this date will not be replicated. +7. For **Lookback Window (in days)** enter the number of days. When set, the connector will always reload data from the past N days, where N is the value set here. This is useful if your data is updated after creation. -### Output schema +## Supported sync modes +The Paystack source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +* Full Refresh +* Incremental -This Source is capable of syncing the following core streams: +## Supported Streams -[Customers](https://paystack.com/docs/api/#customer-list) \(Incremental\) -[Disputes](https://paystack.com/docs/api/#dispute-list) \(Incremental\) -[Invoices](https://paystack.com/docs/api/#invoice-list) \(Incremental\) -[Refunds](https://paystack.com/docs/api/#refund-list) \(Incremental\) -[Settlements](https://paystack.com/docs/api/#settlement) \(Incremental\) -[Subscriptions](https://paystack.com/docs/api/#subscription-list) \(Incremental\) -[Transactions](https://paystack.com/docs/api/#transaction-list) \(Incremental\) -[Transfers](https://paystack.com/docs/api/#transfer-list) \(Incremental\) +* [Customers](https://paystack.com/docs/api/customer#list) \(Incremental\) +* [Disputes](https://paystack.com/docs/api/dispute) \(Incremental\) +* [Invoices](https://paystack.com/docs/api/payment-request) \(Incremental\) +* [Refunds](https://paystack.com/docs/api/refund) \(Incremental\) +* [Settlements](https://paystack.com/docs/api/settlement) \(Incremental\) +* [Subscriptions](https://paystack.com/docs/api/subscription) \(Incremental\) +* [Transactions](https://paystack.com/docs/api/transaction) \(Incremental\) +* [Transfers](https://paystack.com/docs/api/transfer) \(Incremental\) ### Note on Incremental Syncs @@ -23,41 +38,28 @@ The Paystack API does not allow querying objects which were updated since the la If your data is updated after creation, you can use the Loockback Window option when configuring the connector to always reload data from the past N days. This will allow you to pick up updates to the data. -### Data type mapping +## Data type map The [Paystack API](https://paystack.com/docs/api) is compatible with the [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions happen as part of this source. ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Incremental - Dedupe Sync | Yes | -| SSL connection | Yes | +| Feature | Supported? | +|:--------------------------|:-----------| +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Incremental - Dedupe Sync | Yes | +| SSL connection | Yes | ### Performance considerations The Paystack connector should not run into Paystack API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. -## Getting started - -### Requirements - -* Paystack API Secret Key - -### Setup guide - -Visit the [Paystack dashboard settings page](https://dashboard.paystack.com/#/settings/developer) with developer level access or more to see the secret key for your account. Secret keys for the live Paystack environment will be prefixed with `sk_live_`. - -Unfortunately Paystack does not yet support restricted permission levels on secret keys. This means that you will have to use the same secret key here that you use for charging customers. Use at your own risk. In the future Paystack might support restricted access levels and in that case Airbyte only requires a read-only access level key. - -If you would like to test Airbyte using test data on Paystack, `sk_test_` API keys are also supported. - ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | -| 0.1.0 | 2021-10-20 | [7214](https://github.com/airbytehq/airbyte/pull/7214) | Add Paystack source connector | \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------| +| 0.1.2 | 2023-03-15 | [24085](https://github.com/airbytehq/airbyte/pull/24085) | Set additionalProperties: true, add TypeTransformer to Refunds | +| 0.1.1 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | +| 0.1.0 | 2021-10-20 | [7214](https://github.com/airbytehq/airbyte/pull/7214) | Add Paystack source connector | \ No newline at end of file