1
0
mirror of synced 2025-12-23 21:03:15 -05:00

🎉 New Source: Coin API (#18302)

* Initial commit

* Add streams

* Correct acceptance test config

* Make secret

* Update docsurl

* Remove catalog.json

* Add second stream

* Add default to spec

* Change to start_date and end_date

* Add links to CoinAPI docs in spec

* Change to start_date and end_date

* Add docs

* add coin api to source seed file

* remove icon

* generate source spec for coin api

Co-authored-by: Vincent Koc <koconder@users.noreply.github.com>
Co-authored-by: marcosmarxm <marcosmarxm@gmail.com>
This commit is contained in:
Isaac Harris-Holt
2022-10-27 23:08:41 +01:00
committed by GitHub
parent aea4c5392c
commit 13b4a26a5d
26 changed files with 644 additions and 0 deletions

View File

@@ -211,6 +211,13 @@
icon: cockroachdb.svg
sourceType: database
releaseStage: alpha
- name: Coin API
sourceDefinitionId: 919984ef-53a2-479b-8ffe-9c1ddb9fc3f3
dockerRepository: airbyte/source-coin-api
dockerImageTag: 0.1.0
documentationUrl: https://docs.airbyte.com/integrations/sources/coin-api
sourceType: api
releaseStage: alpha
- name: Commercetools
sourceDefinitionId: 008b2e26-11a3-11ec-82a8-0242ac130003
dockerRepository: airbyte/source-commercetools

View File

@@ -2194,6 +2194,69 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-coin-api:0.1.0"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/coin-api"
connectionSpecification:
$schema: "http://json-schema.org/draft-07/schema#"
title: "Coin API Spec"
type: "object"
required:
- "api_key"
- "environment"
- "symbol_id"
- "period"
- "start_date"
properties:
api_key:
type: "string"
description: "API Key"
airbyte_secret: true
order: 0
environment:
type: "string"
description: "The environment to use. Either sandbox or production.\n"
enum:
- "sandbox"
- "production"
default: "sandbox"
order: 1
symbol_id:
type: "string"
description: "The symbol ID to use. See the documentation for a list.\n\
https://docs.coinapi.io/#list-all-symbols-get\n"
order: 2
period:
type: "string"
description: "The period to use. See the documentation for a list. https://docs.coinapi.io/#list-all-periods-get"
examples:
- "5SEC"
- "2MTH"
start_date:
type: "string"
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"
description: "The start date in ISO 8601 format."
examples:
- "2019-01-01T00:00:00"
end_date:
type: "string"
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"
description: "The end date in ISO 8601 format. If not supplied, data will\
\ be returned\nfrom the start date to the current time, or when the count\
\ of result\nelements reaches its limit.\n"
examples:
- "2019-01-01T00:00:00"
limit:
type: "integer"
description: "The maximum number of elements to return. If not supplied,\
\ the default\nis 100. For numbers larger than 100, each 100 items is\
\ counted as one\nrequest for pricing purposes. Maximum value is 100000.\n"
minimum: 1
maximum: 100000
default: 100
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-commercetools:0.1.0"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/commercetools"

View File

@@ -0,0 +1,6 @@
*
!Dockerfile
!main.py
!source_coin_api
!setup.py
!secrets

View File

@@ -0,0 +1,38 @@
FROM python:3.9.11-alpine3.15 as base
# build and load all requirements
FROM base as builder
WORKDIR /airbyte/integration_code
# upgrade pip to the latest version
RUN apk --no-cache upgrade \
&& pip install --upgrade pip \
&& apk --no-cache add tzdata build-base
COPY setup.py ./
# install necessary packages to a temporary folder
RUN pip install --prefix=/install .
# build a clean environment
FROM base
WORKDIR /airbyte/integration_code
# copy all loaded and built libraries to a pure basic image
COPY --from=builder /install /usr/local
# add default timezone settings
COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
RUN echo "Etc/UTC" > /etc/timezone
# bash is installed for more convenient debugging.
RUN apk --no-cache add bash
# copy payload code only
COPY main.py ./
COPY source_coin_api ./source_coin_api
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/source-coin-api

View File

@@ -0,0 +1,79 @@
# Coin Api Source
This is the repository for the Coin Api configuration based source connector.
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/coin-api).
## Local development
#### Building via Gradle
You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow.
To build using Gradle, from the Airbyte repository root, run:
```
./gradlew :airbyte-integrations:connectors:source-coin-api:build
```
#### Create credentials
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coin-api)
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coin_api/spec.yaml` file.
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
See `integration_tests/sample_config.json` for a sample config file.
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source coin-api test creds`
and place them into `secrets/config.json`.
### Locally running the connector docker image
#### Build
First, make sure you build the latest Docker image:
```
docker build . -t airbyte/source-coin-api:dev
```
You can also build the connector image via Gradle:
```
./gradlew :airbyte-integrations:connectors:source-coin-api:airbyteDocker
```
When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
the Dockerfile.
#### Run
Then run any of the connector commands as follows:
```
docker run --rm airbyte/source-coin-api:dev spec
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coin-api:dev check --config /secrets/config.json
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coin-api:dev discover --config /secrets/config.json
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coin-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
```
## Testing
#### Acceptance Tests
Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information.
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
To run your integration tests with docker
### Using gradle to run tests
All commands should be run from airbyte project root.
To run unit tests:
```
./gradlew :airbyte-integrations:connectors:source-coin-api:unitTest
```
To run acceptance and custom integration tests:
```
./gradlew :airbyte-integrations:connectors:source-coin-api:integrationTest
```
## Dependency Management
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
We split dependencies between two groups, dependencies that are:
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
* required for the testing need to go to `TEST_REQUIREMENTS` list
### Publishing a new version of the connector
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
1. Make sure your changes are passing unit and integration tests.
1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)).
1. Create a Pull Request.
1. Pat yourself on the back for being an awesome contributor.
1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.

View File

@@ -0,0 +1,3 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#

View File

@@ -0,0 +1,20 @@
# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
# for more information about how to configure these tests
connector_image: airbyte/source-coin-api:dev
tests:
spec:
- spec_path: "source_coin_api/spec.yaml"
connection:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
discovery:
- config_path: "secrets/config.json"
basic_read:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
empty_streams: []
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"

View File

@@ -0,0 +1,16 @@
#!/usr/bin/env sh
# Build latest connector image
docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
# Pull latest acctest image
docker pull airbyte/source-acceptance-test:latest
# Run
docker run --rm -it \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /tmp:/tmp \
-v $(pwd):/test_input \
airbyte/source-acceptance-test \
--acceptance-test-config /test_input

View File

@@ -0,0 +1,9 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
id 'airbyte-source-acceptance-test'
}
airbytePython {
moduleDirectory 'source_coin_api'
}

View File

@@ -0,0 +1,3 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#

View File

@@ -0,0 +1,10 @@
{
"ohlcv_historical_data": {
"time_start": "2023-01-01T00:00:00",
"time_end": "2024-01-01T00:00:00"
},
"trades_historical_data": {
"time_start": "2023-01-01T00:00:00",
"time_end": "2024-01-01T00:00:00"
}
}

View File

@@ -0,0 +1,14 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
import pytest
pytest_plugins = ("source_acceptance_test.plugin",)
@pytest.fixture(scope="session", autouse=True)
def connector_setup():
"""This fixture is a placeholder for external resources that acceptance test might require."""
yield

View File

@@ -0,0 +1,22 @@
{
"streams": [
{
"stream": {
"name": "ohlcv_historical_data",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "trades_historical_data",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
}
]
}

View File

@@ -0,0 +1,8 @@
{
"api_key": "YOUR_API_KEY",
"environment": "production",
"symbol_id": "KRAKENFTS_PERP_BTC_USD",
"period": "SOME_PERIOD",
"start_date": "2025-01-01T00:00:00",
"limit": 100000
}

View File

@@ -0,0 +1,8 @@
{
"api_key": "YOUR_API_KEY",
"environment": "production",
"symbol_id": "KRAKENFTS_PERP_BTC_USD",
"period": "1DAY",
"start_date": "2021-01-01T00:00:00",
"limit": 1000
}

View File

@@ -0,0 +1,10 @@
{
"ohlcv_historical_data": {
"time_start": "2021-01-01T00:00:00",
"time_end": "2023-01-01T00:00:00"
},
"trades_historical_data": {
"time_start": "2021-01-01T00:00:00",
"time_end": "2023-01-01T00:00:00"
}
}

View File

@@ -0,0 +1,13 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
import sys
from airbyte_cdk.entrypoint import launch
from source_coin_api import SourceCoinApi
if __name__ == "__main__":
source = SourceCoinApi()
launch(source, sys.argv[1:])

View File

@@ -0,0 +1,2 @@
-e ../../bases/source-acceptance-test
-e .

View File

@@ -0,0 +1,29 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
from setuptools import find_packages, setup
MAIN_REQUIREMENTS = [
"airbyte-cdk~=0.1",
]
TEST_REQUIREMENTS = [
"pytest~=6.1",
"pytest-mock~=3.6.1",
"source-acceptance-test",
]
setup(
name="source_coin_api",
description="Source implementation for Coin Api.",
author="Airbyte",
author_email="contact@airbyte.io",
packages=find_packages(),
install_requires=MAIN_REQUIREMENTS,
package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]},
extras_require={
"tests": TEST_REQUIREMENTS,
},
)

View File

@@ -0,0 +1,8 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
from .source import SourceCoinApi
__all__ = ["SourceCoinApi"]

View File

@@ -0,0 +1,68 @@
version: "0.1.0"
definitions:
selector:
extractor:
field_pointer: []
requester:
url_base: "{{ 'https://rest.coinapi.io/v1' if config['environment'] == 'production' else 'https://rest-sandbox.coinapi.io/v1' }}"
http_method: "GET"
authenticator:
type: ApiKeyAuthenticator
header: "X-CoinAPI-Key"
api_token: "{{ config['api_key'] }}"
request_options_provider:
request_parameters:
period_id: "{{ config['period'] }}"
time_start: "{{ config['start_date'] }}"
time_end: "{{ config['end_date'] }}"
limit: "{{ config['limit'] }}"
stream_slicer:
type: DatetimeStreamSlicer
start_datetime:
datetime: "{{ config['start_time'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%S"
end_datetime:
datetime: "{{ config['end_time'] }}"
datetime_format: "%Y-%m-%dT%H:%M:%S"
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
step: 1d
start_time_option:
field_name: time_start
inject_into: request_parameter
end_time_option:
field_name: time_end
inject_into: request_parameter
retriever:
record_selector:
$ref: "*ref(definitions.selector)"
paginator:
type: NoPagination
requester:
$ref: "*ref(definitions.requester)"
base_stream:
retriever:
$ref: "*ref(definitions.retriever)"
ohlcv_historical_data_stream:
$ref: "*ref(definitions.base_stream)"
$options:
name: "ohlcv_historical_data"
primary_key: "time_period_start"
path: "/ohlcv/{{ config['symbol_id'] }}/history"
stream_cursor_field: "time_period_start"
trades_historical_data_stream:
$ref: "*ref(definitions.base_stream)"
$options:
name: "trades_historical_data"
primary_key: "uuid"
path: "/trades/{{ config['symbol_id'] }}/history"
stream_cursor_field: "time_exchange"
streams:
- "*ref(definitions.ohlcv_historical_data_stream)"
- "*ref(definitions.trades_historical_data_stream)"
check:
stream_names:
- "ohlcv_historical_data"

View File

@@ -0,0 +1,40 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"time_period_start": {
"type": ["null", "string"],
"format": "date-time"
},
"time_period_end": {
"type": ["null", "string"],
"format": "date-time"
},
"time_open": {
"type": ["null", "string"],
"format": "date-time"
},
"time_close": {
"type": ["null", "string"],
"format": "date-time"
},
"price_open": {
"type": ["null", "number"]
},
"price_high": {
"type": ["null", "number"]
},
"price_low": {
"type": ["null", "number"]
},
"price_close": {
"type": ["null", "number"]
},
"volume_traded": {
"type": ["null", "number"]
},
"trades_count": {
"type": ["null", "integer"]
}
}
}

View File

@@ -0,0 +1,33 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"symbol_id": {
"type": ["null", "string"]
},
"time_period_end": {
"type": ["null", "string"],
"format": "date-time"
},
"time_exchange": {
"type": ["null", "string"],
"format": "date-time"
},
"time_coinapi": {
"type": ["null", "string"],
"format": "date-time"
},
"uuid": {
"type": ["null", "string"]
},
"price": {
"type": ["null", "number"]
},
"size": {
"type": ["null", "number"]
},
"taker_side": {
"type": ["null", "string"]
}
}
}

View File

@@ -0,0 +1,18 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
"""
This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
source connector.
WARNING: Do not modify this file.
"""
# Declarative Source
class SourceCoinApi(YamlDeclarativeSource):
def __init__(self):
super().__init__(**{"path_to_yaml": "coin_api.yaml"})

View File

@@ -0,0 +1,64 @@
documentationUrl: https://docs.airbyte.com/integrations/sources/coin-api
connectionSpecification:
$schema: http://json-schema.org/draft-07/schema#
title: Coin API Spec
type: object
required:
- api_key
- environment
- symbol_id
- period
- start_date
properties:
api_key:
type: string
description: API Key
airbyte_secret: true
order: 0
environment:
type: string
description: |
The environment to use. Either sandbox or production.
enum:
- sandbox
- production
default: sandbox
order: 1
symbol_id:
type: string
description: |
The symbol ID to use. See the documentation for a list.
https://docs.coinapi.io/#list-all-symbols-get
order: 2
period:
type: string
description:
The period to use. See the documentation for a list.
https://docs.coinapi.io/#list-all-periods-get
examples:
- 5SEC
- 2MTH
start_date:
type: string
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"
description: The start date in ISO 8601 format.
examples:
- "2019-01-01T00:00:00"
end_date:
type: string
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"
description: |
The end date in ISO 8601 format. If not supplied, data will be returned
from the start date to the current time, or when the count of result
elements reaches its limit.
examples:
- "2019-01-01T00:00:00"
limit:
type: integer
description: |
The maximum number of elements to return. If not supplied, the default
is 100. For numbers larger than 100, each 100 items is counted as one
request for pricing purposes. Maximum value is 100000.
minimum: 1
maximum: 100000
default: 100

View File

@@ -0,0 +1,53 @@
# CoinAPI
## Sync overview
This source can sync OHLCV and trades historical data for a single coin listed on
[CoinAPI](https://www.coinapi.io/). It currently only supports Full Refresh
syncs.
### Output schema
This source is capable of syncing the following streams:
* `ohlcv_historical_data`
* `trades_historical_data`
### Features
| Feature | Supported? \(Yes/No\) | Notes |
|:------------------|:----------------------|:--------------------------------------------------------|
| Full Refresh Sync | Yes | |
| Incremental Sync | No | |
| API Environments | Yes | Both sandbox and production environments are supported. |
### Performance considerations
CoinAPI allows only 100 daily requests on the free plan. Use of this connector
may require a paid plan.
## Getting started
### Requirements
1. Obtain an API key from [CoinAPI](https://www.coinapi.io/).
2. Choose a symbol to pull data for. You can find a list of symbols [here](https://docs.coinapi.io/#list-all-symbols-get).
3. Choose a time interval to pull data for. You can find a list of intervals [here](https://docs.coinapi.io/#list-all-periods-get).
### Setup guide
The following fields are required fields for the connector to work:
- `api_key`: Your CoinAPI API key.
- `environment`: The environment to use. Can be either `sandbox` or `production`.
- `symbol_id`: The symbol to pull data for.
- `period`: The time interval to pull data for.
- `start_date`: The start date to pull `history` data from.
- (optional) `end_date`: The end date to pull `history` data until.
- (optional) `limit`: The maximum number of records to pull per request. Defaults to 100.
## Changelog
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:-----------|
| 0.1.0 | 2022-10-21 | [18302](https://github.com/airbytehq/airbyte/pull/18302) | New source |