diff --git a/.gitignore b/.gitignore index f02f5b77b68..45316ca315b 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ resources/examples/airflow/logs/* # Cloud Demo !airbyte-webapp/src/packages/cloud/data + +# Sphinx Docs +_build diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000000..73602b5138e --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,20 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: html + configuration: airbyte-cdk/python/reference_docs/_source/conf.py + +# Set the version of Python and requirements required to build your docs +python: + version: "3.7" + install: + - method: pip + path: airbyte-cdk/python + extra_requirements: + - sphinx-docs diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 33f56cdb010..f230eccf3fc 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.1.32 +- Added Sphinx docs `airbyte-cdk/python/reference_docs` module. +- Added module documents at `airbyte-cdk/python/sphinx-docs.md`. +- Added Read the Docs publishing configuration at `.readthedocs.yaml`. + ## 0.1.31 Transforming Python log levels to Airbyte protocol log levels diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index 5a782bb8d32..2ddbf80fcdd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -39,11 +39,13 @@ class AbstractSource(Source, ABC): @abstractmethod def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """ - :param config: The user-provided configuration as specified by the source's spec. This usually contains information required to check connection e.g. tokens, secrets and keys etc. - :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful and we can connect to the underlying data - source using the provided configuration. - Otherwise, the input config cannot be used to connect to the underlying data source, and the "error" object should describe what went wrong. - The error object will be cast to string to display the problem to the user. + :param config: The user-provided configuration as specified by the source's spec. + This usually contains information required to check connection e.g. tokens, secrets and keys etc. + :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful + and we can connect to the underlying data source using the provided configuration. + Otherwise, the input config cannot be used to connect to the underlying data source, + and the "error" object should describe what went wrong. + The error object will be cast to string to display the problem to the user. """ @abstractmethod diff --git a/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py index 6e02ca318ed..23d46482409 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py @@ -110,7 +110,7 @@ class SingerHelper: """ :param singer_catalog: :param sync_mode_overrides: A dict from stream name to the sync modes it should use. Each stream in this dict must exist in the Singer catalog, - but not every stream in the catalog should exist in this + but not every stream in the catalog should exist in this :param primary_key_overrides: A dict of stream name -> list of fields to be used as PKs. :return: Airbyte Catalog """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py index ba002b3ea40..5dd911c1505 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py @@ -106,7 +106,7 @@ class Stream(ABC): def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: """ :return: string if single primary key, list of strings if composite primary key, list of list of strings if composite primary key consisting of nested fields. - If the stream has no primary keys, return None. + If the stream has no primary keys, return None. """ def stream_slices( diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 77445df9211..8d883287340 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -165,7 +165,7 @@ class HttpStream(Stream, ABC): If returns a ready text that it will be sent as is. If returns a dict that it will be converted to a urlencoded form. - E.g. {"key1": "value1", "key2": "value2"} => "key1=value1&key2=value2" + E.g. {"key1": "value1", "key2": "value2"} => "key1=value1&key2=value2" At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden. """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py index 361a276d8c7..b29b83e2b89 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py @@ -52,6 +52,7 @@ class ResourceSchemaLoader: def __resolve_schema_references(self, raw_schema: dict) -> dict: """ Resolve links to external references and move it to local "definitions" map. + :param raw_schema jsonschema to lookup for external links. :return JSON serializable object with references without external dependencies. """ @@ -78,6 +79,7 @@ class ResourceSchemaLoader: """ Scan resolved schema and compose definitions section, also convert jsonref.JsonRef object to JSON serializable dict. + :param obj - jsonschema object with ref field resovled. :definitions - object for storing generated definitions. :return JSON serializable object with references without external dependencies. @@ -110,6 +112,7 @@ def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: Connector """ Check config object against spec. In case of spec is invalid, throws an exception with validation error description. + :param config - config loaded from file specified over command line :param spec - spec object generated by connector :param logger - Airbyte logger for reporting validation error @@ -135,8 +138,10 @@ def split_config(config: Mapping[str, Any]) -> Tuple[dict, InternalConfig]: Break config map object into 2 instances: first is a dict with user defined configuration and second is internal config that contains private keys for acceptance test configuration. + :param config - Dict object that has been loaded from config file. + :return tuple of user defined config dict with filtered out internal parameters and SAT internal config object. """ diff --git a/airbyte-cdk/python/reference_docs/Makefile b/airbyte-cdk/python/reference_docs/Makefile new file mode 100644 index 00000000000..c7f9263975e --- /dev/null +++ b/airbyte-cdk/python/reference_docs/Makefile @@ -0,0 +1,23 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = _source +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +rst: + sphinx-apidoc -f -F -o $(SOURCEDIR)/api ../airbyte_cdk diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.destinations.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.destinations.rst new file mode 100644 index 00000000000..e554319db2f --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.destinations.rst @@ -0,0 +1,19 @@ + +Submodules +---------- + +airbyte\_cdk.destinations.destination module +-------------------------------------------- + +.. automodule:: airbyte_cdk.destinations.destination + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.destinations + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.models.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.models.rst new file mode 100644 index 00000000000..14d3065dd76 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.models.rst @@ -0,0 +1,19 @@ + +Submodules +---------- + +airbyte\_cdk.models.airbyte\_protocol module +-------------------------------------------- + +.. automodule:: airbyte_cdk.models.airbyte_protocol + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.models + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.rst new file mode 100644 index 00000000000..7cc0db2ed60 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.rst @@ -0,0 +1,45 @@ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + airbyte_cdk.destinations + airbyte_cdk.models + airbyte_cdk.sources + +Submodules +---------- + +airbyte\_cdk.connector module +----------------------------- + +.. automodule:: airbyte_cdk.connector + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.entrypoint module +------------------------------ + +.. automodule:: airbyte_cdk.entrypoint + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.logger module +-------------------------- + +.. automodule:: airbyte_cdk.logger + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.deprecated.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.deprecated.rst new file mode 100644 index 00000000000..3d19a99f04f --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.deprecated.rst @@ -0,0 +1,27 @@ + +Submodules +---------- + +airbyte\_cdk.sources.deprecated.base\_source module +--------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.deprecated.base_source + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.deprecated.client module +--------------------------------------------- + +.. automodule:: airbyte_cdk.sources.deprecated.client + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources.deprecated + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.rst new file mode 100644 index 00000000000..6ae9aa5b4e2 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.rst @@ -0,0 +1,38 @@ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + airbyte_cdk.sources.deprecated + airbyte_cdk.sources.singer + airbyte_cdk.sources.streams + airbyte_cdk.sources.utils + +Submodules +---------- + +airbyte\_cdk.sources.abstract\_source module +-------------------------------------------- + +.. automodule:: airbyte_cdk.sources.abstract_source + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.source module +---------------------------------- + +.. automodule:: airbyte_cdk.sources.source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.singer.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.singer.rst new file mode 100644 index 00000000000..37af436cbc7 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.singer.rst @@ -0,0 +1,27 @@ + +Submodules +---------- + +airbyte\_cdk.sources.singer.singer\_helpers module +-------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.singer.singer_helpers + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.singer.source module +----------------------------------------- + +.. automodule:: airbyte_cdk.sources.singer.source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources.singer + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.auth.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.auth.rst new file mode 100644 index 00000000000..049da2b74e7 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.auth.rst @@ -0,0 +1,35 @@ + +Submodules +---------- + +airbyte\_cdk.sources.streams.http.auth.core module +-------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.auth.core + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.streams.http.auth.oauth module +--------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.auth.oauth + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.streams.http.auth.token module +--------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.auth.token + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources.streams.http.auth + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.requests_native_auth.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.requests_native_auth.rst new file mode 100644 index 00000000000..74f3e41827e --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.requests_native_auth.rst @@ -0,0 +1,27 @@ + +Submodules +---------- + +airbyte\_cdk.sources.streams.http.requests\_native\_auth.oauth module +--------------------------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.requests_native_auth.oauth + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.streams.http.requests\_native\_auth.token module +--------------------------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.requests_native_auth.token + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources.streams.http.requests_native_auth + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.rst new file mode 100644 index 00000000000..920db700aa9 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.rst @@ -0,0 +1,44 @@ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + airbyte_cdk.sources.streams.http.auth + airbyte_cdk.sources.streams.http.requests_native_auth + +Submodules +---------- + +airbyte\_cdk.sources.streams.http.exceptions module +--------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.exceptions + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.streams.http.http module +--------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.http + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.streams.http.rate\_limiting module +------------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.http.rate_limiting + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources.streams.http + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.rst new file mode 100644 index 00000000000..e00b5d0f36f --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.rst @@ -0,0 +1,27 @@ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + airbyte_cdk.sources.streams.http + +Submodules +---------- + +airbyte\_cdk.sources.streams.core module +---------------------------------------- + +.. automodule:: airbyte_cdk.sources.streams.core + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources.streams + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.utils.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.utils.rst new file mode 100644 index 00000000000..a435a52f166 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.utils.rst @@ -0,0 +1,43 @@ + +Submodules +---------- + +airbyte\_cdk.sources.utils.casing module +---------------------------------------- + +.. automodule:: airbyte_cdk.sources.utils.casing + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.utils.catalog\_helpers module +-------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.utils.catalog_helpers + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.utils.schema\_helpers module +------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.utils.schema_helpers + :members: + :undoc-members: + :show-inheritance: + +airbyte\_cdk.sources.utils.transform module +------------------------------------------- + +.. automodule:: airbyte_cdk.sources.utils.transform + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: airbyte_cdk.sources.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/modules.rst b/airbyte-cdk/python/reference_docs/_source/api/modules.rst new file mode 100644 index 00000000000..453898da5f3 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/api/modules.rst @@ -0,0 +1,7 @@ +airbyte_cdk +=========== + +.. toctree:: + :maxdepth: 4 + + airbyte_cdk diff --git a/airbyte-cdk/python/reference_docs/_source/conf.py b/airbyte-cdk/python/reference_docs/_source/conf.py new file mode 100644 index 00000000000..6415fe5195d --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/conf.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath("../..")) + + +# -- Project information ----------------------------------------------------- + +project = "Airbyte Connector Development Kit" +copyright = "2021, Airbyte" +author = "Airbyte" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["sphinx.ext.autodoc"] # API docs + +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] +html_logo = "../_static/img/airbyte_new_logo.svg" +html_theme_options = { + "logo_only": True, + "display_version": False, +} diff --git a/airbyte-cdk/python/reference_docs/_source/index.rst b/airbyte-cdk/python/reference_docs/_source/index.rst new file mode 100644 index 00000000000..eb0ede2d2ad --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/index.rst @@ -0,0 +1,30 @@ +Welcome to Airbyte Connector Development Kit's documentation! +============================================================= +This documentation is autogenerated from source code comments. More extensive overviews and conceptual explanations will be coming soon. +======================================================================================================================================== + +.. toctree:: + :maxdepth: 4 + :caption: Destinations + + api/airbyte_cdk.destinations + +.. toctree:: + :maxdepth: 4 + :caption: Models + + api/airbyte_cdk.models + +.. toctree:: + :maxdepth: 4 + :caption: Sources + + api/airbyte_cdk.sources + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/airbyte-cdk/python/reference_docs/_source/templates/master_doc.rst_t b/airbyte-cdk/python/reference_docs/_source/templates/master_doc.rst_t new file mode 100644 index 00000000000..082c0ce4ed8 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/templates/master_doc.rst_t @@ -0,0 +1,19 @@ +Welcome to Airbyte Connector Development Kit's documentation! +============================================================= +This documentation is autogenerated from source code comments. More extensive overviews and conceptual explanations will be coming soon. +======================================================================================================================================== +{% for module in top_modules %} +.. toctree:: + :maxdepth: {{ maxdepth }} + :caption: {{ module.caption }} + + {{module.path}} +{% endfor %} + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/airbyte-cdk/python/reference_docs/_source/templates/package.rst_t b/airbyte-cdk/python/reference_docs/_source/templates/package.rst_t new file mode 100644 index 00000000000..8c770031061 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_source/templates/package.rst_t @@ -0,0 +1,51 @@ +{%- macro automodule(modname, options) -%} +.. automodule:: {{ modname }} +{%- for option in options %} + :{{ option }}: +{%- endfor %} +{%- endmacro %} + +{%- macro toctree(docnames) -%} +.. toctree:: + :maxdepth: {{ maxdepth }} +{% for docname in docnames %} + {{ docname }} +{%- endfor %} +{%- endmacro %} + +{%- if is_namespace %} +.. py:module:: {{ pkgname }} +{% endif %} + +{%- if modulefirst and not is_namespace %} +{{ automodule(pkgname, automodule_options) }} +{% endif %} + +{%- if subpackages %} +Subpackages +----------- + +{{ toctree(subpackages) }} +{% endif %} + +{%- if submodules %} +Submodules +---------- +{% if separatemodules %} +{{ toctree(submodules) }} +{% else %} +{%- for submodule in submodules %} +{% if show_headings %} +{{- [submodule, "module"] | join(" ") | e | heading(2) }} +{% endif %} +{{ automodule(submodule, automodule_options) }} +{% endfor %} +{%- endif %} +{%- endif %} + +{%- if not modulefirst and not is_namespace %} +Module contents +--------------- + +{{ automodule(pkgname, automodule_options) }} +{% endif %} diff --git a/airbyte-cdk/python/reference_docs/_static/img/airbyte_new_logo.svg b/airbyte-cdk/python/reference_docs/_static/img/airbyte_new_logo.svg new file mode 100644 index 00000000000..463f23913c1 --- /dev/null +++ b/airbyte-cdk/python/reference_docs/_static/img/airbyte_new_logo.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/airbyte-cdk/python/reference_docs/generate_rst_schema.py b/airbyte-cdk/python/reference_docs/generate_rst_schema.py new file mode 100755 index 00000000000..6a9edb3a72d --- /dev/null +++ b/airbyte-cdk/python/reference_docs/generate_rst_schema.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import sys +from os import path +from typing import Any, Dict + +from sphinx.cmd.quickstart import QuickstartRenderer +from sphinx.ext.apidoc import get_parser, main, recurse_tree, write_file +from sphinx.locale import __ +from sphinx.util import ensuredir + + +def write_master_file(templatedir: str, master_name: str, values: Dict, opts: Any): + template = QuickstartRenderer(templatedir=templatedir) + opts.destdir = opts.destdir[: opts.destdir.rfind("/")] + write_file(master_name, template.render(f"{templatedir}/master_doc.rst_t", values), opts) + + +if __name__ == "__main__": + parser = get_parser() + parser.add_argument("--master", metavar="MASTER", default="index", help=__("master document name")) + args = parser.parse_args(sys.argv[1:]) + + rootpath = path.abspath(args.module_path) + + # normalize opts + if args.header is None: + args.header = rootpath.split(path.sep)[-1] + if args.suffix.startswith("."): + args.suffix = args.suffix[1:] + if not path.isdir(rootpath): + print(__(f"{rootpath} is not a directory."), file=sys.stderr) + sys.exit(1) + if not args.dryrun: + ensuredir(args.destdir) + excludes = [path.abspath(exclude) for exclude in args.exclude_pattern] + modules = recurse_tree(rootpath, excludes, args, args.templatedir) + + template_values = { + "top_modules": [{"path": f"api/{module}", "caption": module.split(".")[1].title()} for module in modules if module.count(".") == 1], + "maxdepth": args.maxdepth, + } + write_master_file(templatedir=args.templatedir, master_name=args.master, values=template_values, opts=args) + main() diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 392dc7d22fe..72fa8b3dd5e 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ README = (HERE / "README.md").read_text() setup( name="airbyte-cdk", - version="0.1.31", + version="0.1.32", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -56,7 +56,13 @@ setup( "Deprecated~=1.2", ], python_requires=">=3.7.0", - extras_require={"dev": ["MyPy~=0.812", "pytest", "pytest-cov", "pytest-mock", "requests-mock"]}, + extras_require={ + "dev": ["MyPy~=0.812", "pytest", "pytest-cov", "pytest-mock", "requests-mock"], + "sphinx-docs": [ + "Sphinx~=4.2", + "sphinx-rtd-theme~=1.0", + ], + }, entry_points={ "console_scripts": ["base-python=base_python.entrypoint:main"], }, diff --git a/airbyte-cdk/python/sphinx-docs.md b/airbyte-cdk/python/sphinx-docs.md new file mode 100644 index 00000000000..9cb2eae8e57 --- /dev/null +++ b/airbyte-cdk/python/sphinx-docs.md @@ -0,0 +1,94 @@ +# Sphinx Docs + +We're using the [Sphinx](https://www.sphinx-doc.org/) library in order +to automatically generate the docs for the [airbyte-cdk](https://pypi.org/project/airbyte-cdk/). + +## Updating the docs structure (manually) + +Documentation structure is set in `airbyte-cdk/python/reference_docs/_source`, using the `.rst` files. + +See [reStructuredText docs](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) +for the key concepts. + +Note that `index.rst` is the main index file, where we do define the layout of the main +docs page and relation to other sections. + +Each time a new module added to `airbyte-cdk/python/airbyte_cdk` module you'll need to update the Sphinx rst schema. + +Let's dive into using an example: +- Assuming we're going to add a new package `airbyte_cdk/new_package`; +- Let this file contain a few modules: `airbyte_cdk/new_package/module1.py` and `airbyte_cdk/new_package/module2.py`; +- The above structure should be in `rst` config as: + - Add this block directly into `index.rst`: + ``` + .. toctree:: + :maxdepth: 2 + :caption: New Package + + api/airbyte_cdk.new_package + ``` + - Add a new file `api/airbyte_cdk.new_package.rst` with the following content: + ``` + Submodules + ---------- + + airbyte\_cdk.new\_package.module1 module + -------------------------------------------- + + .. automodule:: airbyte_cdk.new_package.module1 + :members: + :undoc-members: + :show-inheritance: + + .. automodule:: airbyte_cdk.new_package.module2 + :members: + :undoc-members: + :show-inheritance: + + Module contents + --------------- + + .. automodule:: airbyte_cdk.models + :members: + :undoc-members: + :show-inheritance: + ``` + +For more examples see `airbyte-cdk/python/reference_docs/_source` +and read the [docs](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html). + +## Updating the docs structure (automatically) + +It's also possible to generate `.rst` files automatically using `generate_rst_schema.py` script. + +You should also update this script in order to change the docs appearance or structure. + +To generate the docs, +run `python generate_rst_schema.py -o _source/api ../../python/airbyte_cdk -f -t _source/templates` +from the `airbyte-cdk/python/reference_docs` root. + + +## Building the docs locally + +After the `rst` files created to correctly represent current project structure you may build the docs locally. +This build could be useful on each `airbyte-cdk` update, especially if the package structure was changed. + +- Install Sphinx deps with `pip install ".[sphinx-docs]"`; +- Run `make html` from the `airbyte-cdk/python/reference_docs` root; +- Check out the `airbyte-cdk/python/reference_docs/_build` for the new documentation built. + + +## Publishing to Read the Docs + +Our current sphinx docs setup is meant to be published to [readthedocs](https://readthedocs.org/). +So it may be useful to check our docs published at https://airbyte-cdk.readthedocs.io/en/latest/ +for the last build in case if the airbyte-cdk package was updated. + +Publishing process is automatic and implemented via the GitHub incoming webhook. +See https://docs.readthedocs.io/en/stable/webhooks.html. + +To check build logs and state, check the https://readthedocs.org/projects/airbyte-cdk/builds/. +You may also run build manually here if needed. + +Publishing configuration is placed to `.readthedocs.yaml`. +See https://docs.readthedocs.io/en/stable/config-file/v2.html for the config description. \ No newline at end of file