🎉 Source Freshdesk: migrate to CDK

* Source Freshdesk #7764 - migrate to the CDK * Source Freshdesk #7764 - acceptance tests * Source Freshdesk #7764 - update state * Source Freshdesk #7764 - fixing according to PR * Source Freshdesk #7764 - fixing client according to PR * Source Freshdesk #7764 - bump version and update docs
2025-12-25 02:09:19 -05:00 · 2021-12-14 17:05:22 +02:00
parent 3c9fe46be5
commit ad448957c6
25 changed files with 195 additions and 56 deletions
--- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json
+++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json
@@ -2,7 +2,7 @@
  "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567",
  "name": "Freshdesk",
  "dockerRepository": "airbyte/source-freshdesk",
-  "dockerImageTag": "0.2.10",
+  "dockerImageTag": "0.2.11",
  "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshdesk",
  "icon": "freshdesk.svg"
 }
--- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
+++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -190,7 +190,7 @@
 - name: Freshdesk
  sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567
  dockerRepository: airbyte/source-freshdesk
-  dockerImageTag: 0.2.10
+  dockerImageTag: 0.2.11
  documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk
  icon: freshdesk.svg
  sourceType: api
--- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml
+++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1708,7 +1708,7 @@
    supportsNormalization: false
    supportsDBT: false
    supported_destination_sync_modes: []
- dockerImage: "airbyte/source-freshdesk:0.2.10"
+- dockerImage: "airbyte/source-freshdesk:0.2.11"
  spec:
    documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk"
    connectionSpecification:
--- a/airbyte-integrations/connectors/source-freshdesk/.dockerignore
+++ b/airbyte-integrations/connectors/source-freshdesk/.dockerignore
@@ -0,0 +1,6 @@
+*
+!Dockerfile
+!main.py
+!source_freshdesk
+!setup.py
+!secrets
--- a/airbyte-integrations/connectors/source-freshdesk/Dockerfile
+++ b/airbyte-integrations/connectors/source-freshdesk/Dockerfile
@@ -1,18 +1,38 @@
-FROM airbyte/integration-base-python:0.1.1
-
-# Bash is installed for more convenient debugging.
-RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
-
-ENV CODE_PATH="source_freshdesk"
-ENV AIRBYTE_IMPL_MODULE="source_freshdesk"
-ENV AIRBYTE_IMPL_PATH="SourceFreshdesk"
+FROM python:3.7.11-alpine3.14 as base

+# build and load all requirements
+FROM base as builder
 WORKDIR /airbyte/integration_code
-COPY $CODE_PATH ./$CODE_PATH
+
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
+
+
 COPY setup.py ./
-RUN pip install .
+# install necessary packages to a temporary folder
+RUN pip install --prefix=/install .

-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code

-LABEL io.airbyte.version=0.2.10
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY main.py ./
+COPY source_freshdesk ./source_freshdesk
+
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
+
+LABEL io.airbyte.version=0.2.11
 LABEL io.airbyte.name=airbyte/source-freshdesk
--- a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml
+++ b/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml
@@ -0,0 +1,21 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-freshdesk:dev
+tests:
+  spec:
+    - spec_path: "source_freshdesk/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "integration_tests/invalid_config.json"
+      status: "failed"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      empty_streams: ["satisfaction_ratings", "tickets", "time_entries", "conversations"]
+  incremental:
+    - config_path: "secrets/config.json"
+      future_state_path: "integration_tests/abnormal_state.json"
+  full_refresh:
+    - config_path: "secrets/config.json"
--- a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-docker.sh
+++ b/airbyte-integrations/connectors/source-freshdesk/acceptance-test-docker.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+
--- a/airbyte-integrations/connectors/source-freshdesk/build.gradle
+++ b/airbyte-integrations/connectors/source-freshdesk/build.gradle
@@ -1,29 +1,10 @@
 plugins {
    id 'airbyte-python'
    id 'airbyte-docker'
-    id 'airbyte-integration-test-java'
-    id 'airbyte-standard-source-test-file'
+    id 'airbyte-source-acceptance-test'
 }

 airbytePython {
    moduleDirectory 'source_freshdesk'
 }

-airbyteStandardSourceTestFile {
-    // For more information on standard source tests, see https://docs.airbyte.io/connector-development/testing-connectors
-    specPath = "source_freshdesk/spec.json"
-    configPath = "secrets/config.json"
-    configuredCatalogPath = "sample_files/configured_catalog.json"
-}
-
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
-    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
-}
-
-task("pythonIntegrationTests", type: PythonTask, dependsOn: installTestReqs) {
-    module = "pytest"
-    command = "-s integration_tests"
-}
-integrationTest.dependsOn("pythonIntegrationTests")
--- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/init.py
+++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/init.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
--- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/abnormal_state.json
+++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/abnormal_state.json
@@ -0,0 +1,35 @@
+{
+  "agents": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "companies": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "contacts": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "conversations": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "groups": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "roles": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "satisfaction_ratings": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "skills": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "surveys": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "tickets": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  },
+  "time_entries": {
+    "updated_at": "2121-11-01T00:00:00Z"
+  }
+}
--- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py
+++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py
@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """This fixture is a placeholder for external resources that acceptance test might require."""
+    yield
--- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/invalid_config.json
+++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/invalid_config.json
@@ -0,0 +1,4 @@
+{
+  "api_key": "wrong_api_key",
+  "domain": "wrongdomain.freshdesk.com"
+}
--- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_config.json
+++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_config.json
@@ -0,0 +1,4 @@
+{
+  "domain": "account123.freshdesk.com",
+  "api_key": "1234567890abcdefghijk"
+}
--- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_state.json
+++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_state.json
@@ -0,0 +1,35 @@
+{
+  "agents": {
+    "updated_at": "2021-01-01T00:00:00Z"
+  },
+  "companies": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "contacts": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "conversations": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "groups": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "roles": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "satisfaction_ratings": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "skills": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "surveys": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "tickets": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  },
+  "time_entries": {
+    "updated_at": "2021-11-01T00:00:00Z"
+  }
+}
--- a/airbyte-integrations/connectors/source-freshdesk/main_dev.py
+++ b/airbyte-integrations/connectors/source-freshdesk/main_dev.py
@@ -5,7 +5,7 @@

 import sys

-from base_python.entrypoint import launch
+from airbyte_cdk.entrypoint import launch
 from source_freshdesk import SourceFreshdesk

 if __name__ == "__main__":
--- a/airbyte-integrations/connectors/source-freshdesk/requirements.txt
+++ b/airbyte-integrations/connectors/source-freshdesk/requirements.txt
@@ -1,4 +1,3 @@
 # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
-e ../../bases/airbyte-protocol
-e ../../bases/base-python
+-e ../../bases/source-acceptance-test
 -e .
--- a/airbyte-integrations/connectors/source-freshdesk/setup.py
+++ b/airbyte-integrations/connectors/source-freshdesk/setup.py
@@ -6,8 +6,7 @@
 from setuptools import find_packages, setup

 MAIN_REQUIREMENTS = [
-    "airbyte-protocol",
-    "base-python",
+    "airbyte-cdk~=0.1",
    "backoff==1.10.0",
    "requests==2.25.1",
    "pendulum==2.1.2",
@@ -16,6 +15,7 @@ MAIN_REQUIREMENTS = [
 TEST_REQUIREMENTS = [
    "pytest==6.1.2",
    "requests_mock==1.8.0",
+    "source-acceptance-test",
 ]

 setup(
@@ -25,7 +25,7 @@ setup(
    author_email="contact@airbyte.io",
    packages=find_packages(),
    install_requires=MAIN_REQUIREMENTS,
-    package_data={"": ["*.json", "schemas/*.json"]},
+    package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]},
    extras_require={
        "tests": TEST_REQUIREMENTS,
    },
--- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py
+++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py
@@ -10,7 +10,7 @@ from typing import Any, Callable, Iterator, Mapping, MutableMapping, Optional, S

 import pendulum
 import requests
-from base_python.entrypoint import logger  # FIXME (Eugene K): use standard logger
+from airbyte_cdk.entrypoint import logger  # FIXME (Eugene K): use standard logger
 from requests import HTTPError
 from source_freshdesk.errors import (
    FreshdeskAccessDenied,
@@ -158,7 +158,7 @@ class IncrementalStreamAPI(StreamAPI, ABC):
    def state(self) -> Optional[Mapping[str, Any]]:
        """Current state, if wasn't set return None"""
        if self._state:
-            return {self.state_pk: str(self._state)}
+            return {self.state_pk: str(self._state).replace("+00:00", "Z")}
        return None

    @state.setter
--- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py
+++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py
@@ -5,8 +5,8 @@

 from typing import Any, Iterable, Mapping, Tuple

-from airbyte_protocol import AirbyteStream
-from base_python import BaseClient
+from airbyte_cdk.models import AirbyteStream
+from airbyte_cdk.sources.deprecated.client import BaseClient

 from .api import (
    API,
--- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/companies.json
+++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/companies.json
@@ -33,7 +33,7 @@
      "type": ["string", "null"]
    },
    "account_tier": {
-      "type": "string"
+      "type": ["string", "null"]
    },
    "renewal_date": {
      "type": ["string", "null"]
--- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/contacts.json
+++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/contacts.json
@@ -57,7 +57,7 @@
      "type": ["integer", "null"]
    },
    "preferred_source": {
-      "type": "string"
+      "type": ["string", "null"]
    },
    "unique_external_id": {
      "type": ["string", "null"]
--- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py
+++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py
@@ -3,7 +3,7 @@
 #


-from base_python import BaseSource
+from airbyte_cdk.sources.deprecated.base_source import BaseSource

 from .client import Client

--- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/spec.json
+++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/spec.json
@@ -12,7 +12,7 @@
        "description": "Freshdesk domain",
        "title": "Domain",
        "examples": ["myaccount.freshdesk.com"],
-        "pattern": ["^[a-zA-Z0-9._-]*\\.freshdesk\\.com$"]
+        "pattern": "^[a-zA-Z0-9._-]*\\.freshdesk\\.com$"
      },
      "api_key": {
        "type": "string",
--- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/utils.py
+++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/utils.py
@@ -8,7 +8,7 @@ import time

 import backoff
 import requests
-from base_python.entrypoint import logger
+from airbyte_cdk.entrypoint import logger
 from source_freshdesk.errors import FreshdeskRateLimited


--- a/docs/integrations/sources/freshdesk.md
+++ b/docs/integrations/sources/freshdesk.md
@@ -51,10 +51,11 @@ Please read [How to find your API key](https://support.freshdesk.com/support/sol

 ## Changelog

-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.2.10 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description                                      |
-| 0.2.9  | 2021-11-16 | [8017](https://github.com/airbytehq/airbyte/pull/8017) | Bugfix an issue that caused the connector not to sync more than 50000 contacts |
-| 0.2.8  | 2021-10-28 | [7486](https://github.com/airbytehq/airbyte/pull/7486) | Include "requester" and "stats" fields in "tickets" stream                     |
-| 0.2.7  | 2021-10-13 | [6442](https://github.com/airbytehq/airbyte/pull/6442) | Add start_date parameter to specification from which to start pulling data.    |
+| Version | Date       | Pull Request | Subject                                                                        |
+|:--------|:-----------| :--- |:-------------------------------------------------------------------------------|
+| 0.2.11  | 2021-12-14 | [8682](https://github.com/airbytehq/airbyte/pull/8682) | Migrate to the CDK                                                             |
+| 0.2.10  | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description                                      |
+| 0.2.9   | 2021-11-16 | [8017](https://github.com/airbytehq/airbyte/pull/8017) | Bugfix an issue that caused the connector not to sync more than 50000 contacts |
+| 0.2.8   | 2021-10-28 | [7486](https://github.com/airbytehq/airbyte/pull/7486) | Include "requester" and "stats" fields in "tickets" stream                     |
+| 0.2.7   | 2021-10-13 | [6442](https://github.com/airbytehq/airbyte/pull/6442) | Add start_date parameter to specification from which to start pulling data.    |