🎉 Source Mixpanel: Increase unit test coverage (#11633)

2025-12-25 02:09:19 -05:00 · 2022-04-15 16:52:43 +02:00
parent dbe2439598
commit e4a55a4fde
11 changed files with 378 additions and 18 deletions
--- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
+++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -473,7 +473,7 @@
 - name: Mixpanel
  sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a
  dockerRepository: airbyte/source-mixpanel
-  dockerImageTag: 0.1.11
+  dockerImageTag: 0.1.12
  documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel
  icon: mixpanel.svg
  sourceType: api
--- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml
+++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -5004,7 +5004,7 @@
              path_in_connector_config:
              - "credentials"
              - "client_secret"
- dockerImage: "airbyte/source-mixpanel:0.1.11"
+- dockerImage: "airbyte/source-mixpanel:0.1.12"
  spec:
    documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel"
    connectionSpecification:
--- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile
+++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile
@@ -12,5 +12,6 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

-LABEL io.airbyte.version=0.1.11
+
+LABEL io.airbyte.version=0.1.12
 LABEL io.airbyte.name=airbyte/source-mixpanel
--- a/airbyte-integrations/connectors/source-mixpanel/setup.py
+++ b/airbyte-integrations/connectors/source-mixpanel/setup.py
@@ -12,6 +12,8 @@ MAIN_REQUIREMENTS = [
 TEST_REQUIREMENTS = [
    "pytest~=6.1",
    "source-acceptance-test",
+    "pytest-mock~=3.6",
+    "requests_mock~=1.8"
 ]

 setup(
--- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py
+++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py
@@ -804,9 +804,9 @@ class SourceMixpanel(AbstractSource):
        :param logger:  logger object
        :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
        """
-        auth = TokenAuthenticatorBase64(token=config["api_secret"])
-        funnels = FunnelsList(authenticator=auth, **config)
        try:
+            auth = TokenAuthenticatorBase64(token=config["api_secret"])
+            funnels = FunnelsList(authenticator=auth, **config)
            response = requests.request(
                "GET",
                url=funnels.url_base + funnels.path(),
@@ -827,6 +827,7 @@ class SourceMixpanel(AbstractSource):

        return True, None

+
    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
        """
        :param config: A Mapping of the user input configuration as defined in the connector spec.
--- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/init.py
+++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/init.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
--- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py
+++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py
@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+import pendulum
+import pytest
+
+
+@pytest.fixture
+def config():
+    return {
+        "api_secret": "unexisting-secret",
+        "attribution_window": 5,
+        "project_timezone": "UTC",
+        "select_properties_by_default": True,
+        "start_date": pendulum.parse("2017-01-25T00:00:00Z").date(),
+        "end_date": pendulum.parse("2017-02-25T00:00:00Z").date(),
+        "region": "US",
+    }
--- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py
+++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+import pytest
+from airbyte_cdk import AirbyteLogger
+from source_mixpanel.source import FunnelsList, SourceMixpanel, TokenAuthenticatorBase64
+
+from .utils import get_url_to_mock, setup_response
+
+logger = AirbyteLogger()
+
+
+@pytest.fixture
+def check_connection_url(config):
+    auth = TokenAuthenticatorBase64(token=config["api_secret"])
+    funnel_list = FunnelsList(authenticator=auth, **config)
+    return get_url_to_mock(funnel_list)
+
+
+@pytest.mark.parametrize("response_code,expect_success", [(200, True), (400, False)])
+def test_check_connection(requests_mock, check_connection_url, config, response_code, expect_success):
+    requests_mock.register_uri("GET", check_connection_url, setup_response(response_code, {}))
+    ok, error = SourceMixpanel().check_connection(logger, config)
+    assert ok == expect_success and error != expect_success
+
+
+def test_check_connection_bad_config():
+    config = {}
+    ok, error = SourceMixpanel().check_connection(logger, config)
+    assert not ok and error
+
+
+def test_check_connection_incomplete(config):
+    config.pop("api_secret")
+    ok, error = SourceMixpanel().check_connection(logger, config)
+    assert not ok and error
+
+
+def test_streams(config):
+    streams = SourceMixpanel().streams(config)
+    assert len(streams) == 7
--- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py
+++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py
@@ -0,0 +1,280 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+from unittest.mock import MagicMock
+
+import pytest
+from airbyte_cdk import AirbyteLogger
+from airbyte_cdk.models import SyncMode
+from source_mixpanel.source import (
+    Annotations,
+    EngageSchema,
+    Export,
+    ExportSchema,
+    Funnels,
+    FunnelsList,
+    IncrementalMixpanelStream,
+    MixpanelStream,
+    Revenue,
+)
+
+from .utils import get_url_to_mock, setup_response
+
+logger = AirbyteLogger()
+
+
+@pytest.fixture
+def patch_base_class(mocker):
+    # Mock abstract methods to enable instantiating abstract class
+    mocker.patch.object(MixpanelStream, "path", "v0/example_endpoint")
+    mocker.patch.object(MixpanelStream, "primary_key", "test_primary_key")
+    mocker.patch.object(MixpanelStream, "__abstractmethods__", set())
+
+
+@pytest.fixture
+def patch_incremental_base_class(mocker):
+    # Mock abstract methods to enable instantiating abstract class
+    mocker.patch.object(IncrementalMixpanelStream, "path", "v0/example_endpoint")
+    mocker.patch.object(IncrementalMixpanelStream, "primary_key", "test_primary_key")
+    mocker.patch.object(IncrementalMixpanelStream, "cursor_field", "date")
+    mocker.patch.object(IncrementalMixpanelStream, "__abstractmethods__", set())
+
+
+def test_url_base(patch_base_class):
+    stream = MixpanelStream(authenticator=MagicMock())
+
+    assert stream.url_base == "https://mixpanel.com/api/2.0/"
+
+
+def test_request_headers(patch_base_class):
+    stream = MixpanelStream(authenticator=MagicMock())
+
+    assert stream.request_headers(stream_state={}) == {"Accept": "application/json"}
+
+
+def test_updated_state(patch_incremental_base_class):
+    stream = IncrementalMixpanelStream(authenticator=MagicMock())
+
+    updated_state = stream.get_updated_state(
+        current_stream_state={"date": "2021-01-25T00:00:00Z"}, latest_record={"date": "2021-02-25T00:00:00Z"}
+    )
+
+    assert updated_state == {"date": "2021-02-25T00:00:00Z"}
+
+
+def test_cohorts_stream():
+    # tested in itaseskii:mixpanel-incremental-syncs
+    return None
+
+
+@pytest.fixture
+def funnels_list_response():
+    return setup_response(200, [{"funnel_id": 1, "name": "Signup funnel"}])
+
+
+def test_funnels_list_stream(requests_mock, config, funnels_list_response):
+    stream = FunnelsList(authenticator=MagicMock(), **config)
+    requests_mock.register_uri("GET", get_url_to_mock(stream), funnels_list_response)
+
+    records = stream.read_records(sync_mode=SyncMode.full_refresh)
+
+    records_length = sum(1 for _ in records)
+    assert records_length == 1
+
+
+@pytest.fixture
+def funnels_list_url(config):
+    funnel_list = FunnelsList(authenticator=MagicMock(), **config)
+    return get_url_to_mock(funnel_list)
+
+
+@pytest.fixture
+def funnels_response():
+    return setup_response(
+        200,
+        {
+            "meta": {"dates": ["2016-09-12" "2016-09-19" "2016-09-26"]},
+            "data": {
+                "2016-09-12": {
+                    "steps": [],
+                    "analysis": {
+                        "completion": 20524,
+                        "starting_amount": 32688,
+                        "steps": 2,
+                        "worst": 1,
+                    },
+                },
+                "2016-09-19": {
+                    "steps": [],
+                    "analysis": {
+                        "completion": 20500,
+                        "starting_amount": 34750,
+                        "steps": 2,
+                        "worst": 1,
+                    },
+                },
+            },
+        },
+    )
+
+
+def test_funnels_stream(requests_mock, config, funnels_response, funnels_list_response, funnels_list_url):
+    stream = Funnels(authenticator=MagicMock(), **config)
+    requests_mock.register_uri("GET", funnels_list_url, funnels_list_response)
+    requests_mock.register_uri("GET", get_url_to_mock(stream), funnels_response)
+
+    stream_slices = stream.stream_slices(sync_mode=SyncMode.incremental)
+
+    records_arr = []
+    for stream_slice in stream_slices:
+        records = stream.read_records(sync_mode=SyncMode.incremental, stream_slice=stream_slice)
+        for record in records:
+            records_arr.append(record)
+
+    assert len(records_arr) == 4
+
+
+@pytest.fixture
+def engage_schema_response():
+    return setup_response(
+        200,
+        {
+            "results": {
+                "$browser": {"count": 124, "type": "string"},
+                "$browser_version": {"count": 124, "type": "string"},
+                "$created": {"count": 124, "type": "string"},
+            }
+        },
+    )
+
+
+def test_engage_schema(requests_mock, engage_schema_response):
+
+    stream = EngageSchema(authenticator=MagicMock())
+    requests_mock.register_uri("GET", get_url_to_mock(stream), engage_schema_response)
+
+    records = stream.read_records(sync_mode=SyncMode.full_refresh)
+
+    records_length = sum(1 for _ in records)
+    assert records_length == 3
+
+
+@pytest.fixture
+def annotations_response():
+    return setup_response(
+        200,
+        {
+            "annotations": [
+                {"id": 640999, "project_id": 2117889, "date": "2021-06-16 00:00:00", "description": "Looks good"},
+                {"id": 640000, "project_id": 2117889, "date": "2021-06-16 00:00:00", "description": "Looks bad"},
+            ]
+        },
+    )
+
+
+def test_annotations_stream(requests_mock, annotations_response):
+
+    stream = Annotations(authenticator=MagicMock())
+    requests_mock.register_uri("GET", get_url_to_mock(stream), annotations_response)
+
+    stream_slice = {"start_date": "2017-01-25T00:00:00Z", "end_date": "2017-02-25T00:00:00Z"}
+    # read records for single slice
+    records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)
+
+    records_length = sum(1 for _ in records)
+    assert records_length == 2
+
+
+@pytest.fixture
+def revenue_response():
+    return setup_response(
+        200,
+        {
+            "computed_at": "2021-07-03T12:43:48.889421+00:00",
+            "results": {
+                "$overall": {"amount": 0.0, "count": 124, "paid_count": 0},
+                "2021-06-01": {"amount": 0.0, "count": 124, "paid_count": 0},
+                "2021-06-02": {"amount": 0.0, "count": 124, "paid_count": 0},
+            },
+            "session_id": "162...",
+            "status": "ok",
+        },
+    )
+
+
+def test_revenue_stream(requests_mock, revenue_response):
+
+    stream = Revenue(authenticator=MagicMock())
+    requests_mock.register_uri("GET", get_url_to_mock(stream), revenue_response)
+
+    stream_slice = {"start_date": "2017-01-25T00:00:00Z", "end_date": "2017-02-25T00:00:00Z"}
+    # read records for single slice
+    records = stream.read_records(sync_mode=SyncMode.incremental, stream_slice=stream_slice)
+
+    records_length = sum(1 for _ in records)
+    assert records_length == 2
+
+
+@pytest.fixture
+def export_schema_response():
+    return setup_response(
+        200,
+        {
+            "$browser": {"count": 6},
+            "$browser_version": {"count": 6},
+            "$current_url": {"count": 6},
+            "mp_lib": {"count": 6},
+            "noninteraction": {"count": 6},
+            "$event_name": {"count": 6},
+            "$duration_s": {},
+            "$event_count": {},
+            "$origin_end": {},
+            "$origin_start": {},
+        },
+    )
+
+
+def test_export_schema(requests_mock, export_schema_response):
+
+    stream = ExportSchema(authenticator=MagicMock())
+    requests_mock.register_uri("GET", get_url_to_mock(stream), export_schema_response)
+
+    records = stream.read_records(sync_mode=SyncMode.full_refresh)
+
+    records_length = sum(1 for _ in records)
+    assert records_length == 10
+
+
+@pytest.fixture
+def export_response():
+    return setup_response(
+        200,
+        {
+            "event": "Viewed E-commerce Page",
+            "properties": {
+                "time": 1623860880,
+                "distinct_id": "1d694fd9-31a5-4b99-9eef-ae63112063ed",
+                "$browser": "Chrome",
+                "$browser_version": "91.0.4472.101",
+                "$current_url": "https://unblockdata.com/solutions/e-commerce/",
+                "$insert_id": "c5eed127-c747-59c8-a5ed-d766f48e39a4",
+                "$mp_api_endpoint": "api.mixpanel.com",
+                "mp_lib": "Segment: analytics-wordpress",
+                "mp_processing_time_ms": 1623886083321,
+                "noninteraction": True,
+            },
+        },
+    )
+
+
+def test_export_stream(requests_mock, export_response):
+
+    stream = Export(authenticator=MagicMock())
+    requests_mock.register_uri("GET", get_url_to_mock(stream), export_response)
+
+    stream_slice = {"start_date": "2017-01-25T00:00:00Z", "end_date": "2017-02-25T00:00:00Z"}
+    # read records for single slice
+    records = stream.read_records(sync_mode=SyncMode.incremental, stream_slice=stream_slice)
+
+    records_length = sum(1 for _ in records)
+    assert records_length == 1
--- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py
+++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py
@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+import urllib.parse
+
+
+def setup_response(status, body):
+    return [{"json": body, "status_code": status}]
+
+
+def get_url_to_mock(stream):
+    return urllib.parse.urljoin(stream.url_base, stream.path())
--- a/docs/integrations/sources/mixpanel.md
+++ b/docs/integrations/sources/mixpanel.md
@@ -57,17 +57,19 @@ Select the correct region \(EU or US\) for your Mixpanel project. See detail [he

 ## CHANGELOG

-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| `0.1.11` | 2022-04-04 | [11318](https://github.com/airbytehq/airbyte/pull/11318) | Change Response Reading |
-| `0.1.10` | 2022-03-31 | [11227](https://github.com/airbytehq/airbyte/pull/11227) | Fix cohort id always null in the cohort_members stream |
-| `0.1.9` | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions |
-| `0.1.7` | 2021-12-01 | [8381](https://github.com/airbytehq/airbyte/pull/8381) | Increased performance for `discovery` stage during connector setup |
-| `0.1.6` | 2021-11-25 | [8256](https://github.com/airbytehq/airbyte/issues/8256) | Deleted `date_window_size` and fix schemas date type issue |
-| `0.1.5` | 2021-11-10 | [7451](https://github.com/airbytehq/airbyte/issues/7451) | Support `start_date` older than 1 year |
-| `0.1.4` | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies |
-| `0.1.3` | 2021-10-30 | [7505](https://github.com/airbytehq/airbyte/issues/7505) | Guarantee that standard and custom mixpanel properties in the `Engage` stream are written as strings |
-| `0.1.2` | 2021-11-02 | [7439](https://github.com/airbytehq/airbyte/issues/7439) | Added delay for all streams to match API limitation of requests rate |
-| `0.1.1` | 2021-09-16 | [6075](https://github.com/airbytehq/airbyte/issues/6075) | Added option to select project region |
-| `0.1.0` | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector |
+| Version  | Date       | Pull Request                                             | Subject                                                                                              |
+|:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------|
+| `0.1.12` | 2022-03-31 | [11633](https://github.com/airbytehq/airbyte/pull/11633) | Increase unit test coverage                                                                          |  
+| `0.1.11` | 2022-04-04 | [11318](https://github.com/airbytehq/airbyte/pull/11318) | Change Response Reading                                                                              |
+| `0.1.10` | 2022-03-31 | [11227](https://github.com/airbytehq/airbyte/pull/11227) | Fix cohort id always null in the cohort_members stream                                               |
+| `0.1.9`  | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8578)   | Updated titles and descriptions                                                                      |
+| `0.1.7`  | 2021-12-01 | [8381](https://github.com/airbytehq/airbyte/pull/8381)   | Increased performance for `discovery` stage during connector setup                                   |
+| `0.1.6`  | 2021-11-25 | [8256](https://github.com/airbytehq/airbyte/issues/8256) | Deleted `date_window_size` and fix schemas date type issue                                           |
+| `0.1.5`  | 2021-11-10 | [7451](https://github.com/airbytehq/airbyte/issues/7451) | Support `start_date` older than 1 year                                                               |
+| `0.1.4`  | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                                                      |
+| `0.1.3`  | 2021-10-30 | [7505](https://github.com/airbytehq/airbyte/issues/7505) | Guarantee that standard and custom mixpanel properties in the `Engage` stream are written as strings |
+| `0.1.2`  | 2021-11-02 | [7439](https://github.com/airbytehq/airbyte/issues/7439) | Added delay for all streams to match API limitation of requests rate                                 |
+| `0.1.1`  | 2021-09-16 | [6075](https://github.com/airbytehq/airbyte/issues/6075) | Added option to select project region                                                                |
+| `0.1.0`  | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector                                                                |
+