airbyte/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py

#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#

from abc import ABC
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union

import pendulum
import requests
from airbyte_cdk.sources.streams.http import HttpStream
from airbyte_cdk.sources.streams.http.auth import NoAuth, Oauth2Authenticator

PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/"


class PipedriveStream(HttpStream, ABC):
    url_base = PIPEDRIVE_URL_BASE
    primary_key = "id"
    data_field = "data"
    page_size = 50

    def __init__(self, authenticator, replication_start_date=None, **kwargs):
        if isinstance(authenticator, Oauth2Authenticator):
            super().__init__(authenticator=authenticator, **kwargs)
        else:
            super().__init__(**kwargs)
            self._api_token = authenticator["api_token"]

        self._replication_start_date = replication_start_date

    @property
    def cursor_field(self) -> Union[str, List[str]]:
        if self._replication_start_date:
            return "update_time"
        return []

    def path(self, **kwargs) -> str:
        if self._replication_start_date:
            return "recents"

        class_name = self.__class__.__name__
        return f"{class_name[0].lower()}{class_name[1:]}"

    @property
    def path_param(self):
        return self.name[:-1]

    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
        """
        :param response: the most recent response from the API
        :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query
                the next page in the response.
                If there are no more pages in the result, return None.
        """
        pagination_data = response.json().get("additional_data", {}).get("pagination", {})
        if pagination_data.get("more_items_in_collection") and pagination_data.get("start") is not None:
            start = pagination_data.get("start") + self.page_size
            return {"start": start}

    def request_params(
        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None
    ) -> MutableMapping[str, Any]:
        next_page_token = next_page_token or {}
        params = {"limit": self.page_size, **next_page_token}

        if isinstance(self.authenticator, NoAuth):
            params["api_token"] = self._api_token

        replication_start_date = self._replication_start_date
        if replication_start_date:
            if stream_state.get(self.cursor_field):
                replication_start_date = max(pendulum.parse(stream_state[self.cursor_field]), replication_start_date)

            params.update(
                {
                    "items": self.path_param,
                    "since_timestamp": replication_start_date.strftime("%Y-%m-%d %H:%M:%S"),
                }
            )

        return params

    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
        """
        :return an iterable containing each record in the response
        """
        records = response.json().get(self.data_field) or []
        for record in records:
            record = record.get(self.data_field) or record
            if self.primary_key in record and record[self.primary_key] is None:
                # Convert "id: null" fields to "id: 0" since id is primary key and SAT checks if it is not null.
                record[self.primary_key] = 0
            yield record

    def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
        """
        Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object
        and returning an updated state object.
        """
        latest_benchmark = latest_record[self.cursor_field]
        if current_stream_state.get(self.cursor_field):
            return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])}
        return {self.cursor_field: latest_benchmark}


class Deals(PipedriveStream):
    """
    API docs: https://developers.pipedrive.com/docs/api/v1/Deals#getDeals,
    retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
    """


class DealFields(PipedriveStream):
    """https://developers.pipedrive.com/docs/api/v1/DealFields#getDealFields"""


class Leads(PipedriveStream):
    """https://developers.pipedrive.com/docs/api/v1/Leads#getLeads"""


class Activities(PipedriveStream):
    """
    API docs: https://developers.pipedrive.com/docs/api/v1/Activities#getActivities,
    retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
    """

    path_param = "activity"


class ActivityFields(PipedriveStream):
    """https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields"""


class Organizations(PipedriveStream):
    """
    API docs: https://developers.pipedrive.com/docs/api/v1/Organizations#getOrganizations,
    retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
    """


class OrganizationFields(PipedriveStream):
    """https://developers.pipedrive.com/docs/api/v1/OrganizationFields#getOrganizationFields"""


class Persons(PipedriveStream):
    """
    API docs: https://developers.pipedrive.com/docs/api/v1/Persons#getPersons,
    retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
    """


class PersonFields(PipedriveStream):
    """https://developers.pipedrive.com/docs/api/v1/PersonFields#getPersonFields"""


class Pipelines(PipedriveStream):
    """
    API docs: https://developers.pipedrive.com/docs/api/v1/Pipelines#getPipelines,
    retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
    """


class Stages(PipedriveStream):
    """
    API docs: https://developers.pipedrive.com/docs/api/v1/Stages#getStages,
    retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
    """


class Users(PipedriveStream):
    """
    API docs: https://developers.pipedrive.com/docs/api/v1/Users#getUsers,
    retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
    """

    cursor_field = "modified"

    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
        record_gen = super().parse_response(response=response, **kwargs)
        for records in record_gen:
            yield from records