jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

🚨🚨 Source Hubspot: Update Deals Property History and Companies Property History to API v3 (#35662)

Browse Source 
Signed-off-by: Artem Inzhyyants <artem.inzhyyants@gmail.com>
This commit is contained in:
Artem Inzhyyants
2024-03-08 12:24:42 +01:00
committed by GitHub
parent f09c97cae3
commit 63091e5024
11 changed files with 147 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl
View File

@@ -51,10 +51,10 @@
{"stream": "products", "data": {"id": "646176423", "properties": {"amount": null, "createdate": "2021-02-23T20:03:48.577000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": 2430008, "hs_folder_name": "test folder", "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176423, "hs_object_source": "CRM_UI", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product 1", "price": 123, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:48.577Z", "updatedAt": "2021-02-23T20:03:48.577Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:48.577000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": 2430008, "properties_hs_folder_name": "test folder", "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176423, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product 1", "properties_price": 123, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1708014628643}
{"stream": "products", "data": {"id": "646316535", "properties": {"amount": null, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "baseball hat, large", "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": null, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646316535, "hs_object_source": "IMPORT", "hs_object_source_detail_1": null, "hs_object_source_detail_2": null, "hs_object_source_detail_3": null, "hs_object_source_id": null, "hs_object_source_label": "IMPORT", "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-02-23T20:11:54.030Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646316535, "properties_hs_object_source": "IMPORT", "properties_hs_object_source_detail_1": null, "properties_hs_object_source_detail_2": null, "properties_hs_object_source_detail_3": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": "IMPORT", "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1708014628645}
{"stream": "contacts_property_history", "data": {"value": "testo", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1700681340515, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064}
{"stream": "contacts_property_history", "data": {"value": "test", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1675120629904, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064}
{"stream": "companies_property_history", "data": {"name": "hs_analytics_latest_source_data_2", "value": "CRM_UI", "timestamp": 1657222285656, "sourceId": "RollupProperties", "source": "MIGRATION", "sourceVid": [], "property": "hs_analytics_latest_source_data_2", "companyId": 5000526215, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905731242}
{"stream": "companies_property_history", "data": {"name": "hs_analytics_latest_source_data_1", "value": "CONTACTS", "timestamp": 1657222285656, "sourceId": "RollupProperties", "source": "MIGRATION", "sourceVid": [], "property": "hs_analytics_latest_source_data_1", "companyId": 5000526215, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905731242}
{"stream": "deals_property_history", "data": {"name": "dealname", "value": "Test Deal 2", "timestamp": 1610635080797, "source": "API", "sourceVid": [], "requestId": "cdc0501c-7d08-40e4-a937-953492b1a6c2", "property": "dealname", "dealId": 3986867076, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1707258294359}
{"stream":"companies_property_history","data":{"value":"Test","timestamp":"2023-01-30T23:22:56.969Z","sourceType":"CRM_UI","sourceId":"userId:12282590","updatedByUserId":12282590,"property":"name","companyId":"11481383026","archived":false},"emitted_at":1709764013901}
{"stream":"companies_property_history","data":{"value":"CONTACTS","timestamp":"2022-07-07T19:31:25.656Z","sourceType":"MIGRATION","sourceId":"RollupProperties","property":"hs_analytics_latest_source_data_1","companyId":"5000526215","archived":false},"emitted_at":1709764012837}
{"stream":"companies_property_history","data":{"value":"CRM_UI","timestamp":"2022-07-07T19:31:25.656Z","sourceType":"MIGRATION","sourceId":"RollupProperties","property":"hs_analytics_latest_source_data_2","companyId":"5000526215","archived":false},"emitted_at":1709764012837}
{"stream":"deals_property_history","data":{"value":"Test Deal 2","timestamp":"2021-01-14T14:38:00.797Z","sourceType":"API","property":"dealname","dealId":"3986867076","archived":false},"emitted_at":1709751896693}
{"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "portalId": 8727216, "causedByEvent": {"id": "d70b78b9-a411-4d3e-808b-fe931be35b43", "created": 1616173134301}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255435}
{"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "subscriptionId": 10798197, "portalId": 8727216, "causedByEvent": {"id": "ff118718-786d-4a35-94f9-6bbd413654de", "created": 1616173134301}, "changeType": "SUBSCRIPTION_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255436}
{"stream": "subscription_changes", "data": {"timestamp": 1616173106737, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173106737, "portalId": 8727216, "causedByEvent": {"id": "24539f1f-0b20-4296-a5bf-6ba3bb9dc1b8", "created": 1616173106737}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255437}

18
airbyte-integrations/connectors/source-hubspot/metadata.yaml
View File

@@ -10,7 +10,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
dockerImageTag: 3.3.0
dockerImageTag: 4.0.0
dockerRepository: airbyte/source-hubspot
documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot
githubIssueLabel: source-hubspot
@@ -29,11 +29,14 @@ data:
releaseStage: generally_available
releases:
breakingChanges:
2.0.0:
4.0.0:
message: >-
This version eliminates the Property History stream in favor of creating 3 different streams, Contacts, Companies, and Deals, which can now all fetch their property history.
It will affect only users who use Property History stream, who will need to fix schema conflicts and sync Contacts Property History stream instead of Property History.
upgradeDeadline: 2024-01-15
This update brings extended schema with data type changes for the streams `Deals Property History` and `Companies Property History`. Users will need to refresh their schema and reset their streams after upgrading.
upgradeDeadline: 2024-03-10
scopedImpact:
- scopeType: stream
impactedScopes:
["deals_property_history", "companies_property_history"]
3.0.0:
message: >-
This update brings extended schema with data type changes for the Marketing Emails stream.
@@ -42,6 +45,11 @@ data:
scopedImpact:
- scopeType: stream
impactedScopes: ["marketing_emails"]
2.0.0:
message: >-
This version replaces the `Property History` stream in favor of creating 3 different streams: `Contacts`, `Companies`, and `Deals`, which can now all fetch their property history.
It will affect only users who use `Property History` stream, who will need to fix schema conflicts and sync `Contacts Property History` stream instead of `Property History`.
upgradeDeadline: 2024-01-15
suggestedStreams:
streams:
- contacts

2
airbyte-integrations/connectors/source-hubspot/pyproject.toml
View File

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
version = "3.3.0"
version = "4.0.0"
name = "source-hubspot"
description = "Source implementation for HubSpot."
authors = [ "Airbyte <contact@airbyte.io>",]

42
airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json
View File

@@ -6,50 +6,28 @@
"updatedByUserId": {
"type": ["null", "number"]
},
"requestId": {
"type": ["null", "string"]
},
"source": {
"type": ["null", "string"]
},
"portalId": {
"type": ["null", "number"]
},
"isDeleted": {
"type": ["null", "boolean"]
},
"timestamp": {
"type": ["null", "number"]
"type": ["null", "string"],
"format": "date-time",
"airbyte_type": "timestamp_with_timezone"
},
"property": {
"type": ["null", "string"]
},
"persistenceTimestamp": {
"type": ["null", "number"]
},
"name": {
"type": ["null", "string"]
},
"sourceVid": {
"type": ["null", "array"]
},
"useTimestampAsPersistenceTimestamp": {
"type": ["null", "boolean"]
},
"sourceMetadata": {
"type": ["null", "string"]
},
"dataSensitivity": {
"type": ["null", "string"]
},
"companyId": {
"type": ["null", "number"]
"type": ["null", "string"]
},
"sourceType": {
"type": ["null", "string"]
},
"sourceId": {
"type": ["null", "string"]
},
"value": {
"type": ["null", "string"]
},
"archived": {
"type": ["null", "boolean"]
}
}
}

42
airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json
View File

@@ -3,53 +3,31 @@
"type": ["null", "object"],
"additionalProperties": true,
"properties": {
"dataSensitivity": {
"type": ["null", "string"]
},
"updatedByUserId": {
"type": ["null", "number"]
},
"requestId": {
"type": ["null", "string"]
},
"source": {
"type": ["null", "string"]
},
"portalId": {
"type": ["null", "number"]
},
"isDeleted": {
"type": ["null", "boolean"]
},
"timestamp": {
"type": ["null", "number"]
"type": ["null", "string"],
"format": "date-time",
"airbyte_type": "timestamp_with_timezone"
},
"property": {
"type": ["null", "string"]
},
"persistenceTimestamp": {
"type": ["null", "number"]
},
"name": {
"type": ["null", "string"]
},
"sourceVid": {
"type": ["null", "array"]
},
"useTimestampAsPersistenceTimestamp": {
"type": ["null", "boolean"]
},
"sourceMetadata": {
"type": ["null", "string"]
},
"dealId": {
"type": ["null", "number"]
"type": ["null", "string"]
},
"sourceType": {
"type": ["null", "string"]
},
"sourceId": {
"type": ["null", "string"]
},
"value": {
"type": ["null", "string"]
},
"archived": {
"type": ["null", "boolean"]
}
}
}

158
airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
View File

@@ -8,7 +8,7 @@ import sys
import time
from abc import ABC, abstractmethod
from datetime import timedelta
from functools import cached_property, lru_cache, reduce
from functools import cached_property, lru_cache
from http import HTTPStatus
from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
@@ -17,7 +17,6 @@ import pendulum as pendulum
import requests
from airbyte_cdk.entrypoint import logger
from airbyte_cdk.models import FailureType, SyncMode
from airbyte_cdk.models.airbyte_protocol import SyncMode
from airbyte_cdk.sources import Source
from airbyte_cdk.sources.streams import IncrementalMixin, Stream
from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
@@ -1963,67 +1962,54 @@ class ContactsPropertyHistory(PropertyHistory):
return "/contacts/v1/lists/all/contacts/all"
class CompaniesPropertyHistory(PropertyHistory):
class PropertyHistoryV3(PropertyHistory):
@cached_property
def _property_wrapper(self) -> IURLPropertyRepresentation:
properties = list(self.properties.keys())
return APIPropertiesWithHistory(properties=properties)
@property
def scopes(self) -> set:
return {"crm.objects.companies.read"}
@property
def properties_scopes(self) -> set:
return {"crm.schemas.companies.read"}
@property
def page_field(self) -> str:
return "offset"
@property
def limit_field(self) -> str:
return "limit"
@property
def page_filter(self) -> str:
return "offset"
@property
def more_key(self) -> str:
return "has-more"
@property
def entity(self) -> str:
return "companies"
@property
def entity_primary_key(self) -> list:
return "companyId"
@property
def primary_key(self) -> list:
return ["companyId", "property", "timestamp"]
@property
def additional_keys(self) -> list:
return ["portalId", "isDeleted"]
@property
def last_modified_date_field_name(self) -> str:
return "hs_lastmodifieddate"
@property
def data_field(self) -> str:
return "companies"
@property
def url(self) -> str:
return "/companies/v2/companies/paged"
limit = 50
more_key = page_filter = page_field = None
limit_field = "limit"
data_field = "results"
additional_keys = ["archived"]
last_modified_date_field_name = "hs_lastmodifieddate"
def update_request_properties(self, params: Mapping[str, Any], properties: IURLPropertyRepresentation) -> None:
pass
def _transform(self, records: Iterable) -> Iterable:
for record in records:
properties_with_history = record.get("propertiesWithHistory")
primary_key = record.get("id")
additional_keys = {additional_key: record.get(additional_key) for additional_key in self.additional_keys}
for property_name, value_dict in properties_with_history.items():
if property_name == self.last_modified_date_field_name:
# Skipping the lastmodifieddate since it only returns the value
# when one field of a record was changed no matter which
# field was changed. It therefore creates overhead, since for
# every changed property there will be the date it was changed in itself
# and a change in the lastmodifieddate field.
continue
for version in value_dict:
version["property"] = property_name
version[self.entity_primary_key] = primary_key
yield version | additional_keys
class CompaniesPropertyHistory(PropertyHistoryV3):
scopes = {"crm.objects.companies.read"}
properties_scopes = {"crm.schemas.companies.read"}
entity = "companies"
entity_primary_key = "companyId"
primary_key = ["companyId", "property", "timestamp"]
@property
def url(self) -> str:
return "/crm/v3/objects/companies"
def path(
self,
*,
@@ -2035,66 +2021,16 @@ class CompaniesPropertyHistory(PropertyHistory):
return f"{self.url}?{properties.as_url_param()}"
class DealsPropertyHistory(PropertyHistory):
@cached_property
def _property_wrapper(self) -> IURLPropertyRepresentation:
properties = list(self.properties.keys())
return APIPropertiesWithHistory(properties=properties)
@property
def scopes(self) -> set:
return {"crm.objects.deals.read"}
@property
def properties_scopes(self):
return {"crm.schemas.deals.read"}
@property
def page_field(self) -> str:
return "offset"
@property
def limit_field(self) -> str:
return "limit"
@property
def page_filter(self) -> str:
return "offset"
@property
def more_key(self) -> str:
return "hasMore"
@property
def entity(self) -> set:
return "deals"
@property
def entity_primary_key(self) -> list:
return "dealId"
@property
def primary_key(self) -> list:
return ["dealId", "property", "timestamp"]
@property
def additional_keys(self) -> list:
return ["portalId", "isDeleted"]
@property
def last_modified_date_field_name(self) -> str:
return "hs_lastmodifieddate"
@property
def data_field(self) -> str:
return "deals"
class DealsPropertyHistory(PropertyHistoryV3):
scopes = {"crm.objects.deals.read"}
properties_scopes = {"crm.schemas.deals.read"}
entity = "deals"
entity_primary_key = "dealId"
primary_key = ["dealId", "property", "timestamp"]
@property
def url(self) -> str:
return "/deals/v1/deal/paged"
def update_request_properties(self, params: Mapping[str, Any], properties: IURLPropertyRepresentation) -> None:
pass
return "/crm/v3/objects/deals"
def path(
self,

4
airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py
View File

@@ -51,7 +51,7 @@ def config_fixture():
return {
"start_date": "2021-01-10T00:00:00Z",
"credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"},
"enable_experimental_streams": False
"enable_experimental_streams": False,
}
@@ -60,7 +60,7 @@ def config_eperimantal_fixture():
return {
"start_date": "2021-01-10T00:00:00Z",
"credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"},
"enable_experimental_streams": True
"enable_experimental_streams": True,
}

20
airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py
View File

@@ -85,11 +85,11 @@ def test_streams(requests_mock, config):
streams = SourceHubspot().streams(config)
assert len(streams) == 32
assert len(streams) == 33
@mock.patch("source_hubspot.source.SourceHubspot.get_custom_object_streams")
def test_streams(requests_mock, config_experimental):
def test_streams_incremental(requests_mock, config_experimental):
streams = SourceHubspot().streams(config_experimental)
@@ -97,16 +97,10 @@ def test_streams(requests_mock, config_experimental):
def test_custom_streams(config_experimental):
custom_object_stream_instances = [
MagicMock()
]
custom_object_stream_instances = [MagicMock()]
streams = SourceHubspot().get_web_analytics_custom_objects_stream(
custom_object_stream_instances=custom_object_stream_instances,
common_params={
"api": MagicMock(),
"start_date": "2021-01-01T00:00:00Z",
"credentials": config_experimental["credentials"]
}
common_params={"api": MagicMock(), "start_date": "2021-01-01T00:00:00Z", "credentials": config_experimental["credentials"]},
)
assert len(list(streams)) == 1
@@ -481,7 +475,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req
requests_mock.register_uri(
"POST",
"/crm/v4/associations/company/contacts/batch/read",
[{"status_code": 200, "json": {"results": [{"from": {"id": "1"}, "to": [{"toObjectId": "2"}]}]}}]
[{"status_code": 200, "json": {"results": [{"from": {"id": "1"}, "to": [{"toObjectId": "2"}]}]}}],
)
records, _ = read_incremental(test_stream, {})
@@ -709,13 +703,13 @@ def test_pagination_marketing_emails_stream(requests_mock, common_params):
def test_get_granted_scopes(requests_mock, mocker):
authenticator = mocker.Mock()
authenticator.get_access_token.return_value = "the-token"
expected_scopes = ["a", "b", "c"]
response = [
{"json": {"scopes": expected_scopes}, "status_code": 200},
]
requests_mock.register_uri("GET", "https://api.hubapi.com/oauth/v1/access-tokens/the-token", response)
actual_scopes = SourceHubspot().get_granted_scopes(authenticator)
assert expected_scopes == actual_scopes

55
airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py
View File

@@ -521,10 +521,7 @@ def test_web_analytics_stream_slices(common_params, mocker):
assert len(slices) == 2
assert all(map(lambda slice: slice["objectId"] == 1, slices))
assert [
("2021-01-10T00:00:00Z", "2021-02-09T00:00:00Z"),
("2021-02-09T00:00:00Z", "2021-03-01T00:00:00Z")
] == [
assert [("2021-01-10T00:00:00Z", "2021-02-09T00:00:00Z"), ("2021-02-09T00:00:00Z", "2021-03-01T00:00:00Z")] == [
(s["occurredAfter"], s["occurredBefore"]) for s in slices
]
@@ -542,7 +539,9 @@ def test_web_analytics_latest_state(common_params, mocker):
stream = ContactsWebAnalytics(**common_params)
stream.state = {"1": {"occurredAt": "2021-01-01T00:00:00Z"}}
slices = list(stream.stream_slices(SyncMode.incremental, cursor_field="occurredAt"))
records = [list(stream.read_records(SyncMode.incremental, cursor_field="occurredAt", stream_slice=stream_slice)) for stream_slice in slices]
records = [
list(stream.read_records(SyncMode.incremental, cursor_field="occurredAt", stream_slice=stream_slice)) for stream_slice in slices
]
assert len(slices) == 1
assert len(records) == 1
@@ -553,63 +552,33 @@ def test_web_analytics_latest_state(common_params, mocker):
def test_property_history_transform(common_params):
stream = ContactsPropertyHistory(**common_params)
versions = [
{
"value": "Georgia",
"timestamp": 1645135236625
}
]
versions = [{"value": "Georgia", "timestamp": 1645135236625}]
records = [
{
"vid": 1,
"canonical-vid": 1,
"portal-id": 1,
"is-contact": True,
"properties": {
"hs_country": {"versions": versions},
"lastmodifieddate": {"value": 1645135236625}
}
"properties": {"hs_country": {"versions": versions}, "lastmodifieddate": {"value": 1645135236625}},
}
]
assert [
{
"vid": 1,
"canonical-vid": 1,
"portal-id": 1,
"is-contact": True,
"property": "hs_country",
**version
} for version in versions
{"vid": 1, "canonical-vid": 1, "portal-id": 1, "is-contact": True, "property": "hs_country", **version} for version in versions
] == list(stream._transform(records=records))
def test_contacts_membership_transform(common_params):
stream = ContactsListMemberships(**common_params)
versions = [
{
"value": "Georgia",
"timestamp": 1645135236625
}
]
memberships = [
{"membership": 1}
]
versions = [{"value": "Georgia", "timestamp": 1645135236625}]
memberships = [{"membership": 1}]
records = [
{
"vid": 1,
"canonical-vid": 1,
"portal-id": 1,
"is-contact": True,
"properties": {
"hs_country": {"versions": versions},
"lastmodifieddate": {"value": 1645135236625}
},
"list-memberships": memberships
"properties": {"hs_country": {"versions": versions}, "lastmodifieddate": {"value": 1645135236625}},
"list-memberships": memberships,
}
]
assert [
{
"membership": 1,
"canonical-vid": 1
} for _ in versions
] == list(stream._transform(records=records))
assert [{"membership": 1, "canonical-vid": 1} for _ in versions] == list(stream._transform(records=records))

40
docs/integrations/sources/hubspot-migrations.md
View File

@@ -1,5 +1,45 @@
# HubSpot Migration Guide
## Upgrading to 4.0.0
:::note
This change is only breaking if you are syncing streams `Deals Property History` or `Companies Peoperty History`.
:::
This update brings extended schema with data type changes for the Marketing Emails stream.
Users should:
- Refresh the source schema for the Marketing Emails stream.
- Reset the stream after upgrading to ensure uninterrupted syncs.
### Refresh affected schemas and reset data
1. Select **Connections** in the main nav bar.
1. Select the connection affected by the update.
2. Select the **Replication** tab.
1. Select **Refresh source schema**.
2. Select **OK**.
:::note
Any detected schema changes will be listed for your review.
:::
3. Select **Save changes** at the bottom of the page.
1. Ensure the **Reset affected streams** option is checked.
:::note
Depending on destination type you may not be prompted to reset your data.
:::
4. Select **Save connection**.
:::note
This will reset the data in your destination and initiate a fresh sync.
:::
For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset)
## Upgrading to 3.0.0
:::note

1
docs/integrations/sources/hubspot.md
View File

@@ -322,6 +322,7 @@ The connector is restricted by normal HubSpot [rate limitations](https://legacyd
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 4.0.0 | 2024-03-10 | [35662](https://github.com/airbytehq/airbyte/pull/35662) | Update `Deals Property History` and `Companies Property History` schemas |
| 3.3.0 | 2024-02-16 | [34597](https://github.com/airbytehq/airbyte/pull/34597) | Make start date not required, sync all data from default value if it's not provided |
| 3.2.0 | 2024-02-15 | [35328](https://github.com/airbytehq/airbyte/pull/35328) | Add mailingIlsListsIncluded and mailingIlsListsExcluded fields to Marketing emails stream schema |
| 3.1.1 | 2024-02-12 | [35165](https://github.com/airbytehq/airbyte/pull/35165) | Manage dependencies with Poetry. |