feat(source-pipedrive): update cdk to enable concurrency (#48615)
This commit is contained in:
Daryna Ishchenko
@@ -17,6 +17,9 @@ acceptance_tests:
|
||||
basic_read:
|
||||
tests:
|
||||
- config_path: "secrets/config.json"
|
||||
expect_records:
|
||||
path: "integration_tests/expected_records.jsonl"
|
||||
exact_order: no
|
||||
configured_catalog_path: "integration_tests/configured_catalog.json"
|
||||
fail_on_extra_columns: false
|
||||
empty_streams:
|
||||
@@ -30,13 +33,12 @@ acceptance_tests:
|
||||
- name: stages
|
||||
- name: deal_products
|
||||
- name: mail
|
||||
- name: deals
|
||||
- name: users
|
||||
- name: persons
|
||||
|
||||
incremental:
|
||||
tests:
|
||||
- config_path: "secrets/config.json"
|
||||
configured_catalog_path: "integration_tests/configured_catalog.json"
|
||||
future_state:
|
||||
future_state_path: "integration_tests/abnormal_state.json"
|
||||
bypass_reason: "All incremental streams are empty in sandbox account."
|
||||
full_refresh:
|
||||
tests:
|
||||
- config_path: "secrets/config.json"
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
{"stream":"deal_fields","data":{"id":12488,"key":"eae9c2a5b618934581aebed0747cc33cd681379e","name":"Test field 3","order_nr":4,"field_type":"enum","json_column_flag":true,"add_time":"2023-02-22 11:04:13","update_time":"2023-02-22 11:04:13","last_updated_by_user_id":11884360,"edit_flag":true,"details_visible_flag":true,"add_visible_flag":true,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":true,"active_flag":true,"projects_detail_visible_flag":false,"show_in_pipelines":{"show_in_all":true,"pipeline_ids":[]},"options":[{"id":23,"label":"Test"}]},"emitted_at":1733742439563}
|
||||
{"stream":"deal_fields","data":{"id":12489,"key":"bed1d9f4cfdaf761fab04b38df20144b0fd156d6","name":"Test field 4","order_nr":5,"field_type":"varchar","json_column_flag":true,"add_time":"2023-02-22 11:04:30","update_time":"2023-02-22 11:04:30","last_updated_by_user_id":11884360,"edit_flag":true,"details_visible_flag":true,"add_visible_flag":true,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":true,"active_flag":true,"projects_detail_visible_flag":false,"show_in_pipelines":{"show_in_all":true,"pipeline_ids":[]}},"emitted_at":1733742439563}
|
||||
{"stream":"deal_fields","data":{"id":12490,"key":"41505adc22569bf93214dd7f7eaa10eaa387947d","name":"Test field 5","order_nr":6,"field_type":"varchar","json_column_flag":true,"add_time":"2023-02-22 11:04:43","update_time":"2023-02-22 11:04:43","last_updated_by_user_id":11884360,"edit_flag":true,"details_visible_flag":true,"add_visible_flag":true,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":true,"active_flag":true,"projects_detail_visible_flag":false,"show_in_pipelines":{"show_in_all":true,"pipeline_ids":[]}},"emitted_at":1733742439563}
|
||||
{"stream":"goals","data":{"id":"404e9d01204e052dff21ec32121f0270","owner_id":11884360,"title":"Activities added Team Airbyte","type":{"name":"activities_added","params":{"pipeline_id":null,"activity_type_id":[1]}},"assignee":{"type":"person","id":11884360},"expected_outcome":{"tracking_metric":"quantity","target":100},"interval":"weekly","duration":{"start":"2023-10-15","end":"2023-11-03"},"report_ids":["389fb0576579ea5b28761aef85af7e8a"],"is_active":false},"emitted_at":1733742439864}
|
||||
{"stream":"goals","data":{"id":"b1b124b30aea9704b28dde37a41b1cb2","owner_id":11884360,"title":"Activities completed Team Airbyte","type":{"name":"activities_completed","params":{"pipeline_id":null,"activity_type_id":[1]}},"assignee":{"type":"person","id":11884360},"expected_outcome":{"tracking_metric":"quantity","target":10},"interval":"weekly","duration":{"start":"2023-09-01","end":"2023-10-10"},"report_ids":["cf970db4608d730d53dc87875ebb2302"],"is_active":false},"emitted_at":1733742439890}
|
||||
{"stream":"goals","data":{"id":"b1b124b30aea9704b28dde37a41b2bcb","owner_id":11884360,"title":"Deals won Team Airbyte","type":{"name":"deals_won","params":{"pipeline_id":[1]}},"assignee":{"type":"person","id":11884360},"expected_outcome":{"tracking_metric":"sum","currency_id":148,"target":100},"interval":"monthly","duration":{"start":"2023-10-01","end":null},"report_ids":["ddbcc8dc48e84aea5e28a2db3488e89d"],"is_active":true},"emitted_at":1733742439890}
|
||||
{"stream":"lead_labels","data":{"id":"aecece60-c069-11eb-93bf-b59c4f1731e6","name":"Hot","color":"red","add_time":"2021-05-29T10:36:10.182Z","update_time":"2021-05-29T10:36:10.182Z"},"emitted_at":1733742439909}
|
||||
{"stream":"lead_labels","data":{"id":"aecece61-c069-11eb-93bf-b59c4f1731e6","name":"Warm","color":"yellow","add_time":"2021-05-29T10:36:10.182Z","update_time":"2021-05-29T10:36:10.182Z"},"emitted_at":1733742439910}
|
||||
{"stream":"lead_labels","data":{"id":"aecece62-c069-11eb-93bf-b59c4f1731e6","name":"Cold","color":"blue","add_time":"2021-05-29T10:36:10.182Z","update_time":"2021-05-29T10:36:10.182Z"},"emitted_at":1733742439910}
|
||||
{"stream":"activity_types","data":{"id":5,"order_nr":5,"name":"Email","key_string":"email","icon_key":"email","active_flag":true,"is_custom_flag":false,"add_time":"2020-12-10 07:23:48"},"emitted_at":1733742439922}
|
||||
{"stream":"activity_types","data":{"id":6,"order_nr":6,"name":"Lunch","key_string":"lunch","icon_key":"lunch","active_flag":true,"is_custom_flag":false,"add_time":"2020-12-10 07:23:48"},"emitted_at":1733742439922}
|
||||
{"stream":"activity_types","data":{"id":7,"order_nr":7,"name":"Test 1","key_string":"test_1","icon_key":"car","active_flag":true,"is_custom_flag":true,"add_time":"2023-02-22 11:13:54","update_time":"2023-02-22 11:13:54"},"emitted_at":1733742439922}
|
||||
{"stream":"leads","data":{"id":"56b65210-b28f-11ed-8345-ef71fee43bbf","title":"Test Organization 6 lead","owner_id":11884360,"creator_id":11884360,"label_ids":["aecece60-c069-11eb-93bf-b59c4f1731e6"],"value":{"amount":800,"currency":"USD"},"expected_close_date":"2023-03-30","person_id":7,"organization_id":6,"is_archived":false,"source_name":"Manually created","origin":"ManuallyCreated","was_seen":true,"next_activity_id":30,"add_time":"2023-02-22T09:00:22.321Z","update_time":"2023-02-22T11:48:49.834Z","visible_to":"3","cc_email":"airbyte-sandbox+7780468+lead54t3wth3rdv6nou8zygue80qn@pipedrivemail.com"},"emitted_at":1733742439928}
|
||||
{"stream":"leads","data":{"id":"11761620-b29b-11ed-83bc-c5d5bb21e359","title":"Test Organization 8 lead","owner_id":11884360,"creator_id":11884360,"label_ids":["aecece61-c069-11eb-93bf-b59c4f1731e6"],"value":{"amount":3000,"currency":"USD"},"expected_close_date":"2023-04-28","person_id":9,"organization_id":8,"is_archived":false,"source_name":"Manually created","origin":"ManuallyCreated","was_seen":true,"next_activity_id":31,"add_time":"2023-02-22T10:24:20.098Z","update_time":"2023-02-22T11:49:10.328Z","visible_to":"3","cc_email":"airbyte-sandbox+7780468+lead117qmm8o4y5irltqaigfmfdcp@pipedrivemail.com"},"emitted_at":1733742439928}
|
||||
{"stream":"leads","data":{"id":"98b9c5a0-b29b-11ed-83b4-fd61bd275e86","title":"Test Organization 10 lead","owner_id":11884360,"creator_id":11884360,"label_ids":["aecece60-c069-11eb-93bf-b59c4f1731e6"],"value":{"amount":2000,"currency":"USD"},"expected_close_date":"2023-05-30","person_id":11,"organization_id":10,"is_archived":false,"source_name":"Manually created","origin":"ManuallyCreated","was_seen":true,"next_activity_id":32,"add_time":"2023-02-22T10:28:07.034Z","update_time":"2023-02-22T11:49:24.853Z","visible_to":"3","cc_email":"airbyte-sandbox+7780468+lead91i2xdc9n1af14fee3k0llapy@pipedrivemail.com"},"emitted_at":1733742439928}
|
||||
{"stream":"organizations","data":{"id":12,"company_id":7780468,"owner_id":{"id":11884360,"name":"Team Airbyte","email":"integration-test@airbyte.io","has_pic":0,"pic_hash":null,"active_flag":true,"value":11884360},"name":"Test Organization 7","open_deals_count":0,"related_open_deals_count":0,"closed_deals_count":0,"related_closed_deals_count":0,"email_messages_count":0,"people_count":0,"activities_count":0,"done_activities_count":0,"undone_activities_count":0,"files_count":0,"notes_count":0,"followers_count":1,"won_deals_count":0,"related_won_deals_count":0,"lost_deals_count":0,"related_lost_deals_count":0,"active_flag":true,"first_char":"t","update_time":"2023-02-22 08:21:58","add_time":"2023-02-22 08:18:16","visible_to":"3","label":5,"label_ids":[5],"address":"DY Patil College, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra, India","address_subpremise":"","address_street_number":"","address_route":"","address_sublocality":"Pimpri Colony","address_locality":"Pimpri-Chinchwad","address_admin_area_level_1":"Maharashtra","address_admin_area_level_2":"Pune Division","address_country":"India","address_postal_code":"411018","address_formatted_address":"DY Patil College, DR. D Y PATIL MEDICAL COLLEGE, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra 411018, India","owner_name":"Team Airbyte","cc_email":"airbyte-sandbox@pipedrivemail.com"},"emitted_at":1733742440202}
|
||||
{"stream":"organizations","data":{"id":13,"company_id":7780468,"owner_id":{"id":11884360,"name":"Team Airbyte","email":"integration-test@airbyte.io","has_pic":0,"pic_hash":null,"active_flag":true,"value":11884360},"name":"Test Organization 6","open_deals_count":0,"related_open_deals_count":0,"closed_deals_count":0,"related_closed_deals_count":0,"email_messages_count":0,"people_count":0,"activities_count":0,"done_activities_count":0,"undone_activities_count":0,"files_count":0,"notes_count":0,"followers_count":1,"won_deals_count":0,"related_won_deals_count":0,"lost_deals_count":0,"related_lost_deals_count":0,"active_flag":true,"first_char":"t","update_time":"2023-02-22 08:23:17","add_time":"2023-02-22 08:18:33","visible_to":"3","label":5,"label_ids":[5],"address":"Anand Vihar Railway Station, Block D, Anand Vihar, Delhi, Uttar Pradesh, India","address_subpremise":"","address_street_number":"","address_route":"","address_sublocality":"Anand Vihar","address_locality":"Delhi","address_admin_area_level_1":"Uttar Pradesh","address_admin_area_level_2":"Delhi Division","address_country":"India","address_postal_code":"261205","address_formatted_address":"J8X8+F33, Block D, Anand Vihar, Delhi, Uttar Pradesh 261205, India","owner_name":"Team Airbyte","cc_email":"airbyte-sandbox@pipedrivemail.com"},"emitted_at":1733742440202}
|
||||
{"stream":"organizations","data":{"id":14,"company_id":7780468,"owner_id":{"id":11884360,"name":"Team Airbyte","email":"integration-test@airbyte.io","has_pic":0,"pic_hash":null,"active_flag":true,"value":11884360},"name":"Test Organization 8","open_deals_count":1,"related_open_deals_count":0,"closed_deals_count":0,"related_closed_deals_count":0,"email_messages_count":0,"people_count":1,"activities_count":0,"done_activities_count":0,"undone_activities_count":0,"files_count":0,"notes_count":0,"followers_count":1,"won_deals_count":0,"related_won_deals_count":0,"lost_deals_count":0,"related_lost_deals_count":0,"active_flag":true,"first_char":"t","update_time":"2023-10-13 13:25:00","add_time":"2023-02-22 08:18:50","visible_to":"3","label":5,"label_ids":[5],"address":"London Eye, London, UK","address_subpremise":"Riverside Building","address_street_number":"","address_route":"","address_sublocality":"","address_locality":"","address_admin_area_level_1":"","address_admin_area_level_2":"Greater London","address_country":"United Kingdom","address_postal_code":"SE1 7PB","address_formatted_address":"Riverside Building, County Hall, London SE1 7PB, UK","owner_name":"Team Airbyte","16a9e1fcbccc7f7a5f429a1840c9f66db9ee901a":"My Custom Field Value","cc_email":"airbyte-sandbox@pipedrivemail.com"},"emitted_at":1733742440202}
|
||||
{"stream":"organization_fields","data":{"key":"address_formatted_address","name":"Full/combined address of Address","field_type":"varchar","edit_flag":false,"active_flag":true,"is_subfield":true,"mandatory_flag":false,"parent_id":4021,"id_suffix":"formatted_address"},"emitted_at":1733742440211}
|
||||
{"stream":"organization_fields","data":{"id":4023,"key":"label_ids","name":"Labels","order_nr":0,"field_type":"set","json_column_flag":false,"add_time":"2024-04-29 09:28:14","edit_flag":false,"details_visible_flag":true,"add_visible_flag":true,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":false,"active_flag":true,"options":[{"id":5,"label":"Customer","color":"green"},{"id":6,"label":"Hot lead","color":"red"},{"id":7,"label":"Warm lead","color":"yellow"},{"id":8,"label":"Cold lead","color":"blue"}]},"emitted_at":1733742440211}
|
||||
{"stream":"organization_fields","data":{"id":4022,"key":"16a9e1fcbccc7f7a5f429a1840c9f66db9ee901a","name":"Pipedrive Custom Fields","order_nr":1,"field_type":"varchar","json_column_flag":true,"add_time":"2023-10-13 13:23:22","update_time":"2023-10-13 13:23:22","last_updated_by_user_id":11884360,"edit_flag":true,"details_visible_flag":true,"add_visible_flag":true,"important_flag":false,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":true,"active_flag":true},"emitted_at":1733742440211}
|
||||
{"stream":"permission_sets","data":{"id":"fa17fff0-db56-11ec-93f1-e9cfc58fcd59","name":"Global admin","assignment_count":1,"app":"global","type":"admin"},"emitted_at":1733742440211}
|
||||
{"stream":"permission_sets","data":{"id":"fa287ab0-db56-11ec-93f1-e9cfc58fcd59","name":"Global regular user","assignment_count":4,"app":"global","type":"regular"},"emitted_at":1733742440211}
|
||||
{"stream":"permission_sets","data":{"id":"57b60400-ed6c-11ec-88fb-4fe88bf2db36","name":"Account settings","assignment_count":1,"app":"account_settings","type":"admin"},"emitted_at":1733742440211}
|
||||
{"stream":"person_fields","data":{"id":9065,"key":"last_name","name":"Last name","order_nr":0,"field_type":"varchar","json_column_flag":false,"add_time":"2020-12-10 07:23:49","update_time":"2023-07-20 09:24:05","last_updated_by_user_id":0,"edit_flag":false,"details_visible_flag":true,"add_visible_flag":false,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":false,"active_flag":true},"emitted_at":1733742440213}
|
||||
{"stream":"person_fields","data":{"id":9067,"key":"label_ids","name":"Labels","order_nr":0,"field_type":"set","json_column_flag":false,"add_time":"2024-04-29 09:28:14","edit_flag":false,"details_visible_flag":true,"add_visible_flag":true,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":false,"active_flag":true,"options":[{"id":1,"label":"Customer","color":"green"},{"id":2,"label":"Hot lead","color":"red"},{"id":3,"label":"Warm lead","color":"yellow"},{"id":4,"label":"Cold lead","color":"blue"}]},"emitted_at":1733742440213}
|
||||
{"stream":"person_fields","data":{"id":9066,"key":"aa02d059909fdc632d590bd578d7b3baf4bf9780","name":"Custom Field 1","order_nr":1,"field_type":"varchar","json_column_flag":true,"add_time":"2023-07-12 17:53:46","update_time":"2023-07-12 17:53:46","last_updated_by_user_id":11884360,"edit_flag":true,"details_visible_flag":true,"add_visible_flag":false,"important_flag":false,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":true,"active_flag":true},"emitted_at":1733742440213}
|
||||
{"stream":"currencies","data":{"id":166,"code":"ZMK","name":"Zambian Kwacha","symbol":"ZMK","decimal_points":2,"active_flag":true,"is_custom_flag":false},"emitted_at":1733742440218}
|
||||
{"stream":"currencies","data":{"id":201,"code":"ZMW","name":"Zambian Kwacha","symbol":"ZMW","decimal_points":2,"active_flag":true,"is_custom_flag":false},"emitted_at":1733742440218}
|
||||
{"stream":"currencies","data":{"id":167,"code":"ZWL","name":"Zimbabwe Dollar","symbol":"ZWL","decimal_points":2,"active_flag":true,"is_custom_flag":false},"emitted_at":1733742440219}
|
||||
{"stream":"roles","data":{"id":1,"name":"(Unassigned users)","active_flag":true,"assignment_count":"5","sub_role_count":"0","level":1,"description":"This is the default group for managing your visibility settings. New users are added automatically unless you change their group when you invite them."},"emitted_at":1733742444262}
|
||||
{"stream":"product_fields","data":{"id":26,"key":"category","name":"Category","order_nr":0,"field_type":"enum","json_column_flag":false,"add_time":"2020-12-10 07:23:48","update_time":"2023-07-20 09:24:08","last_updated_by_user_id":11884360,"edit_flag":false,"details_visible_flag":true,"add_visible_flag":true,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":false,"active_flag":true,"options":[{"id":12,"label":"Food"}]},"emitted_at":1733742444467}
|
||||
{"stream":"product_fields","data":{"id":27,"key":"description","name":"Description","order_nr":0,"field_type":"text","json_column_flag":false,"add_time":"2020-12-10 07:23:48","update_time":"2023-07-20 09:24:08","last_updated_by_user_id":0,"edit_flag":false,"details_visible_flag":true,"add_visible_flag":false,"important_flag":true,"bulk_edit_allowed":true,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":false,"searchable_flag":false,"active_flag":true},"emitted_at":1733742444468}
|
||||
{"stream":"product_fields","data":{"id":28,"key":"unit_prices","name":"Unit prices","order_nr":0,"field_type":"double","json_column_flag":false,"add_time":"2020-12-10 07:23:48","update_time":"2023-09-21 07:51:19","last_updated_by_user_id":0,"edit_flag":false,"details_visible_flag":false,"add_visible_flag":true,"important_flag":true,"bulk_edit_allowed":false,"filtering_allowed":true,"sortable_flag":true,"mandatory_flag":true,"searchable_flag":false,"active_flag":true},"emitted_at":1733742444468}
|
||||
@@ -17,7 +17,7 @@ data:
|
||||
connectorSubtype: api
|
||||
connectorType: source
|
||||
definitionId: d8286229-c680-4063-8c59-23b9b391c700
|
||||
dockerImageTag: 2.2.28
|
||||
dockerImageTag: 2.3.0
|
||||
dockerRepository: airbyte/source-pipedrive
|
||||
documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive
|
||||
githubIssueLabel: source-pipedrive
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry]
|
||||
version = "2.2.28"
|
||||
version = "2.3.0"
|
||||
name = "source-pipedrive"
|
||||
description = "Source implementation for Pipedrive."
|
||||
authors = [ "Airbyte <contact@airbyte.io>",]
|
||||
@@ -16,8 +16,8 @@ repository = "https://github.com/airbytehq/airbyte"
|
||||
include = "source_pipedrive"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.9,<3.12"
|
||||
airbyte-cdk = "^0"
|
||||
python = "^3.10,<3.12"
|
||||
airbyte-cdk = "^6"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
source-pipedrive = "source_pipedrive.run:run"
|
||||
|
||||
@@ -411,3 +411,46 @@ spec:
|
||||
examples:
|
||||
- "2017-01-25 00:00:00Z"
|
||||
type: string
|
||||
num_workers:
|
||||
title: Number of concurrent workers
|
||||
description: Number of concurrent workers to be used in the sync. This number should be set according to your Pipedrive subscription plan and its rate limits, which coule be found <a href="https://pipedrive.readme.io/docs/core-api-concepts-rate-limiting">here</a> on Pipedrive Rate limiting page. Please note that Pipedrive API rate limits are reset daily. If you experience rate limiting issues, please lower the number of workers according to your plan.
|
||||
type: integer
|
||||
default: 10
|
||||
minimum: 1
|
||||
maximum: 40
|
||||
|
||||
# rate limiting https://pipedrive.readme.io/docs/core-api-concepts-rate-limiting
|
||||
|
||||
# Tokens Daily Budget:
|
||||
# Each company account is allocated a daily API token budget, which is shared among all users within that account.
|
||||
# This budget is exclusively for API traffic authenticated by API tokens or OAuth tokens,
|
||||
# and it does not impact actions performed directly within the Pipedrive user interface.
|
||||
# 30,000 base tokens × subscription plan multiplier × number of seats
|
||||
|
||||
#| Plan | Plan multiplier |
|
||||
#|:-------------|-----------------|
|
||||
#| Essential | 1 |
|
||||
#| Advanced | 2 |
|
||||
#| Professional | 3 |
|
||||
#| Power | 5 |
|
||||
#| Enterprise | 7 |
|
||||
|
||||
# API Requests:
|
||||
|
||||
# Each API request consumes a specific number of tokens, with each API endpoint assigned cost in tokens based on the complexity and resource demand of the endpoint.
|
||||
# When a request is made, the corresponding token cost is deducted from the company’s daily API budget.
|
||||
# Lightweight endpoints consume fewer tokens, while more complex or data-intensive endpoints require a higher token cost.
|
||||
|
||||
# | API Endpoint type | Cost in tokens |
|
||||
# |---------------------------------------------|
|
||||
# | Get list of entities | 20 |
|
||||
|
||||
# We assume that airbyte users mostly on Essential plan, so they have 30k(Tokens Daily Budget) * 20(Cost in tokens) requests per day, which should be enough with 10 workers.
|
||||
# If it's not, users can increase/decrease the number of workers.
|
||||
|
||||
# Max value of workers: 40, Min value of workers: 1, Default: 10.
|
||||
|
||||
concurrency_level:
|
||||
type: ConcurrencyLevel
|
||||
default_concurrency: "{{ config.get('num_workers', 10)}}"
|
||||
max_concurrency: 40
|
||||
|
||||
@@ -1,14 +1,53 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
from airbyte_cdk.entrypoint import launch
|
||||
from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger
|
||||
from airbyte_cdk.exception_handler import init_uncaught_exception_handler
|
||||
from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type
|
||||
from orjson import orjson
|
||||
from source_pipedrive import SourcePipedrive
|
||||
|
||||
|
||||
def run():
|
||||
source = SourcePipedrive()
|
||||
launch(source, sys.argv[1:])
|
||||
def _get_source(args: List[str]):
|
||||
catalog_path = AirbyteEntrypoint.extract_catalog(args)
|
||||
config_path = AirbyteEntrypoint.extract_config(args)
|
||||
state_path = AirbyteEntrypoint.extract_state(args)
|
||||
try:
|
||||
return SourcePipedrive(
|
||||
SourcePipedrive.read_catalog(catalog_path) if catalog_path else None,
|
||||
SourcePipedrive.read_config(config_path) if config_path else None,
|
||||
SourcePipedrive.read_state(state_path) if state_path else None,
|
||||
)
|
||||
except Exception as error:
|
||||
print(
|
||||
orjson.dumps(
|
||||
AirbyteMessageSerializer.dump(
|
||||
AirbyteMessage(
|
||||
type=Type.TRACE,
|
||||
trace=AirbyteTraceMessage(
|
||||
type=TraceType.ERROR,
|
||||
emitted_at=int(datetime.now().timestamp() * 1000),
|
||||
error=AirbyteErrorTraceMessage(
|
||||
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
|
||||
stack_trace=traceback.format_exc(),
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
).decode()
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def run() -> None:
|
||||
init_uncaught_exception_handler(logger)
|
||||
_args = sys.argv[1:]
|
||||
source = _get_source(_args)
|
||||
if source:
|
||||
launch(source, _args)
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
from airbyte_cdk.models import ConfiguredAirbyteCatalog
|
||||
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
||||
from airbyte_cdk.sources.source import TState
|
||||
|
||||
"""
|
||||
This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
|
||||
@@ -14,5 +17,5 @@ WARNING: Do not modify this file.
|
||||
|
||||
# Declarative Source
|
||||
class SourcePipedrive(YamlDeclarativeSource):
|
||||
def __init__(self):
|
||||
super().__init__(**{"path_to_yaml": "manifest.yaml"})
|
||||
def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: TState, **kwargs):
|
||||
super().__init__(catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"})
|
||||
|
||||
@@ -110,57 +110,58 @@ The Pipedrive connector will gracefully handle rate limits. For more information
|
||||
|
||||
## Changelog
|
||||
|
||||
| Version | Date | Pull Request | Subject |
|
||||
| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------- |
|
||||
| 2.2.28 | 2024-12-14 | [49692](https://github.com/airbytehq/airbyte/pull/49692) | Update dependencies |
|
||||
| 2.2.27 | 2024-12-12 | [49041](https://github.com/airbytehq/airbyte/pull/49041) | Starting with this version, the Docker image is now rootless. Please note that this and future versions will not be compatible with Airbyte versions earlier than 0.64 |
|
||||
| 2.2.26 | 2024-11-04 | [48293](https://github.com/airbytehq/airbyte/pull/48293) | Update dependencies |
|
||||
| 2.2.25 | 2024-10-29 | [47743](https://github.com/airbytehq/airbyte/pull/47743) | Update dependencies |
|
||||
| 2.2.24 | 2024-10-28 | [47103](https://github.com/airbytehq/airbyte/pull/47103) | Update dependencies |
|
||||
| 2.2.23 | 2024-10-12 | [46822](https://github.com/airbytehq/airbyte/pull/46822) | Update dependencies |
|
||||
| 2.2.22 | 2024-10-05 | [46487](https://github.com/airbytehq/airbyte/pull/46487) | Update dependencies |
|
||||
| 2.2.21 | 2024-09-28 | [46132](https://github.com/airbytehq/airbyte/pull/46132) | Update dependencies |
|
||||
| 2.2.20 | 2024-09-21 | [45748](https://github.com/airbytehq/airbyte/pull/45748) | Update dependencies |
|
||||
| 2.2.19 | 2024-09-14 | [45556](https://github.com/airbytehq/airbyte/pull/45556) | Update dependencies |
|
||||
| 2.2.18 | 2024-09-07 | [45303](https://github.com/airbytehq/airbyte/pull/45303) | Update dependencies |
|
||||
| 2.2.17 | 2024-08-31 | [44981](https://github.com/airbytehq/airbyte/pull/44981) | Update dependencies |
|
||||
| 2.2.16 | 2024-08-24 | [44644](https://github.com/airbytehq/airbyte/pull/44644) | Update dependencies |
|
||||
| 2.2.15 | 2024-08-17 | [44316](https://github.com/airbytehq/airbyte/pull/44316) | Update dependencies |
|
||||
| 2.2.14 | 2024-08-12 | [43888](https://github.com/airbytehq/airbyte/pull/43888) | Update dependencies |
|
||||
| 2.2.13 | 2024-08-10 | [43679](https://github.com/airbytehq/airbyte/pull/43679) | Update dependencies |
|
||||
| 2.2.12 | 2024-08-03 | [43056](https://github.com/airbytehq/airbyte/pull/43056) | Update dependencies |
|
||||
| 2.2.11 | 2024-07-27 | [42287](https://github.com/airbytehq/airbyte/pull/42287) | Update dependencies |
|
||||
| 2.2.10 | 2024-07-13 | [41729](https://github.com/airbytehq/airbyte/pull/41729) | Update dependencies |
|
||||
| 2.2.9 | 2024-07-10 | [41465](https://github.com/airbytehq/airbyte/pull/41465) | Update dependencies |
|
||||
| 2.2.8 | 2024-07-09 | [41082](https://github.com/airbytehq/airbyte/pull/41082) | Update dependencies |
|
||||
| 2.2.7 | 2024-07-06 | [40778](https://github.com/airbytehq/airbyte/pull/40778) | Update dependencies |
|
||||
| 2.2.6 | 2024-06-25 | [40501](https://github.com/airbytehq/airbyte/pull/40501) | Update dependencies |
|
||||
| 2.2.5 | 2024-06-22 | [40171](https://github.com/airbytehq/airbyte/pull/40171) | Update dependencies |
|
||||
| 2.2.4 | 2024-06-04 | [39095](https://github.com/airbytehq/airbyte/pull/39095) | [autopull] Upgrade base image to v1.2.1 |
|
||||
| 2.2.3 | 2024-05-20 | [38405](https://github.com/airbytehq/airbyte/pull/38405) | [autopull] base image + poetry + up_to_date |
|
||||
| 2.2.2 | 2024-01-11 | [34153](https://github.com/airbytehq/airbyte/pull/34153) | prepare for airbyte-lib |
|
||||
| 2.2.1 | 2023-11-06 | [31147](https://github.com/airbytehq/airbyte/pull/31147) | Bugfix: handle records with a null data field |
|
||||
| 2.2.0 | 2023-10-25 | [31707](https://github.com/airbytehq/airbyte/pull/31707) | Add new stream mail |
|
||||
| 2.1.0 | 2023-10-10 | [31184](https://github.com/airbytehq/airbyte/pull/31184) | Add new stream goals |
|
||||
| 2.0.1 | 2023-10-13 | [31151](https://github.com/airbytehq/airbyte/pull/31151) | Add additionalProperties in schemas to read custom fields |
|
||||
| 2.0.0 | 2023-08-09 | [29293](https://github.com/airbytehq/airbyte/pull/29293) | Migrated to Low-Code CDK |
|
||||
| 1.0.0 | 2023-06-29 | [27832](https://github.com/airbytehq/airbyte/pull/27832) | Remove `followers_count` field from `Products` stream |
|
||||
| 0.1.19 | 2023-07-05 | [27967](https://github.com/airbytehq/airbyte/pull/27967) | Update `OrganizationFields` and `ProductFields` with `display_field` field |
|
||||
| 0.1.18 | 2023-06-02 | [26892](https://github.com/airbytehq/airbyte/pull/26892) | Update `DialFields` schema with `pipeline_ids` property |
|
||||
| 0.1.17 | 2023-03-21 | [24282](https://github.com/airbytehq/airbyte/pull/24282) | Bugfix handle missed `cursor_field` |
|
||||
| 0.1.16 | 2023-03-08 | [23789](https://github.com/airbytehq/airbyte/pull/23789) | Add 11 new streams |
|
||||
| 0.1.15 | 2023-03-02 | [23705](https://github.com/airbytehq/airbyte/pull/23705) | Disable OAuth |
|
||||
| 0.1.14 | 2023-03-01 | [23539](https://github.com/airbytehq/airbyte/pull/23539) | Fix schema for "activities", "check" works if empty "deals" |
|
||||
| 0.1.13 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state |
|
||||
| 0.1.12 | 2022-05-12 | [12806](https://github.com/airbytehq/airbyte/pull/12806) | Remove date-time format from schemas |
|
||||
| 0.1.10 | 2022-04-26 | [11870](https://github.com/airbytehq/airbyte/pull/11870) | Add 3 streams: DealFields, OrganizationFields and PersonFields |
|
||||
| 0.1.9 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description |
|
||||
| 0.1.8 | 2021-11-16 | [7875](https://github.com/airbytehq/airbyte/pull/7875) | Extend schema for "persons" stream |
|
||||
| 0.1.7 | 2021-11-15 | [7968](https://github.com/airbytehq/airbyte/pull/7968) | Update oAuth flow config |
|
||||
| 0.1.6 | 2021-10-05 | [6821](https://github.com/airbytehq/airbyte/pull/6821) | Add OAuth support |
|
||||
| 0.1.5 | 2021-09-27 | [6441](https://github.com/airbytehq/airbyte/pull/6441) | Fix normalization error |
|
||||
| 0.1.4 | 2021-08-26 | [5943](https://github.com/airbytehq/airbyte/pull/5943) | Add organizations stream |
|
||||
| 0.1.3 | 2021-08-26 | [5642](https://github.com/airbytehq/airbyte/pull/5642) | Remove date-time from deals stream |
|
||||
| 0.1.2 | 2021-07-23 | [4912](https://github.com/airbytehq/airbyte/pull/4912) | Update money type to support floating point |
|
||||
| 0.1.1 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Update spec.json |
|
||||
| 0.1.0 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | 🎉 New source: Pipedrive connector |
|
||||
| Version | Date | Pull Request | Subject |
|
||||
|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| 2.3.0 | 2024-12-17 | [48615](https://github.com/airbytehq/airbyte/pull/48615) | Update airbyte-cdk to use concurrency |
|
||||
| 2.2.28 | 2024-12-14 | [49692](https://github.com/airbytehq/airbyte/pull/49692) | Update dependencies |
|
||||
| 2.2.27 | 2024-12-12 | [49041](https://github.com/airbytehq/airbyte/pull/49041) | Starting with this version, the Docker image is now rootless. Please note that this and future versions will not be compatible with Airbyte versions earlier than 0.64 |
|
||||
| 2.2.26 | 2024-11-04 | [48293](https://github.com/airbytehq/airbyte/pull/48293) | Update dependencies |
|
||||
| 2.2.25 | 2024-10-29 | [47743](https://github.com/airbytehq/airbyte/pull/47743) | Update dependencies |
|
||||
| 2.2.24 | 2024-10-28 | [47103](https://github.com/airbytehq/airbyte/pull/47103) | Update dependencies |
|
||||
| 2.2.23 | 2024-10-12 | [46822](https://github.com/airbytehq/airbyte/pull/46822) | Update dependencies |
|
||||
| 2.2.22 | 2024-10-05 | [46487](https://github.com/airbytehq/airbyte/pull/46487) | Update dependencies |
|
||||
| 2.2.21 | 2024-09-28 | [46132](https://github.com/airbytehq/airbyte/pull/46132) | Update dependencies |
|
||||
| 2.2.20 | 2024-09-21 | [45748](https://github.com/airbytehq/airbyte/pull/45748) | Update dependencies |
|
||||
| 2.2.19 | 2024-09-14 | [45556](https://github.com/airbytehq/airbyte/pull/45556) | Update dependencies |
|
||||
| 2.2.18 | 2024-09-07 | [45303](https://github.com/airbytehq/airbyte/pull/45303) | Update dependencies |
|
||||
| 2.2.17 | 2024-08-31 | [44981](https://github.com/airbytehq/airbyte/pull/44981) | Update dependencies |
|
||||
| 2.2.16 | 2024-08-24 | [44644](https://github.com/airbytehq/airbyte/pull/44644) | Update dependencies |
|
||||
| 2.2.15 | 2024-08-17 | [44316](https://github.com/airbytehq/airbyte/pull/44316) | Update dependencies |
|
||||
| 2.2.14 | 2024-08-12 | [43888](https://github.com/airbytehq/airbyte/pull/43888) | Update dependencies |
|
||||
| 2.2.13 | 2024-08-10 | [43679](https://github.com/airbytehq/airbyte/pull/43679) | Update dependencies |
|
||||
| 2.2.12 | 2024-08-03 | [43056](https://github.com/airbytehq/airbyte/pull/43056) | Update dependencies |
|
||||
| 2.2.11 | 2024-07-27 | [42287](https://github.com/airbytehq/airbyte/pull/42287) | Update dependencies |
|
||||
| 2.2.10 | 2024-07-13 | [41729](https://github.com/airbytehq/airbyte/pull/41729) | Update dependencies |
|
||||
| 2.2.9 | 2024-07-10 | [41465](https://github.com/airbytehq/airbyte/pull/41465) | Update dependencies |
|
||||
| 2.2.8 | 2024-07-09 | [41082](https://github.com/airbytehq/airbyte/pull/41082) | Update dependencies |
|
||||
| 2.2.7 | 2024-07-06 | [40778](https://github.com/airbytehq/airbyte/pull/40778) | Update dependencies |
|
||||
| 2.2.6 | 2024-06-25 | [40501](https://github.com/airbytehq/airbyte/pull/40501) | Update dependencies |
|
||||
| 2.2.5 | 2024-06-22 | [40171](https://github.com/airbytehq/airbyte/pull/40171) | Update dependencies |
|
||||
| 2.2.4 | 2024-06-04 | [39095](https://github.com/airbytehq/airbyte/pull/39095) | [autopull] Upgrade base image to v1.2.1 |
|
||||
| 2.2.3 | 2024-05-20 | [38405](https://github.com/airbytehq/airbyte/pull/38405) | [autopull] base image + poetry + up_to_date |
|
||||
| 2.2.2 | 2024-01-11 | [34153](https://github.com/airbytehq/airbyte/pull/34153) | prepare for airbyte-lib |
|
||||
| 2.2.1 | 2023-11-06 | [31147](https://github.com/airbytehq/airbyte/pull/31147) | Bugfix: handle records with a null data field |
|
||||
| 2.2.0 | 2023-10-25 | [31707](https://github.com/airbytehq/airbyte/pull/31707) | Add new stream mail |
|
||||
| 2.1.0 | 2023-10-10 | [31184](https://github.com/airbytehq/airbyte/pull/31184) | Add new stream goals |
|
||||
| 2.0.1 | 2023-10-13 | [31151](https://github.com/airbytehq/airbyte/pull/31151) | Add additionalProperties in schemas to read custom fields |
|
||||
| 2.0.0 | 2023-08-09 | [29293](https://github.com/airbytehq/airbyte/pull/29293) | Migrated to Low-Code CDK |
|
||||
| 1.0.0 | 2023-06-29 | [27832](https://github.com/airbytehq/airbyte/pull/27832) | Remove `followers_count` field from `Products` stream |
|
||||
| 0.1.19 | 2023-07-05 | [27967](https://github.com/airbytehq/airbyte/pull/27967) | Update `OrganizationFields` and `ProductFields` with `display_field` field |
|
||||
| 0.1.18 | 2023-06-02 | [26892](https://github.com/airbytehq/airbyte/pull/26892) | Update `DialFields` schema with `pipeline_ids` property |
|
||||
| 0.1.17 | 2023-03-21 | [24282](https://github.com/airbytehq/airbyte/pull/24282) | Bugfix handle missed `cursor_field` |
|
||||
| 0.1.16 | 2023-03-08 | [23789](https://github.com/airbytehq/airbyte/pull/23789) | Add 11 new streams |
|
||||
| 0.1.15 | 2023-03-02 | [23705](https://github.com/airbytehq/airbyte/pull/23705) | Disable OAuth |
|
||||
| 0.1.14 | 2023-03-01 | [23539](https://github.com/airbytehq/airbyte/pull/23539) | Fix schema for "activities", "check" works if empty "deals" |
|
||||
| 0.1.13 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state |
|
||||
| 0.1.12 | 2022-05-12 | [12806](https://github.com/airbytehq/airbyte/pull/12806) | Remove date-time format from schemas |
|
||||
| 0.1.10 | 2022-04-26 | [11870](https://github.com/airbytehq/airbyte/pull/11870) | Add 3 streams: DealFields, OrganizationFields and PersonFields |
|
||||
| 0.1.9 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description |
|
||||
| 0.1.8 | 2021-11-16 | [7875](https://github.com/airbytehq/airbyte/pull/7875) | Extend schema for "persons" stream |
|
||||
| 0.1.7 | 2021-11-15 | [7968](https://github.com/airbytehq/airbyte/pull/7968) | Update oAuth flow config |
|
||||
| 0.1.6 | 2021-10-05 | [6821](https://github.com/airbytehq/airbyte/pull/6821) | Add OAuth support |
|
||||
| 0.1.5 | 2021-09-27 | [6441](https://github.com/airbytehq/airbyte/pull/6441) | Fix normalization error |
|
||||
| 0.1.4 | 2021-08-26 | [5943](https://github.com/airbytehq/airbyte/pull/5943) | Add organizations stream |
|
||||
| 0.1.3 | 2021-08-26 | [5642](https://github.com/airbytehq/airbyte/pull/5642) | Remove date-time from deals stream |
|
||||
| 0.1.2 | 2021-07-23 | [4912](https://github.com/airbytehq/airbyte/pull/4912) | Update money type to support floating point |
|
||||
| 0.1.1 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Update spec.json |
|
||||
| 0.1.0 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | 🎉 New source: Pipedrive connector |
|
||||
|
||||
Reference in New Issue
Block a user