1
0
mirror of synced 2025-12-25 02:09:19 -05:00

Source Shopify: fix for missing properties & items in schemas (#7783)

This commit is contained in:
Baz
2021-11-30 18:02:06 +02:00
committed by GitHub
parent ff87cffa84
commit a55ef51f48
15 changed files with 2215 additions and 400 deletions

View File

@@ -2,7 +2,7 @@
"sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9",
"name": "Shopify",
"dockerRepository": "airbyte/source-shopify",
"dockerImageTag": "0.1.23",
"dockerImageTag": "0.1.24",
"documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify",
"icon": "shopify.svg"
}

View File

@@ -582,7 +582,7 @@
- name: Shopify
sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
dockerRepository: airbyte/source-shopify
dockerImageTag: 0.1.23
dockerImageTag: 0.1.24
documentationUrl: https://docs.airbyte.io/integrations/sources/shopify
icon: shopify.svg
sourceType: api

View File

@@ -5488,7 +5488,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-shopify:0.1.23"
- dockerImage: "airbyte/source-shopify:0.1.24"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/shopify"
connectionSpecification:

View File

@@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.23
LABEL io.airbyte.version=0.1.24
LABEL io.airbyte.name=airbyte/source-shopify

View File

@@ -17,8 +17,10 @@ tests:
basic_read:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
timeout_seconds: 3600
# some streams hold data only for some time, therefore certain streams could be empty while sync.
# 'abandoned_checkouts' stream holds data up to 1 month.
empty_streams: ["abandoned_checkouts"]
timeout_seconds: 1200
incremental:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
@@ -26,4 +28,4 @@ tests:
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
timeout_seconds: 1200
timeout_seconds: 3600

View File

@@ -1,7 +1,20 @@
{
"type": "object",
"properties": {
"note_attributes": {},
"note_attributes": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"name": {
"type": ["null", "string"]
},
"value": {
"type": ["null", "string"]
}
}
}
},
"location_id": {
"type": ["null", "integer"]
},
@@ -115,7 +128,33 @@
"tax_lines": {
"items": {
"properties": {
"price_set": {},
"price_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"price": {
"type": ["null", "number"]
},
@@ -173,10 +212,93 @@
"line_items": {
"items": {
"properties": {
"applied_discounts": {},
"total_discount_set": {},
"pre_tax_price_set": {},
"price_set": {},
"applied_discounts": {
"type": ["null", "array"],
"items": {
"type": ["null", "string"]
}
},
"total_discount_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"pre_tax_price_set": {
"properties": {
"shop_money": {
"properties": {
"currency_code": {
"type": ["null", "string"]
},
"amount": {
"type": ["null", "number"]
}
},
"type": ["null", "object"]
},
"presentment_money": {
"properties": {
"currency_code": {
"type": ["null", "string"]
},
"amount": {
"type": ["null", "number"]
}
},
"type": ["null", "object"]
}
},
"type": ["null", "object"]
},
"price_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"grams": {
"type": ["null", "integer"]
},
@@ -221,8 +343,8 @@
"type": ["null", "array"]
},
{
"properties": {},
"type": ["null", "object"]
"type": ["null", "object"],
"additionalProperties": true
}
]
},
@@ -230,19 +352,45 @@
"type": ["null", "string"]
},
"discount_allocations": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"discount_application_index": {
"type": ["null", "integer"]
},
"amount_set": {},
"amount_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"amount": {
"type": ["null", "number"]
}
},
"type": ["null", "object"]
},
"type": ["null", "array"]
}
}
},
"admin_graphql_api_id": {
"type": ["null", "string"]
@@ -281,7 +429,33 @@
"tax_lines": {
"items": {
"properties": {
"price_set": {},
"price_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"price": {
"type": ["null", "number"]
},
@@ -412,8 +586,12 @@
"items": {
"type": ["null", "object"],
"properties": {
"applied_discounts": {},
"custom_tax_lines": {},
"applied_discounts": {
"type": ["null", "array"],
"items": {
"type": ["null", "string"]
}
},
"phone": {
"type": ["null", "string"]
},
@@ -444,7 +622,33 @@
"tax_lines": {
"items": {
"properties": {
"price_set": {},
"price_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "number"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"price": {
"type": ["null", "number"]
},
@@ -716,7 +920,7 @@
"tags": {
"type": ["null", "string"]
},
"tax_exempt": {
"tax_exemptions": {
"type": ["null", "boolean"]
},
"id": {

View File

@@ -54,5 +54,5 @@
"type": ["null", "string"]
}
},
"type": "object"
"type": ["null", "object"]
}

View File

@@ -1,5 +1,5 @@
{
"type": "object",
"type": ["null", "object"],
"properties": {
"last_order_name": {
"type": ["null", "string"]

View File

@@ -29,13 +29,12 @@
"type": ["null", "string"]
},
"value": {
"type": ["null", "integer", "object", "string"],
"properties": {}
"type": ["null", "number", "string", "integer", "boolean"]
},
"updated_at": {
"type": ["null", "string"],
"format": "date-time"
}
},
"type": "object"
"type": ["null", "object"]
}

View File

@@ -660,9 +660,6 @@
"marketing_opt_in_level": {
"type": ["null", "string"]
},
"tax_exemptions": {
"type": ["null", "array"]
},
"admin_graphql_api_id": {
"type": ["null", "string"]
},
@@ -724,8 +721,59 @@
}
}
},
"discount_applications": {
"type": ["null", "array"]
"discount_allocations": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "string"]
},
"amount": {
"type": ["null", "string"]
},
"description": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"],
"format": "date-time"
},
"discount_application_index": {
"type": ["null", "number"]
},
"amount_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"application_type": {
"type": ["null", "string"]
}
}
}
},
"fulfillments": {
"type": ["null", "array"],
@@ -1106,9 +1154,19 @@
"items": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "string"]
},
"amount": {
"type": ["null", "string"]
},
"description": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"],
"format": "date-time"
},
"discount_application_index": {
"type": ["null", "number"]
},
@@ -1138,6 +1196,9 @@
}
}
}
},
"application_type": {
"type": ["null", "string"]
}
}
}
@@ -1472,9 +1533,19 @@
"items": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "string"]
},
"amount": {
"type": ["null", "string"]
},
"description": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"],
"format": "date-time"
},
"discount_application_index": {
"type": ["null", "number"]
},
@@ -1504,6 +1575,9 @@
}
}
}
},
"application_type": {
"type": ["null", "string"]
}
}
}
@@ -1532,7 +1606,490 @@
}
},
"refunds": {
"type": ["null", "array"]
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "integer"]
},
"admin_graphql_api_id": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"],
"format": "date-time"
},
"note": {
"type": ["null", "string"]
},
"order_id": {
"type": ["null", "integer"]
},
"processed_at": {
"type": ["null", "string"],
"format": "date-time"
},
"restock": {
"type": ["null", "boolean"]
},
"user_id": {
"type": ["null", "integer"]
},
"order_adjustments": {
"type": ["null", "array"],
"items": {
"type": ["null","object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"amount_set": {
"type": ["null", "object"],
"properties": {
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"id": {
"type": ["null", "integer"]
},
"kind": {
"type": ["null", "string"]
},
"order_id": {
"type": ["null", "integer"]
},
"reason": {
"type": ["null", "string"]
},
"refund_id": {
"type": ["null", "integer"]
},
"tax_amount": {
"type": ["null", "string"]
},
"tax_amount_set": {
"type": ["null", "object"],
"properties": {
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
}
}
}
},
"transactions": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "integer"]
},
"admin_graphql_api_id": {
"type": ["null", "string"]
},
"amount": {
"type": ["null", "string"]
},
"authorization": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"]
},
"currency": {
"type": ["null", "string"]
},
"device_id": {
"type": ["null", "integer"]
},
"error_code": {
"type": ["null", "string"]
},
"gateway": {
"type": ["null", "string"]
},
"kind": {
"type": ["null", "string"]
},
"location_id": {
"type": ["null", "integer"]
},
"message": {
"type": ["null", "string"]
},
"order_id": {
"type": ["null", "integer"]
},
"parent_id": {
"type": ["null", "integer"]
},
"processed_at": {
"type": ["null", "string"]
},
"receipt": {
"type": ["null", "object"],
"properties": {
"paid_amount": {
"type": ["null", "string"]
}
}
},
"source_name": {
"type": ["null", "string"]
},
"status": {
"type": ["null", "string"]
},
"test": {
"type": ["null", "boolean"]
},
"user_id": {
"type": ["null", "integer"]
}
}
}
},
"refund_line_items": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "integer"]
},
"line_item_id": {
"type": ["null", "integer"]
},
"location_id": {
"type": ["null", "integer"]
},
"quantity": {
"type": ["null", "integer"]
},
"restock_type": {
"type": ["null", "string"]
},
"subtotal": {
"type": ["null", "number"]
},
"subtotal_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"total_tax": {
"type": ["null", "number"]
},
"total_tax_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"line_item": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "integer"]
},
"admin_graphql_api_id": {
"type": ["null", "string"]
},
"fulfillable_quantity": {
"type": ["null", "integer"]
},
"fulfillment_service": {
"type": ["null", "string"]
},
"fulfillment_status": {
"type": ["null", "string"]
},
"gift_card": {
"type": ["null", "boolean"]
},
"grams": {
"type": ["null", "number"]
},
"name": {
"type": ["null", "string"]
},
"price": {
"type": ["null", "string"]
},
"price_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"product_exists": {
"type": ["null", "boolean"]
},
"product_id": {
"type": ["null", "integer"]
},
"properties": {
"type": ["null", "array"],
"items": {
"type": ["null", "string"]
}
},
"quantity": {
"type": ["null", "integer"]
},
"requires_shipping": {
"type": ["null", "boolean"]
},
"sku": {
"type": ["null", "string"]
},
"taxable": {
"type": ["null", "boolean"]
},
"title": {
"type": ["null", "string"]
},
"total_discount": {
"type": ["null", "string"]
},
"total_discount_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"variant_id": {
"type": ["null", "integer"]
},
"variant_inventory_management": {
"type": ["null", "string"]
},
"variant_title": {
"type": ["null", "string"]
},
"vendor": {
"type": ["null", "string"]
},
"tax_lines": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"price": {
"type": ["null", "string"]
},
"price_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"rate": {
"type": ["null", "number"]
},
"title": {
"type": ["null", "string"]
}
}
}
},
"discount_allocations": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"amount_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"discount_application_index": {
"type": ["null", "number"]
}
}
}
}
}
}
}
}
}
}
}
},
"shipping_address": {
"type": ["null", "object"],
@@ -1677,7 +2234,58 @@
"type": ["null", "array"]
},
"discount_allocations": {
"type": ["null", "array"]
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"properties": {
"id": {
"type": ["null", "string"]
},
"amount": {
"type": ["null", "string"]
},
"description": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"],
"format": "date-time"
},
"discount_application_index": {
"type": ["null", "number"]
},
"amount_set": {
"type": ["null", "object"],
"properties": {
"shop_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
},
"presentment_money": {
"type": ["null", "object"],
"properties": {
"amount": {
"type": ["null", "string"]
},
"currency_code": {
"type": ["null", "string"]
}
}
}
}
},
"application_type": {
"type": ["null", "string"]
}
}
}
}
}
}

View File

@@ -1,5 +1,5 @@
{
"type": "object",
"type": ["null", "object"],
"properties": {
"allocation_method": {
"type": ["null", "string"]

View File

@@ -87,14 +87,11 @@
"type": ["null", "number"],
"multipleOf": 1e-10
}
},
"patternProperties": {
".+": {}
}
},
"location_id": {
"type": ["null", "integer"]
}
},
"type": "object"
"type": ["null", "object"]
}

View File

@@ -72,7 +72,6 @@ class ShopifyStream(HttpStream, ABC):
"""The name of the field in the response which contains the data"""
# Basic incremental stream
class IncrementalShopifyStream(ShopifyStream, ABC):
# Setting the check point interval to the limit of the records output
@@ -103,7 +102,7 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
# Getting records >= state
if stream_state:
for record in records_slice:
if record.get(self.cursor_field) >= stream_state.get(self.cursor_field):
if record.get(self.cursor_field, "") >= stream_state.get(self.cursor_field):
yield record
else:
yield from records_slice
@@ -343,7 +342,6 @@ class Locations(ShopifyStream):
class InventoryLevels(ChildSubstream):
parent_stream_class: object = Locations
slice_key = "location_id"
cursor_field = "updated_at"
data_field = "inventory_levels"
@@ -359,10 +357,7 @@ class InventoryLevels(ChildSubstream):
return record
# associate the surrogate key
yield from map(
generate_key,
records_stream,
)
yield from map(generate_key, records_stream)
class InventoryItems(ChildSubstream):

View File

@@ -100,6 +100,7 @@ This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applica
| Version | Date | Pull Request | Subject |
| :--- | :--- | :--- | :--- |
| 0.1.24 | 2021-11-30 | [7783](https://github.com/airbytehq/airbyte/pull/7783) | Reviewed and corrected schemas for all streams |
| 0.1.23 | 2021-11-15 | [7973](https://github.com/airbytehq/airbyte/pull/7973) | Added `InventoryItems` |
| 0.1.22 | 2021-10-18 | [7101](https://github.com/airbytehq/airbyte/pull/7107) | Added FulfillmentOrders, Fulfillments streams |
| 0.1.21 | 2021-10-14 | [7382](https://github.com/airbytehq/airbyte/pull/7382) | Fixed `InventoryLevels` primary key |