✨ Source S3: v4 rollout - take 3 (#30153)
Co-authored-by: Catherine Noll <clnoll@users.noreply.github.com>
This commit is contained in:
Maxime Carbonneau-Leclerc
committed by
GitHub
GitHub
parent
de31c73b65
commit
4e7c70f767
@@ -17,5 +17,5 @@ COPY source_s3 ./source_s3
|
||||
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
|
||||
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
|
||||
|
||||
LABEL io.airbyte.version=3.1.11
|
||||
LABEL io.airbyte.version=4.0.0
|
||||
LABEL io.airbyte.name=airbyte/source-s3
|
||||
|
||||
@@ -6,47 +6,47 @@ acceptance_tests:
|
||||
path: integration_tests/expected_records/csv.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_custom_encoding_config.json
|
||||
- config_path: secrets/v4_csv_custom_encoding_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_custom_encoding.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_custom_format_config.json
|
||||
- config_path: secrets/v4_csv_custom_format_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_custom_format.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_user_schema_config.json
|
||||
- config_path: secrets/v4_csv_user_schema_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_user_schema.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_no_header_config.json
|
||||
- config_path: secrets/v4_csv_no_header_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_no_header.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_skip_rows_config.json
|
||||
- config_path: secrets/v4_csv_skip_rows_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_skip_rows.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_skip_rows_no_header_config.json
|
||||
- config_path: secrets/v4_csv_skip_rows_no_header_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_with_nulls_config.json
|
||||
- config_path: secrets/v4_csv_with_nulls_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_with_nulls.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_csv_with_null_bools_config.json
|
||||
- config_path: secrets/v4_csv_with_null_bools_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_csv_with_null_bools.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/parquet_config.json
|
||||
- config_path: secrets/v4_parquet_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/parquet.jsonl
|
||||
exact_order: true
|
||||
@@ -56,21 +56,21 @@ acceptance_tests:
|
||||
path: integration_tests/expected_records/parquet_dataset.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/legacy_parquet_decimal_config.json
|
||||
- config_path: secrets/v4_parquet_decimal_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/legacy_parquet_decimal.jsonl
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/avro_config.json
|
||||
- config_path: secrets/v4_avro_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/avro.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/jsonl_config.json
|
||||
- config_path: secrets/v4_jsonl_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/jsonl.jsonl
|
||||
exact_order: true
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/jsonl_newlines_config.json
|
||||
- config_path: secrets/v4_jsonl_newlines_config.json
|
||||
expect_records:
|
||||
path: integration_tests/expected_records/jsonl_newlines.jsonl
|
||||
exact_order: true
|
||||
@@ -80,60 +80,82 @@ acceptance_tests:
|
||||
tests:
|
||||
- config_path: secrets/config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_custom_encoding_config.json
|
||||
- config_path: secrets/v4_csv_custom_encoding_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_custom_format_config.json
|
||||
- config_path: secrets/v4_csv_custom_format_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_user_schema_config.json
|
||||
- config_path: secrets/v4_csv_user_schema_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_no_header_config.json
|
||||
- config_path: secrets/v4_csv_no_header_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_skip_rows_config.json
|
||||
- config_path: secrets/v4_csv_skip_rows_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_skip_rows_no_header_config.json
|
||||
- config_path: secrets/v4_csv_skip_rows_no_header_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_with_nulls_config.json
|
||||
- config_path: secrets/v4_csv_with_nulls_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/legacy_csv_with_null_bools_config.json
|
||||
- config_path: secrets/v4_csv_with_null_bools_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/parquet_config.json
|
||||
- config_path: secrets/v4_parquet_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/avro_config.json
|
||||
- config_path: secrets/v4_avro_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/jsonl_config.json
|
||||
- config_path: secrets/v4_jsonl_config.json
|
||||
status: succeed
|
||||
- config_path: secrets/jsonl_newlines_config.json
|
||||
- config_path: secrets/v4_jsonl_newlines_config.json
|
||||
status: succeed
|
||||
- config_path: integration_tests/invalid_config.json
|
||||
status: failed
|
||||
discovery:
|
||||
tests:
|
||||
- config_path: secrets/config.json
|
||||
- config_path: secrets/legacy_csv_custom_encoding_config.json
|
||||
- config_path: secrets/legacy_csv_custom_format_config.json
|
||||
- config_path: secrets/legacy_csv_user_schema_config.json
|
||||
- config_path: secrets/legacy_csv_no_header_config.json
|
||||
- config_path: secrets/legacy_csv_skip_rows_config.json
|
||||
- config_path: secrets/legacy_csv_with_nulls_config.json
|
||||
- config_path: secrets/parquet_config.json
|
||||
- config_path: secrets/avro_config.json
|
||||
- config_path: secrets/jsonl_config.json
|
||||
- config_path: secrets/jsonl_newlines_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_csv_custom_encoding_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_csv_custom_format_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_csv_user_schema_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_csv_no_header_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_csv_skip_rows_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_csv_with_nulls_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_parquet_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_avro_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_jsonl_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
- config_path: secrets/v4_jsonl_newlines_config.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
full_refresh:
|
||||
tests:
|
||||
- config_path: secrets/config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/csv.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/parquet_config.json
|
||||
- config_path: secrets/v4_parquet_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/parquet.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/avro_config.json
|
||||
- config_path: secrets/v4_avro_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/avro.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/jsonl_config.json
|
||||
- config_path: secrets/v4_jsonl_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/jsonl_newlines_config.json
|
||||
- config_path: secrets/v4_jsonl_newlines_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
|
||||
timeout_seconds: 1800
|
||||
|
||||
@@ -147,7 +169,7 @@ acceptance_tests:
|
||||
future_state:
|
||||
future_state_path: integration_tests/abnormal_state.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/parquet_config.json
|
||||
- config_path: secrets/v4_parquet_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/parquet.json
|
||||
cursor_paths:
|
||||
test:
|
||||
@@ -155,7 +177,7 @@ acceptance_tests:
|
||||
future_state:
|
||||
future_state_path: integration_tests/abnormal_state.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/avro_config.json
|
||||
- config_path: secrets/v4_avro_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/avro.json
|
||||
cursor_paths:
|
||||
test:
|
||||
@@ -163,7 +185,7 @@ acceptance_tests:
|
||||
future_state:
|
||||
future_state_path: integration_tests/abnormal_state.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/jsonl_config.json
|
||||
- config_path: secrets/v4_jsonl_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
|
||||
cursor_paths:
|
||||
test:
|
||||
@@ -171,7 +193,7 @@ acceptance_tests:
|
||||
future_state:
|
||||
future_state_path: integration_tests/abnormal_state.json
|
||||
timeout_seconds: 1800
|
||||
- config_path: secrets/jsonl_newlines_config.json
|
||||
- config_path: secrets/v4_jsonl_newlines_config.json
|
||||
configured_catalog_path: integration_tests/configured_catalogs/jsonl.json
|
||||
cursor_paths:
|
||||
test:
|
||||
@@ -182,5 +204,7 @@ acceptance_tests:
|
||||
spec:
|
||||
tests:
|
||||
- spec_path: integration_tests/spec.json
|
||||
backward_compatibility_tests_config:
|
||||
disable_for_version: "3.1.11" # Switch to v4 changed config shape
|
||||
connector_image: airbyte/source-s3:dev
|
||||
test_strictness_level: high
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
from typing import Any
|
||||
|
||||
from airbyte_cdk import AirbyteLogger
|
||||
|
||||
from .integration_test import TestIncrementalFileStreamS3
|
||||
|
||||
LOGGER = AirbyteLogger()
|
||||
|
||||
|
||||
def pytest_sessionfinish(session: Any, exitstatus: Any) -> None:
|
||||
"""tries to find and remove all temp buckets"""
|
||||
instance = TestIncrementalFileStreamS3()
|
||||
instance._s3_connect(instance.credentials)
|
||||
temp_buckets = []
|
||||
for bucket in instance.s3_resource.buckets.all():
|
||||
if bucket.name.startswith(instance.temp_bucket_prefix):
|
||||
temp_buckets.append(bucket.name)
|
||||
for bucket_name in temp_buckets:
|
||||
bucket = instance.s3_resource.Bucket(bucket_name)
|
||||
bucket.objects.all().delete()
|
||||
bucket.delete()
|
||||
LOGGER.info(f"S3 Bucket {bucket_name} is now deleted")
|
||||
@@ -1,10 +1,10 @@
|
||||
{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,55 +1,55 @@
|
||||
{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02.000000Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02.000000Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000}
|
||||
@@ -1,2 +1,2 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00.000000Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00.000000Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000}
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1\u20ac", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1\u20ac", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmj|b1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmj|b1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"id": 1.0, "name": "PVdhmjb1", "valid": false, "valid_string": "False", "array": "[\"a\", \"b\", \"c\"]", "dict": "{\"key\": \"value\"}","_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2.0, "name": "j4DyXTS7", "valid": true, "valid_string": "True", "array": "[\"a\", \"b\"]","dict": "{\"key\": \"value_with_comma\\,\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3.0, "name": "v0w8fTME", "valid": false, "valid_string": "False", "array": "[\"a\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4.0, "name": "1q6jD8Np", "valid": false, "valid_string": "False", "array": "[]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5.0, "name": "77h4aiMP", "valid": true, "valid_string": "True", "array": "[\"b\", \"c\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6.0, "name": "Le35Wyic", "valid": true, "valid_string": "True", "array": "[\"a\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7.0, "name": "xZhh1Kyl", "valid": false, "valid_string": "False", "array": "[\"b\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8.0, "name": "M2t286iJ", "valid": false, "valid_string": "False", "array": "[\"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 1.0, "name": "PVdhmjb1", "valid": false, "valid_string": "False", "array": "[\"a\", \"b\", \"c\"]", "dict": "{\"key\": \"value\"}","_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2.0, "name": "j4DyXTS7", "valid": true, "valid_string": "True", "array": "[\"a\", \"b\"]","dict": "{\"key\": \"value_with_comma\\,\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3.0, "name": "v0w8fTME", "valid": false, "valid_string": "False", "array": "[\"a\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4.0, "name": "1q6jD8Np", "valid": false, "valid_string": "False", "array": "[]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5.0, "name": "77h4aiMP", "valid": true, "valid_string": "True", "array": "[\"b\", \"c\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6.0, "name": "Le35Wyic", "valid": true, "valid_string": "True", "array": "[\"a\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7.0, "name": "xZhh1Kyl", "valid": false, "valid_string": "False", "array": "[\"b\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8.0, "name": "M2t286iJ", "valid": false, "valid_string": "False", "array": "[\"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": "null", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "NULL", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": "null", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": "NULL", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{"stream": "test", "data": {"id": 1, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 1, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 162727468000}
|
||||
{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 3, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000}
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
{"stream":"test","data":{"id":"row1","value": 12.345,"_ab_source_file_last_modified":"2023-08-06T18:05:00Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637642}
|
||||
{"stream":"test","data":{"id":"row2","value": 67.89,"_ab_source_file_last_modified":"2023-08-06T18:05:00Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637643}
|
||||
{"stream":"test","data":{"id":"row1","value": 12.345,"_ab_source_file_last_modified":"2023-08-06T18:05:00.000000Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637642}
|
||||
{"stream":"test","data":{"id":"row2","value": 67.89,"_ab_source_file_last_modified":"2023-08-06T18:05:00.000000Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637643}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312}
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Iterator, List, Mapping
|
||||
|
||||
import boto3
|
||||
from airbyte_cdk import AirbyteLogger
|
||||
from botocore.errorfactory import ClientError
|
||||
from source_s3.stream import IncrementalFileStreamS3
|
||||
|
||||
from .integration_test_abstract import HERE, SAMPLE_DIR, AbstractTestIncrementalFileStream
|
||||
|
||||
LOGGER = AirbyteLogger()
|
||||
|
||||
|
||||
class TestIncrementalFileStreamS3(AbstractTestIncrementalFileStream):
|
||||
region = "eu-west-3"
|
||||
|
||||
@property
|
||||
def stream_class(self) -> type:
|
||||
return IncrementalFileStreamS3
|
||||
|
||||
@property
|
||||
def credentials(self) -> Mapping:
|
||||
filename = HERE.parent / "secrets/config.json"
|
||||
with open(filename) as json_file:
|
||||
config = json.load(json_file)
|
||||
return {
|
||||
"aws_access_key_id": config["provider"]["aws_access_key_id"],
|
||||
"aws_secret_access_key": config["provider"]["aws_secret_access_key"],
|
||||
}
|
||||
|
||||
def provider(self, bucket_name: str) -> Mapping:
|
||||
return {"storage": "S3", "bucket": bucket_name}
|
||||
|
||||
def _s3_connect(self, credentials: Mapping) -> None:
|
||||
self.s3_client = boto3.client(
|
||||
"s3",
|
||||
aws_access_key_id=credentials["aws_access_key_id"],
|
||||
aws_secret_access_key=credentials["aws_secret_access_key"],
|
||||
region_name=self.region,
|
||||
)
|
||||
self.s3_resource = boto3.resource(
|
||||
"s3", aws_access_key_id=credentials["aws_access_key_id"], aws_secret_access_key=credentials["aws_secret_access_key"]
|
||||
)
|
||||
|
||||
def cloud_files(self, cloud_bucket_name: str, credentials: Mapping, files_to_upload: List, private: bool = True) -> Iterator[str]:
|
||||
self._s3_connect(credentials)
|
||||
|
||||
location = {"LocationConstraint": self.region}
|
||||
bucket_name = cloud_bucket_name
|
||||
|
||||
print("\n")
|
||||
LOGGER.info(f"Uploading {len(files_to_upload)} file(s) to {'private' if private else 'public'} aws bucket '{bucket_name}'")
|
||||
try:
|
||||
self.s3_client.head_bucket(Bucket=bucket_name)
|
||||
except ClientError:
|
||||
if private:
|
||||
self.s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location)
|
||||
else:
|
||||
self.s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location, ObjectOwnership="ObjectWriter")
|
||||
self.s3_client.delete_public_access_block(Bucket=bucket_name)
|
||||
self.s3_client.put_bucket_acl(Bucket=bucket_name, ACL="public-read")
|
||||
|
||||
# wait here until the bucket is ready
|
||||
ready = False
|
||||
attempts, max_attempts = 0, 30
|
||||
while not ready:
|
||||
time.sleep(1)
|
||||
try:
|
||||
self.s3_client.head_bucket(Bucket=bucket_name)
|
||||
except ClientError:
|
||||
attempts += 1
|
||||
if attempts >= max_attempts:
|
||||
raise RuntimeError(f"Couldn't get a successful ping on bucket after ~{max_attempts} seconds")
|
||||
else:
|
||||
ready = True
|
||||
LOGGER.info(f"bucket {bucket_name} initialised")
|
||||
|
||||
extra_args = {}
|
||||
if not private:
|
||||
extra_args = {"ACL": "public-read"}
|
||||
for filepath in files_to_upload:
|
||||
upload_path = str(filepath).replace(str(SAMPLE_DIR), "")
|
||||
upload_path = upload_path[1:] if upload_path[0] == "/" else upload_path
|
||||
self.s3_client.upload_file(str(filepath), bucket_name, upload_path, ExtraArgs=extra_args)
|
||||
yield f"{bucket_name}/{upload_path}"
|
||||
|
||||
def teardown_infra(self, cloud_bucket_name: str, credentials: Mapping) -> None:
|
||||
self._s3_connect(credentials)
|
||||
bucket = self.s3_resource.Bucket(cloud_bucket_name)
|
||||
bucket.objects.all().delete()
|
||||
bucket.delete()
|
||||
LOGGER.info(f"S3 Bucket {cloud_bucket_name} is now deleted")
|
||||
@@ -1,526 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator, List, Mapping
|
||||
from uuid import uuid4
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
from airbyte_cdk.logger import AirbyteLogger
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from source_s3.source_files_abstract.formats.csv_parser import CsvParser
|
||||
from source_s3.source_files_abstract.stream import FileStream
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
SAMPLE_DIR = HERE.joinpath("sample_files/")
|
||||
LOGGER = AirbyteLogger()
|
||||
JSONTYPE_TO_PYTHONTYPE = {"string": str, "number": float, "integer": int, "object": dict, "array": list, "boolean": bool, "null": None}
|
||||
|
||||
|
||||
class AbstractTestIncrementalFileStream(ABC):
|
||||
"""Prefix this class with Abstract so the tests don't run here but only in the children"""
|
||||
|
||||
temp_bucket_prefix = "airbytetest-"
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cloud_bucket_prefix(self) -> str:
|
||||
return self.temp_bucket_prefix
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def format(self) -> Mapping[str, Any]:
|
||||
return {"filetype": "csv"}
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def airbyte_system_columns(self) -> Mapping[str, str]:
|
||||
return {
|
||||
FileStream.ab_last_mod_col: {"type": "string", "format": "date-time"},
|
||||
FileStream.ab_file_name_col: {"type": "string"}
|
||||
}
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def stream_class(self) -> type:
|
||||
"""
|
||||
:return: provider specific FileStream class (e.g. IncrementalFileStreamS3)
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def credentials(self) -> Mapping:
|
||||
"""
|
||||
These will be added automatically to the provider property
|
||||
|
||||
:return: mapping of provider specific credentials
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def provider(self, bucket_name: str) -> Mapping:
|
||||
"""
|
||||
:return: provider specific provider dict as described in spec.json (leave out credentials, they will be added automatically)
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def cloud_files(self, cloud_bucket_name: str, credentials: Mapping, files_to_upload: List, private: bool = True) -> Iterator[str]:
|
||||
"""
|
||||
See S3 for example what the override of this needs to achieve.
|
||||
|
||||
:param cloud_bucket_name: name of bucket (or equivalent)
|
||||
:param credentials: mapping of provider specific credentials
|
||||
:param files_to_upload: list of paths to local files to upload, pass empty list to test zero files case
|
||||
:param private: whether or not to make the files private and require credentials to read, defaults to True
|
||||
:yield: url filepath to uploaded file
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def teardown_infra(self, cloud_bucket_name: str, credentials: Mapping) -> None:
|
||||
"""
|
||||
Provider-specific logic to tidy up any cloud resources.
|
||||
See S3 for example.
|
||||
|
||||
:param cloud_bucket_name: bucket (or equivalent) name
|
||||
:param credentials: mapping of provider specific credentials
|
||||
"""
|
||||
|
||||
def _stream_records_test_logic(
|
||||
self,
|
||||
cloud_bucket_name: str,
|
||||
format: Mapping[str, str],
|
||||
airbyte_system_columns: Mapping[str, str],
|
||||
sync_mode: Any,
|
||||
files: List[str],
|
||||
path_pattern: str,
|
||||
private: bool,
|
||||
num_columns: Any,
|
||||
num_records: Any,
|
||||
expected_schema: Mapping[str, Any],
|
||||
user_schema: Mapping[str, Any],
|
||||
fails: Any,
|
||||
state: Any = None,
|
||||
) -> Any:
|
||||
uploaded_files = [fpath for fpath in self.cloud_files(cloud_bucket_name, self.credentials, files, private)]
|
||||
LOGGER.info(f"file(s) uploaded: {uploaded_files}")
|
||||
|
||||
# emulate state for incremental testing
|
||||
# since we're not actually saving state out to file here, we pass schema in to our FileStream creation...
|
||||
# this isn't how it will work in Airbyte but it's a close enough emulation
|
||||
current_state = state if state is not None else {FileStream.ab_last_mod_col: "1970-01-01T00:00:00Z"}
|
||||
if (user_schema is None) and ("schema" in current_state.keys()):
|
||||
user_schema = current_state["schema"]
|
||||
|
||||
full_expected_schema = {
|
||||
"type": "object",
|
||||
"properties": {**expected_schema, **airbyte_system_columns},
|
||||
}
|
||||
|
||||
str_user_schema = str(user_schema).replace("'", '"') if user_schema is not None else None
|
||||
total_num_columns = num_columns + len(airbyte_system_columns.keys())
|
||||
provider = {**self.provider(cloud_bucket_name), **self.credentials} if private else self.provider(cloud_bucket_name)
|
||||
|
||||
if not fails:
|
||||
fs = self.stream_class("dataset", provider, format, path_pattern, str_user_schema)
|
||||
LOGGER.info(f"Testing stream_records() in SyncMode:{sync_mode.value}")
|
||||
|
||||
# check we return correct schema from get_json_schema()
|
||||
assert fs.get_json_schema() == full_expected_schema
|
||||
|
||||
records = []
|
||||
for stream_slice in fs.stream_slices(sync_mode=sync_mode, stream_state=current_state):
|
||||
if stream_slice is not None:
|
||||
# we need to do this in order to work out which extra columns (if any) we expect in this stream_slice
|
||||
expected_columns = []
|
||||
for file_dict in stream_slice["files"]:
|
||||
# TODO: if we ever test other filetypes in these tests this will need fixing
|
||||
file_reader = CsvParser(format)
|
||||
storage_file = file_dict["storage_file"]
|
||||
with storage_file.open(file_reader.is_binary) as f:
|
||||
expected_columns.extend(list(file_reader.get_inferred_schema(f, storage_file.file_info).keys()))
|
||||
expected_columns = set(expected_columns) # de-dupe
|
||||
|
||||
for record in fs.read_records(sync_mode, stream_slice=stream_slice):
|
||||
# check actual record values match expected schema
|
||||
jsonschema.validate(record, full_expected_schema)
|
||||
records.append(record)
|
||||
|
||||
assert all([len(r.keys()) == total_num_columns for r in records])
|
||||
assert len(records) == num_records
|
||||
|
||||
# returning state by simulating call to get_updated_state() with final record so we can test incremental
|
||||
return fs.get_updated_state(current_stream_state=current_state, latest_record=records[-1])
|
||||
|
||||
else:
|
||||
with pytest.raises(Exception) as e_info:
|
||||
fs = self.stream_class("dataset", provider, format, path_pattern, str_user_schema)
|
||||
LOGGER.info(f"Testing EXPECTED FAILURE stream_records() in SyncMode:{sync_mode.value}")
|
||||
|
||||
fs.get_json_schema()
|
||||
|
||||
records = []
|
||||
for stream_slice in fs.stream_slices(sync_mode=sync_mode, stream_state=current_state):
|
||||
for record in fs.read_records(sync_mode, stream_slice=stream_slice):
|
||||
records.append(record)
|
||||
|
||||
LOGGER.info(f"Failed as expected, error: {e_info}")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
# make user_schema None to test auto-inference. Exclude any _airbyte system columns in expected_schema.
|
||||
"files, path_pattern, private, num_columns, num_records, expected_schema, user_schema, incremental, fails",
|
||||
[
|
||||
# single file tests
|
||||
( # public
|
||||
[SAMPLE_DIR.joinpath("simple_test.csv")],
|
||||
"**",
|
||||
False,
|
||||
3,
|
||||
8,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # private
|
||||
[SAMPLE_DIR.joinpath("simple_test.csv")],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
8,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # provided schema exact match to actual schema
|
||||
[SAMPLE_DIR.joinpath("simple_test.csv")],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
8,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # provided schema not matching datatypes, expect successful coercion
|
||||
[SAMPLE_DIR.joinpath("simple_test.csv")],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
8,
|
||||
{"id": "string", "name": "string", "valid": "string"},
|
||||
{"id": "string", "name": "string", "valid": "string"},
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # provided incompatible schema, expect fail
|
||||
[SAMPLE_DIR.joinpath("simple_test.csv")],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
8,
|
||||
{"id": "boolean", "name": "boolean", "valid": "boolean"},
|
||||
{"id": "boolean", "name": "boolean", "valid": "boolean"},
|
||||
False,
|
||||
True,
|
||||
),
|
||||
# multiple file tests (all have identical schemas)
|
||||
( # public, auto-infer
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_3.csv"),
|
||||
],
|
||||
"**",
|
||||
False,
|
||||
3,
|
||||
17,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # private, auto-infer
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_3.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
17,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # provided schema exact match to actual schema
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_3.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
17,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # provided schema not matching datatypes, expect successful coercion
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_3.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
17,
|
||||
{"id": "string", "name": "string", "valid": "string"},
|
||||
{"id": "string", "name": "string", "valid": "string"},
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # provided incompatible schema, expect fail
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_3.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
17,
|
||||
{"id": "boolean", "name": "boolean", "valid": "boolean"},
|
||||
{"id": "boolean", "name": "boolean", "valid": "boolean"},
|
||||
False,
|
||||
True,
|
||||
),
|
||||
( # provided schema, not containing all columns (extra columns should go into FileStream.ab_additional_col)
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"),
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
3,
|
||||
17,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
False,
|
||||
False,
|
||||
),
|
||||
# pattern matching tests with additional files present that we don't want to read
|
||||
( # at top-level of bucket
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("file_to_skip.csv"),
|
||||
SAMPLE_DIR.joinpath("file_to_skip.txt"),
|
||||
],
|
||||
"simple*",
|
||||
True,
|
||||
3,
|
||||
11,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
),
|
||||
( # at multiple levels of bucket
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("file_to_skip.csv"),
|
||||
SAMPLE_DIR.joinpath("file_to_skip.txt"),
|
||||
SAMPLE_DIR.joinpath("pattern_match_test/this_folder/simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("pattern_match_test/not_this_folder/file_to_skip.csv"),
|
||||
SAMPLE_DIR.joinpath("pattern_match_test/not_this_folder/file_to_skip.txt"),
|
||||
],
|
||||
"**/simple*",
|
||||
True,
|
||||
3,
|
||||
19,
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
),
|
||||
# incremental tests (passing num_records/num_columns/fails as lists holding value for each file in order)
|
||||
( # auto-infer, all same schema
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_3.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
[3, 3, 3],
|
||||
[8, 3, 6],
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
True,
|
||||
[False, False, False],
|
||||
),
|
||||
( # provided schema, all same schema
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_2.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test_3.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
[3, 3, 3],
|
||||
[8, 3, 6],
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
True,
|
||||
[False, False, False],
|
||||
),
|
||||
( # auto-infer, (different but merge-able schemas)
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"),
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
[3, 3, 3],
|
||||
[8, 3, 6],
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
True,
|
||||
[False, False, False],
|
||||
),
|
||||
( # same as previous but change order and expect 5 columns instead of 3 in all
|
||||
[
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"),
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
[5, 5, 5],
|
||||
[6, 3, 8],
|
||||
{"id": "integer", "name": "string", "valid": "boolean", "percentage": "number", "nullable": "string"},
|
||||
None,
|
||||
True,
|
||||
[False, False, False],
|
||||
),
|
||||
( # like previous test but with a user_schema limiting columns
|
||||
[
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"),
|
||||
SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"),
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
[2, 2, 2],
|
||||
[6, 3, 8],
|
||||
{"id": "integer", "name": "string"},
|
||||
{"id": "integer", "name": "string"},
|
||||
True,
|
||||
[False, False, False],
|
||||
),
|
||||
( # fail when 2nd file has incompatible schema, auto-infer
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("incompatible_schema.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
[3, 3],
|
||||
[8, 8],
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
None,
|
||||
True,
|
||||
[False, True],
|
||||
),
|
||||
( # fail when 2nd file has incompatible schema, provided schema
|
||||
[
|
||||
SAMPLE_DIR.joinpath("simple_test.csv"),
|
||||
SAMPLE_DIR.joinpath("incompatible_schema.csv"),
|
||||
],
|
||||
"**",
|
||||
True,
|
||||
[3, 3],
|
||||
[8, 8],
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
{"id": "integer", "name": "string", "valid": "boolean"},
|
||||
True,
|
||||
[False, True],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_stream_records(
|
||||
self,
|
||||
cloud_bucket_prefix: str,
|
||||
format: Mapping[str, Any],
|
||||
airbyte_system_columns: Mapping[str, str],
|
||||
files: List[str],
|
||||
path_pattern: str,
|
||||
private: bool,
|
||||
num_columns: List[int],
|
||||
num_records: List[int],
|
||||
expected_schema: Mapping[str, Any],
|
||||
user_schema: Mapping[str, Any],
|
||||
incremental: bool,
|
||||
fails: List[bool],
|
||||
) -> None:
|
||||
expected_schema = {k: {"type": ["null", v]} for k, v in expected_schema.items()}
|
||||
try:
|
||||
if not incremental: # we expect matching behaviour here in either sync_mode
|
||||
for sync_mode in [
|
||||
SyncMode("full_refresh"),
|
||||
SyncMode("incremental"),
|
||||
]:
|
||||
cloud_bucket_name = f"{cloud_bucket_prefix}{uuid4()}"
|
||||
self._stream_records_test_logic(
|
||||
cloud_bucket_name,
|
||||
format,
|
||||
airbyte_system_columns,
|
||||
sync_mode,
|
||||
files,
|
||||
path_pattern,
|
||||
private,
|
||||
num_columns,
|
||||
num_records,
|
||||
expected_schema,
|
||||
user_schema,
|
||||
fails,
|
||||
)
|
||||
self.teardown_infra(cloud_bucket_name, self.credentials)
|
||||
else:
|
||||
cloud_bucket_name = f"{cloud_bucket_prefix}{uuid4()}"
|
||||
latest_state = None
|
||||
for i in range(len(files)):
|
||||
latest_state = self._stream_records_test_logic(
|
||||
cloud_bucket_name,
|
||||
format,
|
||||
airbyte_system_columns,
|
||||
SyncMode("incremental"),
|
||||
[files[i]],
|
||||
path_pattern,
|
||||
private,
|
||||
num_columns[i],
|
||||
num_records[i],
|
||||
expected_schema,
|
||||
user_schema,
|
||||
fails[i],
|
||||
state=latest_state,
|
||||
)
|
||||
LOGGER.info(f"incremental state: {latest_state}")
|
||||
# small delay to ensure next file gets later last_modified timestamp
|
||||
time.sleep(1)
|
||||
self.teardown_infra(cloud_bucket_name, self.credentials)
|
||||
|
||||
except Exception as e:
|
||||
self.teardown_infra(cloud_bucket_name, self.credentials)
|
||||
raise e
|
||||
@@ -1,32 +1,342 @@
|
||||
{
|
||||
"documentationUrl": "https://docs.airbyte.com/integrations/sources/s3",
|
||||
"changelogUrl": "https://docs.airbyte.com/integrations/sources/s3",
|
||||
"connectionSpecification": {
|
||||
"title": "S3 Source Spec",
|
||||
"title": "Config",
|
||||
"description": "NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes\nbecause it is responsible for converting legacy S3 v3 configs into v4 configs using the File-Based CDK.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset": {
|
||||
"title": "Output Stream Name",
|
||||
"description": "The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.",
|
||||
"pattern": "^([A-Za-z0-9-_]+)$",
|
||||
"start_date": {
|
||||
"title": "Start Date",
|
||||
"description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.",
|
||||
"examples": ["2021-01-01T00:00:00.000000Z"],
|
||||
"format": "date-time",
|
||||
"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$",
|
||||
"pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ",
|
||||
"order": 1,
|
||||
"type": "string"
|
||||
},
|
||||
"streams": {
|
||||
"title": "The list of streams to sync",
|
||||
"description": "Each instance of this configuration defines a <a href=\"https://docs.airbyte.com/cloud/core-concepts#stream\">stream</a>. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.",
|
||||
"order": 10,
|
||||
"type": "array",
|
||||
"items": {
|
||||
"title": "FileBasedStreamConfig",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"title": "Name",
|
||||
"description": "The name of the stream.",
|
||||
"type": "string"
|
||||
},
|
||||
"file_type": {
|
||||
"title": "File Type",
|
||||
"description": "The data file type that is being extracted for a stream.",
|
||||
"type": "string"
|
||||
},
|
||||
"globs": {
|
||||
"title": "Globs",
|
||||
"description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look <a href=\"https://en.wikipedia.org/wiki/Glob_(programming)\">here</a>.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"legacy_prefix": {
|
||||
"title": "Legacy Prefix",
|
||||
"description": "The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.",
|
||||
"airbyte_hidden": true,
|
||||
"type": "string"
|
||||
},
|
||||
"validation_policy": {
|
||||
"title": "Validation Policy",
|
||||
"description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.",
|
||||
"default": "Emit Record",
|
||||
"enum": ["Emit Record", "Skip Record", "Wait for Discover"]
|
||||
},
|
||||
"input_schema": {
|
||||
"title": "Input Schema",
|
||||
"description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.",
|
||||
"type": "string"
|
||||
},
|
||||
"primary_key": {
|
||||
"title": "Primary Key",
|
||||
"description": "The column or columns (for a composite key) that serves as the unique identifier of a record.",
|
||||
"type": "string"
|
||||
},
|
||||
"days_to_sync_if_history_is_full": {
|
||||
"title": "Days To Sync If History Is Full",
|
||||
"description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.",
|
||||
"default": 3,
|
||||
"type": "integer"
|
||||
},
|
||||
"format": {
|
||||
"title": "Format",
|
||||
"description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.",
|
||||
"type": "object",
|
||||
"oneOf": [
|
||||
{
|
||||
"title": "Avro Format",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filetype": {
|
||||
"title": "Filetype",
|
||||
"default": "avro",
|
||||
"const": "avro",
|
||||
"type": "string"
|
||||
},
|
||||
"double_as_string": {
|
||||
"title": "Convert Double Fields to Strings",
|
||||
"description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "CSV Format",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filetype": {
|
||||
"title": "Filetype",
|
||||
"default": "csv",
|
||||
"const": "csv",
|
||||
"type": "string"
|
||||
},
|
||||
"delimiter": {
|
||||
"title": "Delimiter",
|
||||
"description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.",
|
||||
"default": ",",
|
||||
"type": "string"
|
||||
},
|
||||
"quote_char": {
|
||||
"title": "Quote Character",
|
||||
"description": "The character used for quoting CSV values. To disallow quoting, make this field blank.",
|
||||
"default": "\"",
|
||||
"type": "string"
|
||||
},
|
||||
"escape_char": {
|
||||
"title": "Escape Character",
|
||||
"description": "The character used for escaping special characters. To disallow escaping, leave this field blank.",
|
||||
"type": "string"
|
||||
},
|
||||
"encoding": {
|
||||
"title": "Encoding",
|
||||
"description": "The character encoding of the CSV data. Leave blank to default to <strong>UTF8</strong>. See <a href=\"https://docs.python.org/3/library/codecs.html#standard-encodings\" target=\"_blank\">list of python encodings</a> for allowable options.",
|
||||
"default": "utf8",
|
||||
"type": "string"
|
||||
},
|
||||
"double_quote": {
|
||||
"title": "Double Quote",
|
||||
"description": "Whether two quotes in a quoted CSV value denote a single quote in the data.",
|
||||
"default": true,
|
||||
"type": "boolean"
|
||||
},
|
||||
"null_values": {
|
||||
"title": "Null Values",
|
||||
"description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.",
|
||||
"default": [],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"uniqueItems": true
|
||||
},
|
||||
"strings_can_be_null": {
|
||||
"title": "Strings Can Be Null",
|
||||
"description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.",
|
||||
"default": true,
|
||||
"type": "boolean"
|
||||
},
|
||||
"skip_rows_before_header": {
|
||||
"title": "Skip Rows Before Header",
|
||||
"description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.",
|
||||
"default": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
"skip_rows_after_header": {
|
||||
"title": "Skip Rows After Header",
|
||||
"description": "The number of rows to skip after the header row.",
|
||||
"default": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
"header_definition": {
|
||||
"title": "CSV Header Definition",
|
||||
"description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.",
|
||||
"default": {
|
||||
"header_definition_type": "From CSV"
|
||||
},
|
||||
"oneOf": [
|
||||
{
|
||||
"title": "From CSV",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"header_definition_type": {
|
||||
"title": "Header Definition Type",
|
||||
"default": "From CSV",
|
||||
"const": "From CSV",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Autogenerated",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"header_definition_type": {
|
||||
"title": "Header Definition Type",
|
||||
"default": "Autogenerated",
|
||||
"const": "Autogenerated",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "User Provided",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"header_definition_type": {
|
||||
"title": "Header Definition Type",
|
||||
"default": "User Provided",
|
||||
"const": "User Provided",
|
||||
"type": "string"
|
||||
},
|
||||
"column_names": {
|
||||
"title": "Column Names",
|
||||
"description": "The column names that will be used while emitting the CSV records",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["column_names"]
|
||||
}
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"true_values": {
|
||||
"title": "True Values",
|
||||
"description": "A set of case-sensitive strings that should be interpreted as true values.",
|
||||
"default": ["y", "yes", "t", "true", "on", "1"],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"uniqueItems": true
|
||||
},
|
||||
"false_values": {
|
||||
"title": "False Values",
|
||||
"description": "A set of case-sensitive strings that should be interpreted as false values.",
|
||||
"default": ["n", "no", "f", "false", "off", "0"],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"uniqueItems": true
|
||||
},
|
||||
"inference_type": {
|
||||
"title": "Inference Type",
|
||||
"description": "How to infer the types of the columns. If none, inference default to strings.",
|
||||
"default": "None",
|
||||
"airbyte_hidden": true,
|
||||
"enum": ["None", "Primitive Types Only"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Jsonl Format",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filetype": {
|
||||
"title": "Filetype",
|
||||
"default": "jsonl",
|
||||
"const": "jsonl",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Parquet Format",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filetype": {
|
||||
"title": "Filetype",
|
||||
"default": "parquet",
|
||||
"const": "parquet",
|
||||
"type": "string"
|
||||
},
|
||||
"decimal_as_float": {
|
||||
"title": "Convert Decimal Fields to Floats",
|
||||
"description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"schemaless": {
|
||||
"title": "Schemaless",
|
||||
"description": "When enabled, syncs will not validate or structure records against the stream's schema.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": ["name", "file_type"]
|
||||
}
|
||||
},
|
||||
"bucket": {
|
||||
"title": "Bucket",
|
||||
"description": "Name of the S3 bucket where the file(s) exist.",
|
||||
"order": 0,
|
||||
"type": "string"
|
||||
},
|
||||
"aws_access_key_id": {
|
||||
"title": "AWS Access Key ID",
|
||||
"description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.",
|
||||
"airbyte_secret": true,
|
||||
"order": 2,
|
||||
"type": "string"
|
||||
},
|
||||
"aws_secret_access_key": {
|
||||
"title": "AWS Secret Access Key",
|
||||
"description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.",
|
||||
"airbyte_secret": true,
|
||||
"order": 3,
|
||||
"type": "string"
|
||||
},
|
||||
"endpoint": {
|
||||
"title": "Endpoint",
|
||||
"description": "Endpoint to an S3 compatible service. Leave empty to use AWS.",
|
||||
"default": "",
|
||||
"order": 4,
|
||||
"type": "string"
|
||||
},
|
||||
"dataset": {
|
||||
"title": "Output Stream Name",
|
||||
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.name instead. The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.",
|
||||
"pattern": "^([A-Za-z0-9-_]+)$",
|
||||
"order": 100,
|
||||
"type": "string",
|
||||
"airbyte_hidden": true
|
||||
},
|
||||
"path_pattern": {
|
||||
"title": "Pattern of files to replicate",
|
||||
"description": "A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See <a href=\"https://facelessuser.github.io/wcmatch/glob/\" target=\"_blank\">this page</a> to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern <strong>**</strong> to pick up all files.",
|
||||
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.globs instead. A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See <a href=\"https://facelessuser.github.io/wcmatch/glob/\" target=\"_blank\">this page</a> to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern <strong>**</strong> to pick up all files.",
|
||||
"examples": [
|
||||
"**",
|
||||
"myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv"
|
||||
],
|
||||
"order": 10,
|
||||
"type": "string"
|
||||
"order": 110,
|
||||
"type": "string",
|
||||
"airbyte_hidden": true
|
||||
},
|
||||
"format": {
|
||||
"title": "File Format",
|
||||
"description": "The format of the files you'd like to replicate",
|
||||
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.format instead. The format of the files you'd like to replicate",
|
||||
"default": "csv",
|
||||
"order": 20,
|
||||
"order": 120,
|
||||
"type": "object",
|
||||
"oneOf": [
|
||||
{
|
||||
@@ -199,17 +509,19 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
],
|
||||
"airbyte_hidden": true
|
||||
},
|
||||
"schema": {
|
||||
"title": "Manually enforced data schema",
|
||||
"description": "Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of <strong>{ \"column\" : \"type\" }</strong>, where types are valid <a href=\"https://json-schema.org/understanding-json-schema/reference/type.html\" target=\"_blank\">JSON Schema datatypes</a>. Leave as {} to auto-infer the schema.",
|
||||
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.input_schema instead. Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of <strong>{ \"column\" : \"type\" }</strong>, where types are valid <a href=\"https://json-schema.org/understanding-json-schema/reference/type.html\" target=\"_blank\">JSON Schema datatypes</a>. Leave as {} to auto-infer the schema.",
|
||||
"default": "{}",
|
||||
"examples": [
|
||||
"{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}"
|
||||
],
|
||||
"order": 30,
|
||||
"type": "string"
|
||||
"order": 130,
|
||||
"type": "string",
|
||||
"airbyte_hidden": true
|
||||
},
|
||||
"provider": {
|
||||
"title": "S3: Amazon Web Services",
|
||||
@@ -259,13 +571,12 @@
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["bucket"],
|
||||
"order": 11,
|
||||
"description": "Use this to load files from S3 or S3-compatible services"
|
||||
"required": [],
|
||||
"order": 111,
|
||||
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use bucket, aws_access_key_id, aws_secret_access_key and endpoint instead. Use this to load files from S3 or S3-compatible services",
|
||||
"airbyte_hidden": true
|
||||
}
|
||||
},
|
||||
"required": ["dataset", "path_pattern", "provider"]
|
||||
},
|
||||
"supportsIncremental": true,
|
||||
"supported_destination_sync_modes": ["overwrite", "append", "append_dedup"]
|
||||
"required": ["streams", "bucket"]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,10 +4,39 @@
|
||||
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
|
||||
from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type
|
||||
from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader
|
||||
|
||||
|
||||
def get_source(args: List[str]):
|
||||
catalog_path = AirbyteEntrypoint.extract_catalog(args)
|
||||
try:
|
||||
return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor)
|
||||
except Exception:
|
||||
print(
|
||||
AirbyteMessage(
|
||||
type=Type.TRACE,
|
||||
trace=AirbyteTraceMessage(
|
||||
type=TraceType.ERROR,
|
||||
emitted_at=int(datetime.now().timestamp() * 1000),
|
||||
error=AirbyteErrorTraceMessage(
|
||||
message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.",
|
||||
stack_trace=traceback.format_exc(),
|
||||
),
|
||||
),
|
||||
).json()
|
||||
)
|
||||
return None
|
||||
|
||||
from airbyte_cdk.entrypoint import launch
|
||||
from source_s3 import SourceS3
|
||||
|
||||
if __name__ == "__main__":
|
||||
source = SourceS3()
|
||||
launch(source, sys.argv[1:])
|
||||
_args = sys.argv[1:]
|
||||
source = get_source(_args)
|
||||
|
||||
if source:
|
||||
launch(source, _args)
|
||||
|
||||
@@ -5,7 +5,7 @@ data:
|
||||
connectorSubtype: file
|
||||
connectorType: source
|
||||
definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
|
||||
dockerImageTag: 3.1.11
|
||||
dockerImageTag: 4.0.0
|
||||
dockerRepository: airbyte/source-s3
|
||||
githubIssueLabel: source-s3
|
||||
icon: s3.svg
|
||||
@@ -20,6 +20,11 @@ data:
|
||||
documentationUrl: https://docs.airbyte.com/integrations/sources/s3
|
||||
tags:
|
||||
- language:python
|
||||
releases:
|
||||
breakingChanges:
|
||||
4.0.0:
|
||||
message: "New release using file-based CDK"
|
||||
upgradeDeadline: "2023-08-25"
|
||||
ab_internal:
|
||||
sl: 300
|
||||
ql: 400
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
MAIN_REQUIREMENTS = [
|
||||
"airbyte-cdk>=0.50.2",
|
||||
"airbyte-cdk>=0.51.11",
|
||||
"pyarrow==12.0.1",
|
||||
"smart-open[s3]==5.1.0",
|
||||
"wcmatch==8.4",
|
||||
|
||||
@@ -26,7 +26,7 @@ class Config(AbstractFileBasedSpec):
|
||||
description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper "
|
||||
"permissions. If accessing publicly available data, this field is not necessary.",
|
||||
airbyte_secret=True,
|
||||
order=1,
|
||||
order=2,
|
||||
)
|
||||
|
||||
aws_secret_access_key: Optional[str] = Field(
|
||||
@@ -35,7 +35,7 @@ class Config(AbstractFileBasedSpec):
|
||||
description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper "
|
||||
"permissions. If accessing publicly available data, this field is not necessary.",
|
||||
airbyte_secret=True,
|
||||
order=2,
|
||||
order=3,
|
||||
)
|
||||
|
||||
endpoint: Optional[str] = Field(
|
||||
|
||||
@@ -117,8 +117,17 @@ class LegacyConfigTransformer:
|
||||
csv_options["skip_rows_before_header"] = skip_rows
|
||||
if skip_rows_after_names := advanced_options.pop("skip_rows_after_names", None):
|
||||
csv_options["skip_rows_after_header"] = skip_rows_after_names
|
||||
if autogenerate_column_names := advanced_options.pop("autogenerate_column_names", None):
|
||||
csv_options["autogenerate_column_names"] = autogenerate_column_names
|
||||
|
||||
if column_names := advanced_options.pop("column_names", None):
|
||||
csv_options["header_definition"] = {
|
||||
"header_definition_type": "User Provided",
|
||||
"column_names": column_names,
|
||||
}
|
||||
advanced_options.pop("autogenerate_column_names", None)
|
||||
elif advanced_options.pop("autogenerate_column_names", None):
|
||||
csv_options["header_definition"] = {"header_definition_type": "Autogenerated"}
|
||||
else:
|
||||
csv_options["header_definition"] = {"header_definition_type": "From CSV"}
|
||||
|
||||
cls._filter_legacy_noops(advanced_options)
|
||||
|
||||
@@ -151,7 +160,7 @@ class LegacyConfigTransformer:
|
||||
|
||||
@staticmethod
|
||||
def _filter_legacy_noops(advanced_options: Dict[str, Any]):
|
||||
ignore_all = ("auto_dict_encode", "timestamp_parsers")
|
||||
ignore_all = ("auto_dict_encode", "timestamp_parsers", "block_size")
|
||||
ignore_by_value = (("check_utf8", False),)
|
||||
|
||||
for option in ignore_all:
|
||||
|
||||
@@ -2,12 +2,22 @@
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
from typing import Any, Mapping
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
|
||||
from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message
|
||||
from airbyte_cdk.models import ConnectorSpecification
|
||||
from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource
|
||||
from source_s3.source import SourceS3Spec
|
||||
from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer
|
||||
|
||||
_V3_DEPRECATION_FIELD_MAPPING = {
|
||||
"dataset": "streams.name",
|
||||
"format": "streams.format",
|
||||
"path_pattern": "streams.globs",
|
||||
"provider": "bucket, aws_access_key_id, aws_secret_access_key and endpoint",
|
||||
"schema": "streams.input_schema",
|
||||
}
|
||||
|
||||
|
||||
class SourceS3(FileBasedSource):
|
||||
def read_config(self, config_path: str) -> Mapping[str, Any]:
|
||||
@@ -17,7 +27,77 @@ class SourceS3(FileBasedSource):
|
||||
validate the config against the new spec.
|
||||
"""
|
||||
config = super().read_config(config_path)
|
||||
if not config.get("streams"):
|
||||
if not self._is_v4_config(config):
|
||||
parsed_legacy_config = SourceS3Spec(**config)
|
||||
return LegacyConfigTransformer.convert(parsed_legacy_config)
|
||||
converted_config = LegacyConfigTransformer.convert(parsed_legacy_config)
|
||||
emit_configuration_as_airbyte_control_message(converted_config)
|
||||
return converted_config
|
||||
return config
|
||||
|
||||
def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
|
||||
s3_spec = SourceS3Spec.schema()
|
||||
s4_spec = self.spec_class.schema()
|
||||
|
||||
if s3_spec["properties"].keys() & s4_spec["properties"].keys():
|
||||
raise ValueError("Overlapping properties between V3 and V4")
|
||||
|
||||
for v3_property_key, v3_property_value in s3_spec["properties"].items():
|
||||
s4_spec["properties"][v3_property_key] = v3_property_value
|
||||
s4_spec["properties"][v3_property_key]["airbyte_hidden"] = True
|
||||
s4_spec["properties"][v3_property_key]["order"] += 100
|
||||
s4_spec["properties"][v3_property_key]["description"] = (
|
||||
SourceS3._create_description_with_deprecation_prefix(_V3_DEPRECATION_FIELD_MAPPING.get(v3_property_key, None))
|
||||
+ s4_spec["properties"][v3_property_key]["description"]
|
||||
)
|
||||
self._clean_required_fields(s4_spec["properties"][v3_property_key])
|
||||
|
||||
return ConnectorSpecification(
|
||||
documentationUrl=self.spec_class.documentation_url(),
|
||||
connectionSpecification=s4_spec,
|
||||
)
|
||||
|
||||
def _is_v4_config(self, config: Mapping[str, Any]) -> bool:
|
||||
return "streams" in config
|
||||
|
||||
@staticmethod
|
||||
def _clean_required_fields(v3_field: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Not having V3 fields root level as part of the `required` field is not enough as the platform will create empty objects for those.
|
||||
For example, filling all non-hidden fields from the form will create a config like:
|
||||
```
|
||||
{
|
||||
<...>
|
||||
"provider": {},
|
||||
<...>
|
||||
}
|
||||
```
|
||||
|
||||
As the field `provider` exists, the JSON validation will be applied and as `provider.bucket` is needed, the validation will fail
|
||||
with the following error:
|
||||
```
|
||||
"errors": {
|
||||
"connectionConfiguration": {
|
||||
"provider": {
|
||||
"bucket": {
|
||||
"message": "form.empty.error",
|
||||
"type": "required"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Hence, we need to make any V3 nested fields not required.
|
||||
"""
|
||||
if "properties" not in v3_field:
|
||||
return
|
||||
|
||||
v3_field["required"] = []
|
||||
for neste_field in v3_field["properties"]:
|
||||
SourceS3._clean_required_fields(neste_field)
|
||||
|
||||
@staticmethod
|
||||
def _create_description_with_deprecation_prefix(new_fields: Optional[str]) -> str:
|
||||
if new_fields:
|
||||
return f"Deprecated and will be removed soon. Please do not use this field anymore and use {new_fields} instead. "
|
||||
return "Deprecated and will be removed soon. Please do not use this field anymore. "
|
||||
|
||||
@@ -48,15 +48,13 @@ class SourceS3StreamReader(AbstractFileBasedStreamReader):
|
||||
# list or read files.
|
||||
raise ValueError("Source config is missing; cannot create the S3 client.")
|
||||
if self._s3_client is None:
|
||||
if self.config.endpoint:
|
||||
client_kv_args = _get_s3_compatible_client_args(self.config)
|
||||
self._s3_client = boto3.client("s3", **client_kv_args)
|
||||
else:
|
||||
self._s3_client = boto3.client(
|
||||
"s3",
|
||||
aws_access_key_id=self.config.aws_access_key_id,
|
||||
aws_secret_access_key=self.config.aws_secret_access_key,
|
||||
)
|
||||
client_kv_args = _get_s3_compatible_client_args(self.config) if self.config.endpoint else {}
|
||||
self._s3_client = boto3.client(
|
||||
"s3",
|
||||
aws_access_key_id=self.config.aws_access_key_id,
|
||||
aws_secret_access_key=self.config.aws_secret_access_key,
|
||||
**client_kv_args,
|
||||
)
|
||||
return self._s3_client
|
||||
|
||||
def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]:
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "test",
|
||||
"json_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": ["null", "integer"]
|
||||
},
|
||||
"name": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"valid": {
|
||||
"type": ["null", "boolean"]
|
||||
},
|
||||
"_ab_source_file_last_modified": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"_ab_source_file_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"],
|
||||
"source_defined_cursor": true,
|
||||
"default_cursor_field": ["_ab_source_file_last_modified"]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "append"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"dataset": "test",
|
||||
"provider": {
|
||||
"bucket": "a bucket",
|
||||
"aws_access_key_id": "a key id",
|
||||
"aws_secret_access_key": "an access key",
|
||||
"path_prefix": ""
|
||||
},
|
||||
"format": { "filetype": "csv" },
|
||||
"path_pattern": "**",
|
||||
"schema": "{}"
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"bucket": "a-bucket",
|
||||
"streams": [
|
||||
{
|
||||
"name": "output-stream-name",
|
||||
"format": { "filetype": "csv" },
|
||||
"file_type": "csv",
|
||||
"validation_policy": "Emit Record"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -127,8 +127,8 @@ def test_convert_legacy_config(legacy_config, expected_config):
|
||||
"double_quote": False,
|
||||
"newlines_in_values": True,
|
||||
"additional_reader_options": '{"strings_can_be_null": true, "null_values": ["NULL", "NONE"], "true_values": ["yes", "y"], "false_values": ["no", "n"]}',
|
||||
"advanced_options": '{"skip_rows": 3, "skip_rows_after_names": 5, "autogenerate_column_names": true}',
|
||||
"blocksize": 20000,
|
||||
"advanced_options": '{"skip_rows": 3, "skip_rows_after_names": 5, "autogenerate_column_names": true, "block_size": 20000}',
|
||||
"block_size": 20000,
|
||||
},
|
||||
{
|
||||
"filetype": "csv",
|
||||
@@ -144,7 +144,7 @@ def test_convert_legacy_config(legacy_config, expected_config):
|
||||
"strings_can_be_null": True,
|
||||
"skip_rows_before_header": 3,
|
||||
"skip_rows_after_header": 5,
|
||||
"autogenerate_column_names": True,
|
||||
"header_definition": {"header_definition_type": "Autogenerated"},
|
||||
},
|
||||
None,
|
||||
id="test_csv_all_legacy_options_set",
|
||||
@@ -169,6 +169,7 @@ def test_convert_legacy_config(legacy_config, expected_config):
|
||||
"false_values": ["0", "False", "FALSE", "false"],
|
||||
"inference_type": "Primitive Types Only",
|
||||
"strings_can_be_null": False,
|
||||
"header_definition": {"header_definition_type": "From CSV"},
|
||||
},
|
||||
None,
|
||||
id="test_csv_only_required_options",
|
||||
@@ -187,6 +188,7 @@ def test_convert_legacy_config(legacy_config, expected_config):
|
||||
"false_values": ["0", "False", "FALSE", "false"],
|
||||
"inference_type": "Primitive Types Only",
|
||||
"strings_can_be_null": False,
|
||||
"header_definition": {"header_definition_type": "From CSV"},
|
||||
},
|
||||
None,
|
||||
id="test_csv_empty_format",
|
||||
@@ -218,15 +220,6 @@ def test_convert_legacy_config(legacy_config, expected_config):
|
||||
ValueError,
|
||||
id="test_malformed_advanced_options",
|
||||
),
|
||||
pytest.param(
|
||||
"csv",
|
||||
{
|
||||
"advanced_options": '{"column_names": ""}',
|
||||
},
|
||||
None,
|
||||
ValueError,
|
||||
id="test_unsupported_advanced_options",
|
||||
),
|
||||
pytest.param(
|
||||
"csv",
|
||||
{
|
||||
@@ -261,6 +254,7 @@ def test_convert_legacy_config(legacy_config, expected_config):
|
||||
"false_values": ["0", "False", "FALSE", "false"],
|
||||
"inference_type": "Primitive Types Only",
|
||||
"strings_can_be_null": False,
|
||||
"header_definition": {"header_definition_type": "From CSV"},
|
||||
},
|
||||
None,
|
||||
id="test_unsupported_advanced_options_by_value_succeeds_if_value_matches_ignored_values",
|
||||
@@ -308,10 +302,42 @@ def test_convert_legacy_config(legacy_config, expected_config):
|
||||
"false_values": ["0", "False", "FALSE", "false"],
|
||||
"inference_type": "Primitive Types Only",
|
||||
"strings_can_be_null": False,
|
||||
"header_definition": {"header_definition_type": "From CSV"},
|
||||
},
|
||||
None,
|
||||
id="test_ignored_advanced_options",
|
||||
),
|
||||
pytest.param(
|
||||
"csv",
|
||||
{
|
||||
"filetype": "csv",
|
||||
"delimiter": "&",
|
||||
"infer_datatypes": False,
|
||||
"quote_char": "^",
|
||||
"escape_char": "$",
|
||||
"encoding": "ansi",
|
||||
"double_quote": False,
|
||||
"newlines_in_values": True,
|
||||
"additional_reader_options": '{"strings_can_be_null": true, "null_values": ["NULL", "NONE"], "true_values": ["yes", "y"], "false_values": ["no", "n"]}',
|
||||
"advanced_options": '{"autogenerate_column_names": true, "column_names": ["first", "second"]}',
|
||||
},
|
||||
{
|
||||
"filetype": "csv",
|
||||
"delimiter": "&",
|
||||
"quote_char": "^",
|
||||
"escape_char": "$",
|
||||
"encoding": "ansi",
|
||||
"double_quote": False,
|
||||
"null_values": ["NULL", "NONE"],
|
||||
"true_values": ["yes", "y"],
|
||||
"false_values": ["no", "n"],
|
||||
"inference_type": "None",
|
||||
"strings_can_be_null": True,
|
||||
"header_definition": {"header_definition_type": "User Provided", "column_names": ["first", "second"]},
|
||||
},
|
||||
None,
|
||||
id="test_csv_user_provided_column_names",
|
||||
),
|
||||
pytest.param(
|
||||
"jsonl",
|
||||
{
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from source_s3.v4 import Config, SourceS3, SourceS3StreamReader
|
||||
|
||||
_V3_FIELDS = ["dataset", "format", "path_pattern", "provider", "schema"]
|
||||
TEST_FILES_FOLDER = Path(__file__).resolve().parent.parent.joinpath("sample_files")
|
||||
|
||||
|
||||
class SourceTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._stream_reader = Mock(spec=SourceS3StreamReader)
|
||||
self._source = SourceS3(
|
||||
self._stream_reader,
|
||||
Config,
|
||||
str(TEST_FILES_FOLDER.joinpath("catalog.json"))
|
||||
)
|
||||
|
||||
@patch("source_s3.v4.source.emit_configuration_as_airbyte_control_message")
|
||||
def test_given_config_is_v3_when_read_config_then_emit_new_config(self, emit_config_mock) -> None:
|
||||
self._source.read_config(str(TEST_FILES_FOLDER.joinpath("v3_config.json")))
|
||||
assert emit_config_mock.call_count == 1
|
||||
|
||||
@patch("source_s3.v4.source.emit_configuration_as_airbyte_control_message")
|
||||
def test_given_config_is_v4_when_read_config_then_do_not_emit_new_config(self, emit_config_mock) -> None:
|
||||
self._source.read_config(str(TEST_FILES_FOLDER.joinpath("v4_config.json")))
|
||||
assert emit_config_mock.call_count == 0
|
||||
|
||||
def test_when_spec_then_v3_fields_not_required(self) -> None:
|
||||
spec = self._source.spec()
|
||||
assert all(field not in spec.connectionSpecification["required"] for field in _V3_FIELDS)
|
||||
|
||||
def test_when_spec_then_v3_fields_are_hidden(self) -> None:
|
||||
spec = self._source.spec()
|
||||
assert all(spec.connectionSpecification["properties"][field]["airbyte_hidden"] for field in _V3_FIELDS)
|
||||
|
||||
def test_when_spec_then_v3_fields_descriptions_are_prefixed_with_deprecation_warning(self) -> None:
|
||||
spec = self._source.spec()
|
||||
assert all(
|
||||
spec.connectionSpecification["properties"][field]["description"].startswith("Deprecated and will be removed soon")
|
||||
for field in _V3_FIELDS
|
||||
)
|
||||
|
||||
def test_when_spec_then_v3_nested_fields_are_not_required(self) -> None:
|
||||
spec = self._source.spec()
|
||||
assert not spec.connectionSpecification["properties"]["provider"]["required"]
|
||||
Reference in New Issue
Block a user