* initial commit * fix test error * Update get_gcs_blobs logic * add docs * Update source_definitions.yaml * Update airbyte-integrations/connectors/source-gcs/source_gcs/source.py Co-authored-by: sh4sh <6833405+sh4sh@users.noreply.github.com> * Update airbyte-config/init/src/main/resources/seed/source_definitions.yaml Co-authored-by: Denys Davydov <davydov.den18@gmail.com> * Update airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py Co-authored-by: Denys Davydov <davydov.den18@gmail.com> * Update airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py Co-authored-by: Denys Davydov <davydov.den18@gmail.com> * update docker file for pandas package * reimplement read_csv file * add logic to filter selected streams * close file_obj after reading * fix format and tests * add another stream * auto-bump connector version --------- Co-authored-by: Sunny <6833405+sh4sh@users.noreply.github.com> Co-authored-by: Denys Davydov <davydov.den18@gmail.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
35 lines
1007 B
Python
35 lines
1007 B
Python
#
|
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
#
|
|
|
|
import unittest
|
|
from io import BytesIO
|
|
|
|
import pandas as pd
|
|
from source_gcs.helpers import construct_file_schema
|
|
|
|
|
|
class TestGCSFunctions(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
# Initialize the mock config
|
|
self.config = {
|
|
'service_account': '{"test_key": "test_value"}',
|
|
'gcs_bucket': 'test_bucket',
|
|
'gcs_path': 'test_path'
|
|
}
|
|
|
|
def test_construct_file_schema(self):
|
|
# Test that the function correctly constructs a JSON schema for a DataFrame
|
|
df = pd.read_csv(BytesIO(b"id,name\n1,Alice\n2,Bob\n3,Charlie\n"))
|
|
schema = construct_file_schema(df)
|
|
expected_schema = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"properties": {
|
|
"id": {"type": "string"},
|
|
"name": {"type": "string"}
|
|
}
|
|
}
|
|
self.assertEqual(schema, expected_schema)
|