jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

🎉Updated normalization to handle new datatypes (#19721)

Browse Source 
* Updated normalization simple stream processing to handle new datatypes

* Updated normalization nested stream processing to handle new datatypes

* Updated normalization nested stream processing to handle new datatypes

* Updated normalization drop_scd_catalog processing to handle new datatypes

* Updated normalization ephemeral test processing to handle new datatypes

* fixed more tests for normalization

* fixed more tests for normalization

* fixed more tests for normalization

* fixed more tests for normalization

* fixed more issues

* fixed more issues (clickhouse)

* fixed more issues

* fixed more issues

* fixed more issues

* added binary type processing for some DBs

* cleared commented code and moved some hardcodes to processing as macro

* fixed codestyle and cleared commented code

* minor refactor

* minor refactor

* minor refactor

* fixed bool cast error

* fixed dict->str cast error

* fixed is_combining_node cast py check

* removed commented code

* removed commented code

* committed autogenerated normalization_test_output files

* committed autogenerated normalization_test_output files (new files)

* refactored utils.py

* Updated utils.py to use Callable functions and get rid of property_type in is_number and is_bool functions

* committed autogenerated normalization_test_output files (new files)

* fixed typo in TIMESTAMP_WITH_TIMEZONE_TYPE

* updated stream_processor to handle string type first as a wider type

* fixed arrays normalization by updating is_simple_property method as per new approaches

* format

Co-authored-by: Edward Gao <edward.gao@airbyte.io>
This commit is contained in:
Eugene
2023-01-06 23:26:40 +02:00
committed by GitHub
parent ab528885a5
commit c1d7736639
234 changed files with 4177 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql
View File

@@ -8,6 +8,14 @@
string
{% endmacro %}
{%- macro type_binary() -%}
{{ adapter.dispatch('type_binary')() }}
{%- endmacro -%}
{%- macro default__type_binary() -%}
binary
{%- endmacro -%}
{%- macro redshift__type_json() -%}
{%- if redshift_super_type() -%}
super
@@ -72,6 +80,28 @@
char(1000)
{%- endmacro -%}
{# binary data ------------------------------------------------- #}
{%- macro postgres__type_binary() -%}
bytea
{%- endmacro -%}
{%- macro bigquery__type_binary() -%}
bytes
{%- endmacro -%}
{%- macro mssql__type_binary() -%}
VARBINARY(MAX)
{%- endmacro -%}
{%- macro snowflake__type_binary() -%}
VARBINARY
{%- endmacro -%}
{%- macro clickhouse__type_binary() -%}
VARBINARY
{%- endmacro -%}
{# float ------------------------------------------------- #}
{% macro mysql__type_float() %}
float

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql
View File

@@ -34,12 +34,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

5
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -24,6 +24,7 @@ select
json_extract_scalar(_airbyte_data, "$['datetime_no_tz']") as datetime_no_tz,
json_extract_scalar(_airbyte_data, "$['time_tz']") as time_tz,
json_extract_scalar(_airbyte_data, "$['time_no_tz']") as time_no_tz,
json_extract_scalar(_airbyte_data, "$['property_binary_data']") as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP() as _airbyte_normalized_at
@@ -74,6 +75,7 @@ select
cast(nullif(time_no_tz, '') as
time
) as time_no_tz,
cast(FROM_BASE64(property_binary_data) as bytes) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP() as _airbyte_normalized_at
@@ -111,6 +113,8 @@ select
string
), ''), '-', coalesce(cast(time_no_tz as
string
), ''), '-', coalesce(cast(property_binary_data as
string
), '')) as
string
))) as _airbyte_exchange_rate_hashid,
@@ -134,6 +138,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP() as _airbyte_normalized_at,

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -34,12 +34,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

17
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -3,6 +3,22 @@
partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"},
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model
@@ -21,6 +37,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at,

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -34,12 +34,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -3,6 +3,22 @@
partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"},
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

5
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -24,6 +24,7 @@ select
json_extract_scalar(_airbyte_data, "$['datetime_no_tz']") as datetime_no_tz,
json_extract_scalar(_airbyte_data, "$['time_tz']") as time_tz,
json_extract_scalar(_airbyte_data, "$['time_no_tz']") as time_no_tz,
json_extract_scalar(_airbyte_data, "$['property_binary_data']") as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP() as _airbyte_normalized_at
@@ -74,6 +75,7 @@ select
cast(nullif(time_no_tz, '') as
time
) as time_no_tz,
cast(FROM_BASE64(property_binary_data) as bytes) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP() as _airbyte_normalized_at
@@ -111,6 +113,8 @@ select
string
), ''), '-', coalesce(cast(time_no_tz as
string
), ''), '-', coalesce(cast(property_binary_data as
string
), '')) as
string
))) as _airbyte_exchange_rate_hashid,
@@ -134,6 +138,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP() as _airbyte_normalized_at,

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql
View File

@@ -1,6 +1,6 @@
create view _airbyte_test_normalization.dedup_exchange_rate_ab1__dbt_tmp
create view _airbyte_test_normalization.dedup_exchange_rate_ab1
as (

10
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql
View File

@@ -1,26 +1,26 @@
create view _airbyte_test_normalization.dedup_exchange_rate_ab2__dbt_tmp
create view _airbyte_test_normalization.dedup_exchange_rate_ab2
as (
-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type
-- depends_on: _airbyte_test_normalization.dedup_exchange_rate_ab1
select
accurateCastOrNull(id, '
accurateCastOrNull(trim(BOTH '"' from id), '
BIGINT
') as id,
nullif(accurateCastOrNull(trim(BOTH '"' from currency), 'String'), 'null') as currency,
toDate(parseDateTimeBestEffortOrNull(trim(BOTH '"' from nullif(date, '')))) as date,
parseDateTime64BestEffortOrNull(trim(BOTH '"' from nullif(timestamp_col, ''))) as timestamp_col,
accurateCastOrNull("HKD@spéçiäl & characters", '
accurateCastOrNull(trim(BOTH '"' from "HKD@spéçiäl & characters"), '
Float64
') as "HKD@spéçiäl & characters",
nullif(accurateCastOrNull(trim(BOTH '"' from HKD_special___characters), 'String'), 'null') as HKD_special___characters,
accurateCastOrNull(NZD, '
accurateCastOrNull(trim(BOTH '"' from NZD), '
Float64
') as NZD,
accurateCastOrNull(USD, '
accurateCastOrNull(trim(BOTH '"' from USD), '
Float64
') as USD,
_airbyte_ab_id,

6
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql
View File

@@ -1,5 +1,7 @@
insert into test_normalization.dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid")
@@ -101,4 +103,4 @@ select
_airbyte_dedup_cdc_excluded_hashid
from dedup_data where _airbyte_row_num = 1

6
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -1,5 +1,7 @@
insert into test_normalization.dedup_exchange_rate_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid")
@@ -105,4 +107,4 @@ select
_airbyte_dedup_exchange_rate_hashid
from dedup_data where _airbyte_row_num = 1

6
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql
View File

@@ -1,5 +1,7 @@
insert into test_normalization.renamed_dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid")
@@ -87,4 +89,4 @@ select
_airbyte_renamed_dedup_cdc_excluded_hashid
from dedup_data where _airbyte_row_num = 1

6
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql
View File

@@ -1,5 +1,7 @@
insert into test_normalization.dedup_exchange_rate ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid")
@@ -26,4 +28,4 @@ where 1 = 1
and _airbyte_active_row = 1

6
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql
View File

@@ -1,5 +1,7 @@
insert into test_normalization.renamed_dedup_cdc_excluded ("_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid")
@@ -20,4 +22,4 @@ where 1 = 1
and _airbyte_active_row = 1

5
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -1,7 +1,9 @@
insert into test_normalization.exchange_rate__dbt_tmp ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid")
insert into test_normalization.exchange_rate ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "property_binary_data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid")
-- Final base SQL model
-- depends_on: _airbyte_test_normalization.exchange_rate_ab3
@@ -19,6 +21,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at,

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql
View File

@@ -1,6 +1,6 @@
create view _airbyte_test_normalization.dedup_exchange_rate_stg__dbt_tmp
create view _airbyte_test_normalization.dedup_exchange_rate_stg
as (

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql
View File

@@ -1,6 +1,6 @@
create view _airbyte_test_normalization.multiple_column_names_conflicts_stg__dbt_tmp
create view _airbyte_test_normalization.multiple_column_names_conflicts_stg
as (

8
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql
View File

@@ -6,14 +6,14 @@
-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type
-- depends_on: {{ ref('dedup_exchange_rate_ab1') }}
select
accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id,
accurateCastOrNull(trim(BOTH '"' from id), '{{ dbt_utils.type_bigint() }}') as id,
nullif(accurateCastOrNull(trim(BOTH '"' from currency), '{{ dbt_utils.type_string() }}'), 'null') as currency,
toDate(parseDateTimeBestEffortOrNull(trim(BOTH '"' from {{ empty_string_to_null('date') }}))) as date,
parseDateTime64BestEffortOrNull(trim(BOTH '"' from {{ empty_string_to_null('timestamp_col') }})) as timestamp_col,
accurateCastOrNull({{ quote('HKD@spéçiäl & characters') }}, '{{ dbt_utils.type_float() }}') as {{ quote('HKD@spéçiäl & characters') }},
accurateCastOrNull(trim(BOTH '"' from {{ quote('HKD@spéçiäl & characters') }}), '{{ dbt_utils.type_float() }}') as {{ quote('HKD@spéçiäl & characters') }},
nullif(accurateCastOrNull(trim(BOTH '"' from HKD_special___characters), '{{ dbt_utils.type_string() }}'), 'null') as HKD_special___characters,
accurateCastOrNull(NZD, '{{ dbt_utils.type_float() }}') as NZD,
accurateCastOrNull(USD, '{{ dbt_utils.type_float() }}') as USD,
accurateCastOrNull(trim(BOTH '"' from NZD), '{{ dbt_utils.type_float() }}') as NZD,
accurateCastOrNull(trim(BOTH '"' from USD), '{{ dbt_utils.type_float() }}') as USD,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql
View File

@@ -6,8 +6,8 @@
-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type
-- depends_on: {{ ref('renamed_dedup_cdc_excluded_ab1') }}
select
accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id,
accurateCastOrNull(_ab_cdc_updated_at, '{{ dbt_utils.type_float() }}') as _ab_cdc_updated_at,
accurateCastOrNull(trim(BOTH '"' from id), '{{ dbt_utils.type_bigint() }}') as id,
accurateCastOrNull(trim(BOTH '"' from _ab_cdc_updated_at), '{{ dbt_utils.type_float() }}') as _ab_cdc_updated_at,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at

17
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -1,6 +1,22 @@
{{ config(
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model
@@ -19,6 +35,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at,

8
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql
View File

@@ -7,20 +7,20 @@
-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type
-- depends_on: _airbyte_test_normalization.dedup_exchange_rate_ab1
select
accurateCastOrNull(id, '
accurateCastOrNull(trim(BOTH '"' from id), '
BIGINT
') as id,
nullif(accurateCastOrNull(trim(BOTH '"' from currency), 'String'), 'null') as currency,
toDate(parseDateTimeBestEffortOrNull(trim(BOTH '"' from nullif(date, '')))) as date,
parseDateTime64BestEffortOrNull(trim(BOTH '"' from nullif(timestamp_col, ''))) as timestamp_col,
accurateCastOrNull("HKD@spéçiäl & characters", '
accurateCastOrNull(trim(BOTH '"' from "HKD@spéçiäl & characters"), '
Float64
') as "HKD@spéçiäl & characters",
nullif(accurateCastOrNull(trim(BOTH '"' from HKD_special___characters), 'String'), 'null') as HKD_special___characters,
accurateCastOrNull(NZD, '
accurateCastOrNull(trim(BOTH '"' from NZD), '
Float64
') as NZD,
accurateCastOrNull(USD, '
accurateCastOrNull(trim(BOTH '"' from USD), '
Float64
') as USD,
_airbyte_ab_id,

7
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -1,6 +1,3 @@
insert into test_normalization.dedup_exchange_rate_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid")
select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid"
from dedup_exchange_rate_scd__dbt_tmp
create table test_normalization.dedup_exchange_rate_scd__dbt_tmp as test_normalization.dedup_exchange_rate_scd__dbt_new_data

7
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql
View File

@@ -1,6 +1,3 @@
insert into test_normalization.renamed_dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid")
select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid"
from renamed_dedup_cdc_excluded_scd__dbt_tmp
create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp as test_normalization.renamed_dedup_cdc_excluded_scd__dbt_new_data

7
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql
View File

@@ -1,6 +1,3 @@
insert into test_normalization.dedup_exchange_rate ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid")
select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid"
from dedup_exchange_rate__dbt_tmp
create table test_normalization.dedup_exchange_rate__dbt_tmp as test_normalization.dedup_exchange_rate__dbt_new_data

7
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql
View File

@@ -1,6 +1,3 @@
insert into test_normalization.renamed_dedup_cdc_excluded ("_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid")
select "_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid"
from renamed_dedup_cdc_excluded__dbt_tmp
create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp as test_normalization.renamed_dedup_cdc_excluded__dbt_new_data

5
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -1,7 +1,9 @@
insert into test_normalization.exchange_rate__dbt_tmp ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid")
insert into test_normalization.exchange_rate__dbt_backup ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "property_binary_data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid")
-- Final base SQL model
-- depends_on: _airbyte_test_normalization.exchange_rate_ab3
@@ -19,6 +21,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at,

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql
View File

@@ -1,6 +1,6 @@
create view _airbyte_test_normalization.dedup_exchange_rate_stg__dbt_tmp
create view _airbyte_test_normalization.dedup_exchange_rate_stg
as (

138
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml
View File

@@ -1,45 +1,29 @@
# This file is necessary to install dbt-utils with dbt deps
# the content will be overwritten by the transform function
# Name your package! Package names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
name: "airbyte_utils"
version: "1.0"
name: airbyte_utils
version: '1.0'
config-version: 2
# This setting configures which "profile" dbt uses for this project. Profiles contain
# database connection information, and should be configured in the ~/.dbt/profiles.yml file
profile: "normalize"
# These configurations specify where dbt should look for different types of files.
# The `model-paths` config, for example, states that source models can be found
# in the "models/" directory. You probably won't need to change these!
model-paths: ["models"]
docs-paths: ["docs"]
analysis-paths: ["analysis"]
test-paths: ["tests"]
seed-paths: ["data"]
macro-paths: ["macros"]
target-path: "../build" # directory which will store compiled SQL files
log-path: "../logs" # directory which will store DBT logs
packages-install-path: "/dbt" # directory which will store external DBT dependencies
clean-targets: # directories to be removed by `dbt clean`
- "build"
- "dbt_modules"
profile: normalize
model-paths:
- models
docs-paths:
- docs
analysis-paths:
- analysis
test-paths:
- tests
seed-paths:
- data
macro-paths:
- macros
target-path: ../build
log-path: ../logs
packages-install-path: /dbt
clean-targets:
- build
- dbt_modules
quoting:
database: true
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
# all schemas should be unquoted
schema: false
identifier: true
# You can define configurations for models in the `model-paths` directory here.
# Using these configurations, you can enable or disable models, change how they
# are materialized, and more!
models:
airbyte_utils:
+materialized: table
@@ -56,6 +40,82 @@ models:
airbyte_views:
+tags: airbyte_internal_views
+materialized: view
vars:
dbt_utils_dispatch_list: ["airbyte_utils"]
dbt_utils_dispatch_list:
- airbyte_utils
json_column: _airbyte_data
models_to_source:
nested_stream_with_co__lting_into_long_names_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co__lting_into_long_names_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co__lting_into_long_names_stg: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co__lting_into_long_names_scd: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co__lting_into_long_names: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
non_nested_stream_wit__lting_into_long_names_ab1: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_wit__lting_into_long_names_ab2: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_wit__lting_into_long_names_ab3: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_wit__lting_into_long_names: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
some_stream_that_was_empty_ab1: test_normalization._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_ab2: test_normalization._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_stg: test_normalization._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_scd: test_normalization._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty: test_normalization._airbyte_raw_some_stream_that_was_empty
simple_stream_with_na__lting_into_long_names_ab1: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
simple_stream_with_na__lting_into_long_names_ab2: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
simple_stream_with_na__lting_into_long_names_ab3: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
simple_stream_with_na__lting_into_long_names: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_scalar_ab1: test_normalization._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar_ab2: test_normalization._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar_ab3: test_normalization._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar: test_normalization._airbyte_raw_conflict_stream_scalar
conflict_stream_array_ab1: test_normalization._airbyte_raw_conflict_stream_array
conflict_stream_array_ab2: test_normalization._airbyte_raw_conflict_stream_array
conflict_stream_array_ab3: test_normalization._airbyte_raw_conflict_stream_array
conflict_stream_array: test_normalization._airbyte_raw_conflict_stream_array
unnest_alias_ab1: test_normalization._airbyte_raw_unnest_alias
unnest_alias_ab2: test_normalization._airbyte_raw_unnest_alias
unnest_alias_ab3: test_normalization._airbyte_raw_unnest_alias
unnest_alias: test_normalization._airbyte_raw_unnest_alias
arrays_ab1: test_normalization._airbyte_raw_arrays
arrays_ab2: test_normalization._airbyte_raw_arrays
arrays_ab3: test_normalization._airbyte_raw_arrays
arrays: test_normalization._airbyte_raw_arrays
nested_stream_with_co___long_names_partition_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co___long_names_partition_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co___long_names_partition_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co___long_names_partition: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
conflict_stream_name_conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name
unnest_alias_children_ab1: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children_ab2: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children_ab3: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children: test_normalization._airbyte_raw_unnest_alias
arrays_nested_array_parent_ab1: test_normalization._airbyte_raw_arrays
arrays_nested_array_parent_ab2: test_normalization._airbyte_raw_arrays
arrays_nested_array_parent_ab3: test_normalization._airbyte_raw_arrays
arrays_nested_array_parent: test_normalization._airbyte_raw_arrays
nested_stream_with_co__ion_double_array_data_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co__ion_double_array_data_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co__ion_double_array_data_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co__ion_double_array_data: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co___names_partition_data_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co___names_partition_data_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co___names_partition_data_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_co___names_partition_data: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
conflict_stream_name____conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name____conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name____conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name
conflict_stream_name____conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name
unnest_alias_children_owner_ab1: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children_owner_ab2: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children_owner_ab3: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children_owner: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children__column___with__quotes_ab1: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children__column___with__quotes_ab2: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children__column___with__quotes_ab3: test_normalization._airbyte_raw_unnest_alias
unnest_alias_children__column___with__quotes: test_normalization._airbyte_raw_unnest_alias

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql
View File

@@ -32,12 +32,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

1
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/sources.yml
View File

@@ -6,6 +6,7 @@ sources:
schema: false
identifier: false
tables:
- name: _airbyte_raw_arrays
- name: _airbyte_raw_conflict_stream_array
- name: _airbyte_raw_conflict_stream_name
- name: _airbyte_raw_conflict_stream_scalar

103
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml
View File

@@ -1,45 +1,29 @@
# This file is necessary to install dbt-utils with dbt deps
# the content will be overwritten by the transform function
# Name your package! Package names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
name: "airbyte_utils"
version: "1.0"
name: airbyte_utils
version: '1.0'
config-version: 2
# This setting configures which "profile" dbt uses for this project. Profiles contain
# database connection information, and should be configured in the ~/.dbt/profiles.yml file
profile: "normalize"
# These configurations specify where dbt should look for different types of files.
# The `model-paths` config, for example, states that source models can be found
# in the "models/" directory. You probably won't need to change these!
model-paths: ["models"]
docs-paths: ["docs"]
analysis-paths: ["analysis"]
test-paths: ["tests"]
seed-paths: ["data"]
macro-paths: ["macros"]
target-path: "../build" # directory which will store compiled SQL files
log-path: "../logs" # directory which will store DBT logs
packages-install-path: "/dbt" # directory which will store external DBT dependencies
clean-targets: # directories to be removed by `dbt clean`
- "build"
- "dbt_modules"
profile: normalize
model-paths:
- models
docs-paths:
- docs
analysis-paths:
- analysis
test-paths:
- tests
seed-paths:
- data
macro-paths:
- macros
target-path: ../build
log-path: ../logs
packages-install-path: /dbt
clean-targets:
- build
- dbt_modules
quoting:
database: true
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
# all schemas should be unquoted
schema: false
identifier: true
# You can define configurations for models in the `model-paths` directory here.
# Using these configurations, you can enable or disable models, change how they
# are materialized, and more!
models:
airbyte_utils:
+materialized: table
@@ -56,6 +40,47 @@ models:
airbyte_views:
+tags: airbyte_internal_views
+materialized: view
vars:
dbt_utils_dispatch_list: ["airbyte_utils"]
dbt_utils_dispatch_list:
- airbyte_utils
json_column: _airbyte_data
models_to_source:
exchange_rate_ab1: test_normalization._airbyte_raw_exchange_rate
exchange_rate_ab2: test_normalization._airbyte_raw_exchange_rate
exchange_rate_ab3: test_normalization._airbyte_raw_exchange_rate
exchange_rate: test_normalization._airbyte_raw_exchange_rate
dedup_exchange_rate_ab1: test_normalization._airbyte_raw_dedup_exchange_rate
dedup_exchange_rate_ab2: test_normalization._airbyte_raw_dedup_exchange_rate
dedup_exchange_rate_stg: test_normalization._airbyte_raw_dedup_exchange_rate
dedup_exchange_rate_scd: test_normalization._airbyte_raw_dedup_exchange_rate
dedup_exchange_rate: test_normalization._airbyte_raw_dedup_exchange_rate
renamed_dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded
renamed_dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded
renamed_dedup_cdc_excluded_stg: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded
renamed_dedup_cdc_excluded_scd: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded
renamed_dedup_cdc_excluded: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded
dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_dedup_cdc_excluded
dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_dedup_cdc_excluded
dedup_cdc_excluded_stg: test_normalization._airbyte_raw_dedup_cdc_excluded
dedup_cdc_excluded_scd: test_normalization._airbyte_raw_dedup_cdc_excluded
dedup_cdc_excluded: test_normalization._airbyte_raw_dedup_cdc_excluded
pos_dedup_cdcx_ab1: test_normalization._airbyte_raw_pos_dedup_cdcx
pos_dedup_cdcx_ab2: test_normalization._airbyte_raw_pos_dedup_cdcx
pos_dedup_cdcx_stg: test_normalization._airbyte_raw_pos_dedup_cdcx
pos_dedup_cdcx_scd: test_normalization._airbyte_raw_pos_dedup_cdcx
pos_dedup_cdcx: test_normalization._airbyte_raw_pos_dedup_cdcx
_1_prefix_startwith_number_ab1: test_normalization._airbyte_raw_1_prefix_startwith_number
_1_prefix_startwith_number_ab2: test_normalization._airbyte_raw_1_prefix_startwith_number
_1_prefix_startwith_number_stg: test_normalization._airbyte_raw_1_prefix_startwith_number
_1_prefix_startwith_number_scd: test_normalization._airbyte_raw_1_prefix_startwith_number
_1_prefix_startwith_number: test_normalization._airbyte_raw_1_prefix_startwith_number
multiple_column_names_conflicts_ab1: test_normalization._airbyte_raw_multiple_column_names_conflicts
multiple_column_names_conflicts_ab2: test_normalization._airbyte_raw_multiple_column_names_conflicts
multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts
multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts
multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts
types_testing_ab1: test_normalization._airbyte_raw_types_testing
types_testing_ab2: test_normalization._airbyte_raw_types_testing
types_testing_stg: test_normalization._airbyte_raw_types_testing
types_testing_scd: test_normalization._airbyte_raw_types_testing
types_testing: test_normalization._airbyte_raw_types_testing

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -37,6 +37,7 @@ select
json_value(_airbyte_data, ''$."datetime_no_tz"'') as datetime_no_tz,
json_value(_airbyte_data, ''$."time_tz"'') as time_tz,
json_value(_airbyte_data, ''$."time_no_tz"'') as time_no_tz,
json_value(_airbyte_data, ''$."property_binary_data"'') as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
SYSDATETIME() as _airbyte_normalized_at
@@ -74,6 +75,7 @@ select
cast(nullif(time_no_tz, '''') as
time
) as time_no_tz,
CAST(property_binary_data as XML ).value(''.'',''binary'') as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
SYSDATETIME() as _airbyte_normalized_at
@@ -102,6 +104,7 @@ select
NVARCHAR(max)), ''''), ''-'', coalesce(cast(datetime_no_tz as
NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_tz as
NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_no_tz as
NVARCHAR(max)), ''''), ''-'', coalesce(cast(property_binary_data as
NVARCHAR(max)), ''''),''''), '''') as
NVARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid,
tmp.*
@@ -124,6 +127,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
SYSDATETIME() as _airbyte_normalized_at,

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -32,12 +32,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

17
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -1,6 +1,22 @@
{{ config(
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model
@@ -19,6 +35,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at,

1
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml
View File

@@ -13,3 +13,4 @@ sources:
- name: _airbyte_raw_multiple_column_names_conflicts
- name: _airbyte_raw_pos_dedup_cdcx
- name: _airbyte_raw_renamed_dedup_cdc_excluded
- name: _airbyte_raw_types_testing

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -37,6 +37,7 @@ select
json_value(_airbyte_data, ''$."datetime_no_tz"'') as datetime_no_tz,
json_value(_airbyte_data, ''$."time_tz"'') as time_tz,
json_value(_airbyte_data, ''$."time_no_tz"'') as time_no_tz,
json_value(_airbyte_data, ''$."property_binary_data"'') as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
SYSDATETIME() as _airbyte_normalized_at
@@ -74,6 +75,7 @@ select
cast(nullif(time_no_tz, '''') as
time
) as time_no_tz,
CAST(property_binary_data as XML ).value(''.'',''binary'') as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
SYSDATETIME() as _airbyte_normalized_at
@@ -102,6 +104,7 @@ select
NVARCHAR(max)), ''''), ''-'', coalesce(cast(datetime_no_tz as
NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_tz as
NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_no_tz as
NVARCHAR(max)), ''''), ''-'', coalesce(cast(property_binary_data as
NVARCHAR(max)), ''''),''''), '''') as
NVARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid,
tmp.*
@@ -124,6 +127,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
SYSDATETIME() as _airbyte_normalized_at,

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql
View File

@@ -32,12 +32,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

50
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -3,41 +3,43 @@
create table
test_normalization.`exchange_rate__dbt_tmp`
as (
with __dbt__cte__exchange_rate_ab1 as (
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
-- depends_on: test_normalization._airbyte_raw_exchange_rate
select
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."id"' RETURNING CHAR) as id,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."currency"' RETURNING CHAR) as currency,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."date"' RETURNING CHAR) as `date`,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."timestamp_col"' RETURNING CHAR) as timestamp_col,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."NZD"' RETURNING CHAR) as nzd,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."USD"' RETURNING CHAR) as usd,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."column___with__quotes"' RETURNING CHAR) as `column__'with"_quotes`,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."datetime_tz"' RETURNING CHAR) as datetime_tz,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."datetime_no_tz"' RETURNING CHAR) as datetime_no_tz,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."time_tz"' RETURNING CHAR) as time_tz,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."time_no_tz"' RETURNING CHAR) as time_no_tz,
json_value(_airbyte_data,
'$."property_binary_data"' RETURNING CHAR) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP
as _airbyte_normalized_at
from test_normalization._airbyte_raw_exchange_rate as table_alias
@@ -48,7 +50,7 @@ where 1 = 1
-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type
-- depends_on: __dbt__cte__exchange_rate_ab1
select
cast(id as
cast(id as
signed
) as id,
cast(currency as char(1024)) as currency,
@@ -57,14 +59,14 @@ select
end as `date`
,
cast(nullif(timestamp_col, '') as char(1024)) as timestamp_col,
cast(`HKD@spéçiäl & characters` as
cast(`HKD@spéçiäl & characters` as
float
) as `HKD@spéçiäl & characters`,
cast(hkd_special___characters as char(1024)) as hkd_special___characters,
cast(nzd as
cast(nzd as
float
) as nzd,
cast(usd as
cast(usd as
float
) as usd,
cast(`column__'with"_quotes` as char(1024)) as `column__'with"_quotes`,
@@ -74,12 +76,13 @@ select
end as datetime_no_tz
,
nullif(cast(time_tz as char(1024)), "") as time_tz,
nullif(cast(time_no_tz as
nullif(cast(time_no_tz as
time
), "") as time_no_tz,
cast(FROM_BASE64(property_binary_data) as binary) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP
as _airbyte_normalized_at
from __dbt__cte__exchange_rate_ab1
@@ -90,7 +93,7 @@ where 1 = 1
-- SQL model to build a hash column based on the values of this record
-- depends_on: __dbt__cte__exchange_rate_ab2
select
md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), '')) as char)) as _airbyte_exchange_rate_hashid,
md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), ''), '-', coalesce(cast(property_binary_data as char), '')) as char)) as _airbyte_exchange_rate_hashid,
tmp.*
from __dbt__cte__exchange_rate_ab2 tmp
-- exchange_rate
@@ -111,9 +114,10 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP
as _airbyte_normalized_at,
_airbyte_exchange_rate_hashid

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -32,12 +32,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

17
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -1,6 +1,22 @@
{{ config(
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model
@@ -19,6 +35,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at,

50
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -3,41 +3,43 @@
create table
test_normalization.`exchange_rate__dbt_tmp`
as (
with __dbt__cte__exchange_rate_ab1 as (
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
-- depends_on: test_normalization._airbyte_raw_exchange_rate
select
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."id"' RETURNING CHAR) as id,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."currency"' RETURNING CHAR) as currency,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."date"' RETURNING CHAR) as `date`,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."timestamp_col"' RETURNING CHAR) as timestamp_col,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."NZD"' RETURNING CHAR) as nzd,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."USD"' RETURNING CHAR) as usd,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."column___with__quotes"' RETURNING CHAR) as `column__'with"_quotes`,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."datetime_tz"' RETURNING CHAR) as datetime_tz,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."datetime_no_tz"' RETURNING CHAR) as datetime_no_tz,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."time_tz"' RETURNING CHAR) as time_tz,
json_value(_airbyte_data,
json_value(_airbyte_data,
'$."time_no_tz"' RETURNING CHAR) as time_no_tz,
json_value(_airbyte_data,
'$."property_binary_data"' RETURNING CHAR) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP
as _airbyte_normalized_at
from test_normalization._airbyte_raw_exchange_rate as table_alias
@@ -48,7 +50,7 @@ where 1 = 1
-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type
-- depends_on: __dbt__cte__exchange_rate_ab1
select
cast(id as
cast(id as
signed
) as id,
cast(currency as char(1024)) as currency,
@@ -57,14 +59,14 @@ select
end as `date`
,
cast(nullif(timestamp_col, '') as char(1024)) as timestamp_col,
cast(`HKD@spéçiäl & characters` as
cast(`HKD@spéçiäl & characters` as
float
) as `HKD@spéçiäl & characters`,
cast(hkd_special___characters as char(1024)) as hkd_special___characters,
cast(nzd as
cast(nzd as
float
) as nzd,
cast(usd as
cast(usd as
float
) as usd,
cast(`column__'with"_quotes` as char(1024)) as `column__'with"_quotes`,
@@ -74,12 +76,13 @@ select
end as datetime_no_tz
,
nullif(cast(time_tz as char(1024)), "") as time_tz,
nullif(cast(time_no_tz as
nullif(cast(time_no_tz as
time
), "") as time_no_tz,
cast(FROM_BASE64(property_binary_data) as binary) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP
as _airbyte_normalized_at
from __dbt__cte__exchange_rate_ab1
@@ -90,7 +93,7 @@ where 1 = 1
-- SQL model to build a hash column based on the values of this record
-- depends_on: __dbt__cte__exchange_rate_ab2
select
md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), '')) as char)) as _airbyte_exchange_rate_hashid,
md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), ''), '-', coalesce(cast(property_binary_data as char), '')) as char)) as _airbyte_exchange_rate_hashid,
tmp.*
from __dbt__cte__exchange_rate_ab2 tmp
-- exchange_rate
@@ -111,9 +114,10 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
CURRENT_TIMESTAMP
as _airbyte_normalized_at,
_airbyte_exchange_rate_hashid

8
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -22,6 +22,7 @@ select
json_value("_AIRBYTE_DATA", '$."datetime_no_tz"') as datetime_no_tz,
json_value("_AIRBYTE_DATA", '$."time_tz"') as time_tz,
json_value("_AIRBYTE_DATA", '$."time_no_tz"') as time_no_tz,
json_value("_AIRBYTE_DATA", '$."property_binary_data"') as property_binary_data,
"_AIRBYTE_AB_ID",
"_AIRBYTE_EMITTED_AT",
@@ -68,6 +69,7 @@ select
cast(nullif(time_no_tz, '') as
varchar2(4000)
) as time_no_tz,
cast(property_binary_data as varchar2(4000)) as property_binary_data,
"_AIRBYTE_AB_ID",
"_AIRBYTE_EMITTED_AT",
@@ -119,7 +121,10 @@ select
time_tz || '~' ||
time_no_tz
time_no_tz || '~' ||
property_binary_data
) as "_AIRBYTE_EXCHANGE_RATE_HASHID",
tmp.*
@@ -142,6 +147,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
"_AIRBYTE_AB_ID",
"_AIRBYTE_EMITTED_AT",

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -32,12 +32,12 @@
from (
select distinct {{ quote('_AIRBYTE_UNIQUE_KEY') }} as unique_key
from {{ this }}
where 1=1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), this.schema + '.' + quote('dedup_exchange_rate')) }}
where 1=1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), quote(this.schema) + '.' + quote('dedup_exchange_rate')) }}
) recent_records
left join (
select {{ quote('_AIRBYTE_UNIQUE_KEY') }} as unique_key, count({{ quote('_AIRBYTE_UNIQUE_KEY') }}) as active_count
from {{ this }}
where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), this.schema + '.' + quote('dedup_exchange_rate')) }}
where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), quote(this.schema) + '.' + quote('dedup_exchange_rate')) }}
group by {{ quote('_AIRBYTE_UNIQUE_KEY') }}
) active_counts
on recent_records.unique_key = active_counts.unique_key

17
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -1,6 +1,22 @@
{{ config(
unique_key = quote('_AIRBYTE_AB_ID'),
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model
@@ -19,6 +35,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
{{ quote('_AIRBYTE_AB_ID') }},
{{ quote('_AIRBYTE_EMITTED_AT') }},
{{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }},

8
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -22,6 +22,7 @@ select
json_value("_AIRBYTE_DATA", '$."datetime_no_tz"') as datetime_no_tz,
json_value("_AIRBYTE_DATA", '$."time_tz"') as time_tz,
json_value("_AIRBYTE_DATA", '$."time_no_tz"') as time_no_tz,
json_value("_AIRBYTE_DATA", '$."property_binary_data"') as property_binary_data,
"_AIRBYTE_AB_ID",
"_AIRBYTE_EMITTED_AT",
@@ -68,6 +69,7 @@ select
cast(nullif(time_no_tz, '') as
varchar2(4000)
) as time_no_tz,
cast(property_binary_data as varchar2(4000)) as property_binary_data,
"_AIRBYTE_AB_ID",
"_AIRBYTE_EMITTED_AT",
@@ -119,7 +121,10 @@ select
time_tz || '~' ||
time_no_tz
time_no_tz || '~' ||
property_binary_data
) as "_AIRBYTE_EXCHANGE_RATE_HASHID",
tmp.*
@@ -142,6 +147,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
"_AIRBYTE_AB_ID",
"_AIRBYTE_EMITTED_AT",

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('some_stream_that_was_empty')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('some_stream_that_was_empty')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('some_stream_that_was_empty')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('some_stream_that_was_empty')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='arrays_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays_nested_array_parent.sql
View File

@@ -1,6 +1,22 @@
{{ config(
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='arrays_nested_array_parent_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "nested" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='conflict_stream_array_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='conflict_stream_name_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql
View File

@@ -1,6 +1,22 @@
{{ config(
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='conflict_stream_name___conflict_stream_name_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "nested" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql
View File

@@ -1,6 +1,22 @@
{{ config(
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='conflict_stream_name_conflict_stream_name_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "nested" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='conflict_stream_scalar_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='non_nested_stream_wi__lting_into_long_names_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='unnest_alias_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql
View File

@@ -1,6 +1,22 @@
{{ config(
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='unnest_alias_childre__column___with__quotes_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "nested" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql
View File

@@ -1,6 +1,22 @@
{{ config(
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='unnest_alias_children_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "nested" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql
View File

@@ -1,6 +1,22 @@
{{ config(
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='unnest_alias_children_owner_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "nested" ]
) }}
-- Final base SQL model

5
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -21,6 +21,7 @@ select
jsonb_extract_path_text(_airbyte_data, 'datetime_no_tz') as datetime_no_tz,
jsonb_extract_path_text(_airbyte_data, 'time_tz') as time_tz,
jsonb_extract_path_text(_airbyte_data, 'time_no_tz') as time_no_tz,
jsonb_extract_path_text(_airbyte_data, 'property_binary_data') as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at
@@ -65,6 +66,7 @@ select
cast(nullif(time_no_tz, '') as
time
) as time_no_tz,
cast(decode(property_binary_data, 'base64') as bytea) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at
@@ -76,7 +78,7 @@ where 1 = 1
-- SQL model to build a hash column based on the values of this record
-- depends_on: __dbt__cte__exchange_rate_ab2
select
md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') as text)) as _airbyte_exchange_rate_hashid,
md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid,
tmp.*
from __dbt__cte__exchange_rate_ab2 tmp
-- exchange_rate
@@ -97,6 +99,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at,

1
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql
View File

@@ -20,6 +20,7 @@ select
{{ json_extract_scalar('_airbyte_data', ['datetime_no_tz'], ['datetime_no_tz']) }} as datetime_no_tz,
{{ json_extract_scalar('_airbyte_data', ['time_tz'], ['time_tz']) }} as time_tz,
{{ json_extract_scalar('_airbyte_data', ['time_no_tz'], ['time_no_tz']) }} as time_no_tz,
{{ json_extract_scalar('_airbyte_data', ['property_binary_data'], ['property_binary_data']) }} as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at

1
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql
View File

@@ -20,6 +20,7 @@ select
cast({{ empty_string_to_null('datetime_no_tz') }} as {{ type_timestamp_without_timezone() }}) as datetime_no_tz,
cast({{ empty_string_to_null('time_tz') }} as {{ type_time_with_timezone() }}) as time_tz,
cast({{ empty_string_to_null('time_no_tz') }} as {{ type_time_without_timezone() }}) as time_no_tz,
cast(decode(property_binary_data, 'base64') as {{ type_binary() }}) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at

1
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql
View File

@@ -21,6 +21,7 @@ select
'datetime_no_tz',
'time_tz',
'time_no_tz',
'property_binary_data',
]) }} as _airbyte_exchange_rate_hashid,
tmp.*
from {{ ref('exchange_rate_ab2') }} tmp

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/1_prefix_startwith_number_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('1_prefix_startwith_number')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('1_prefix_startwith_number')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('1_prefix_startwith_number')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('1_prefix_startwith_number')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/multiple_column_names_conflicts_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('multiple_column_names_conflicts')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('multiple_column_names_conflicts')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('multiple_column_names_conflicts')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('multiple_column_names_conflicts')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('pos_dedup_cdcx')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('pos_dedup_cdcx')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('pos_dedup_cdcx')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('pos_dedup_cdcx')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('types_testing')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('types_testing')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('types_testing')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('types_testing')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

17
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model
@@ -20,6 +36,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at,

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql
View File

@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -2,6 +2,22 @@
indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}],
unique_key = '_airbyte_ab_id',
schema = "test_normalization",
post_hook = ["
{%
set scd_table_relation = adapter.get_relation(
database=this.database,
schema=this.schema,
identifier='exchange_rate_scd'
)
%}
{%
if scd_table_relation is not none
%}
{%
do adapter.drop_relation(scd_table_relation)
%}
{% endif %}
"],
tags = [ "top-level" ]
) }}
-- Final base SQL model

5
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql
View File

@@ -21,6 +21,7 @@ select
jsonb_extract_path_text(_airbyte_data, 'datetime_no_tz') as datetime_no_tz,
jsonb_extract_path_text(_airbyte_data, 'time_tz') as time_tz,
jsonb_extract_path_text(_airbyte_data, 'time_no_tz') as time_no_tz,
jsonb_extract_path_text(_airbyte_data, 'property_binary_data') as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at
@@ -65,6 +66,7 @@ select
cast(nullif(time_no_tz, '') as
time
) as time_no_tz,
cast(decode(property_binary_data, 'base64') as bytea) as property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at
@@ -76,7 +78,7 @@ where 1 = 1
-- SQL model to build a hash column based on the values of this record
-- depends_on: __dbt__cte__exchange_rate_ab2
select
md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') as text)) as _airbyte_exchange_rate_hashid,
md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid,
tmp.*
from __dbt__cte__exchange_rate_ab2 tmp
-- exchange_rate
@@ -97,6 +99,7 @@ select
datetime_no_tz,
time_tz,
time_no_tz,
property_binary_data,
_airbyte_ab_id,
_airbyte_emitted_at,
now() as _airbyte_normalized_at,

140
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml
View File

@@ -51,77 +51,77 @@ dispatch:
vars:
json_column: _airbyte_data
models_to_source:
nested_stream_with_complex_columns_resulting_into_long_names_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_stg: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_scd: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names_ab1: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names_ab2: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names_ab3: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
some_stream_that_was_empty_ab1: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_ab2: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_stg: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_scd: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty
nested_stream_with_complex_columns_resulting_into_long_names_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_stg: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_scd: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names_ab1: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names_ab2: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names_ab3: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
non_nested_stream_without_namespace_resulting_into_long_names: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
some_stream_that_was_empty_ab1: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_ab2: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_stg: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty_scd: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty
some_stream_that_was_empty: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty
simple_stream_with_namespace_resulting_into_long_names_ab1: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
simple_stream_with_namespace_resulting_into_long_names_ab2: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
simple_stream_with_namespace_resulting_into_long_names_ab3: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
simple_stream_with_namespace_resulting_into_long_names: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
conflict_stream_name_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_scalar_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar
conflict_stream_array_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_array
conflict_stream_array_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_array
conflict_stream_array_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_array
conflict_stream_array: test_normalization_xjvlg._airbyte_raw_conflict_stream_array
unnest_alias_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias: test_normalization_xjvlg._airbyte_raw_unnest_alias
arrays_ab1: test_normalization_xjvlg._airbyte_raw_arrays
arrays_ab2: test_normalization_xjvlg._airbyte_raw_arrays
arrays_ab3: test_normalization_xjvlg._airbyte_raw_arrays
arrays: test_normalization_xjvlg._airbyte_raw_arrays
nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
conflict_stream_name_conflict_stream_name_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
unnest_alias_children_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children: test_normalization_xjvlg._airbyte_raw_unnest_alias
arrays_nested_array_parent_ab1: test_normalization_xjvlg._airbyte_raw_arrays
arrays_nested_array_parent_ab2: test_normalization_xjvlg._airbyte_raw_arrays
arrays_nested_array_parent_ab3: test_normalization_xjvlg._airbyte_raw_arrays
arrays_nested_array_parent: test_normalization_xjvlg._airbyte_raw_arrays
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_conflict_stream_name: test_normalization_xjvlg._airbyte_raw_conflict_stream_name
unnest_alias_children_owner_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_owner_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_owner_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_owner: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes: test_normalization_xjvlg._airbyte_raw_unnest_alias
conflict_stream_name_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_scalar_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar
conflict_stream_scalar: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar
conflict_stream_array_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_array
conflict_stream_array_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_array
conflict_stream_array_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_array
conflict_stream_array: test_normalization_iprwf._airbyte_raw_conflict_stream_array
unnest_alias_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias: test_normalization_iprwf._airbyte_raw_unnest_alias
arrays_ab1: test_normalization_iprwf._airbyte_raw_arrays
arrays_ab2: test_normalization_iprwf._airbyte_raw_arrays
arrays_ab3: test_normalization_iprwf._airbyte_raw_arrays
arrays: test_normalization_iprwf._airbyte_raw_arrays
nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
conflict_stream_name_conflict_stream_name_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name: test_normalization_iprwf._airbyte_raw_conflict_stream_name
unnest_alias_children_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children: test_normalization_iprwf._airbyte_raw_unnest_alias
arrays_nested_array_parent_ab1: test_normalization_iprwf._airbyte_raw_arrays
arrays_nested_array_parent_ab2: test_normalization_iprwf._airbyte_raw_arrays
arrays_nested_array_parent_ab3: test_normalization_iprwf._airbyte_raw_arrays
arrays_nested_array_parent: test_normalization_iprwf._airbyte_raw_arrays
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
nested_stream_with_complex_columns_resulting_into_long_names_partition_data: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_name
conflict_stream_name_conflict_stream_name_conflict_stream_name: test_normalization_iprwf._airbyte_raw_conflict_stream_name
unnest_alias_children_owner_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_owner_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_owner_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_owner: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias
unnest_alias_children_owner_column___with__quotes: test_normalization_iprwf._airbyte_raw_unnest_alias

15
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql Normal file
View File

@@ -0,0 +1,15 @@
delete from "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_scd"
where (_airbyte_unique_key_scd) in (
select (_airbyte_unique_key_scd)
from "nested_stream_with_complex_columns_resulti__dbt_tmp185252546953"
);
insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid")
(
select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid"
from "nested_stream_with_complex_columns_resulti__dbt_tmp185252546953"
)

6
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql
View File

@@ -2,7 +2,7 @@
create table
"integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd"
"normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd"
compound sortkey(_airbyte_active_row,_airbyte_unique_key_scd,_airbyte_emitted_at)
@@ -14,8 +14,8 @@ with
input_data as (
select *
from "integrationtests"._airbyte_test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_stg"
-- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
from "normalization_tests"._airbyte_test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_stg"
-- nested_stream_with_complex_columns_resulting_into_long_names from "normalization_tests".test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
),
scd_data as (

29
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql
View File

@@ -1,29 +0,0 @@
create table
"integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names"
compound sortkey(_airbyte_unique_key,_airbyte_emitted_at)
as (
-- Final base SQL model
-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd"
select
_airbyte_unique_key,
id,
date,
"partition",
_airbyte_ab_id,
_airbyte_emitted_at,
getdate() as _airbyte_normalized_at,
_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid
from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd"
-- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
where 1 = 1
and _airbyte_active_row = 1
);

15
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql Normal file
View File

@@ -0,0 +1,15 @@
delete from "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names"
where (_airbyte_unique_key) in (
select (_airbyte_unique_key)
from "nested_stream_with_complex_columns_resulti__dbt_tmp185302582647"
);
insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid")
(
select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid"
from "nested_stream_with_complex_columns_resulti__dbt_tmp185302582647"
)

9
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql Normal file
View File

@@ -0,0 +1,9 @@
insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid")
(
select "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid"
from "nested_stream_with_complex_columns_resulti__dbt_tmp185307922088"
)

9
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql Normal file
View File

@@ -0,0 +1,9 @@
insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid")
(
select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid"
from "nested_stream_with_complex_columns_resulti__dbt_tmp185315344939"
)

9
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql Normal file
View File

@@ -0,0 +1,9 @@
insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid")
(
select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid"
from "nested_stream_with_complex_columns_resulti__dbt_tmp185315267996"
)

29
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql Normal file
View File

@@ -0,0 +1,29 @@
create table
"normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names"
compound sortkey(_airbyte_unique_key,_airbyte_emitted_at)
as (
-- Final base SQL model
-- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd"
select
_airbyte_unique_key,
id,
date,
"partition",
_airbyte_ab_id,
_airbyte_emitted_at,
getdate() as _airbyte_normalized_at,
_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid
from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd"
-- nested_stream_with_complex_columns_resulting_into_long_names from "normalization_tests".test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names
where 1 = 1
and _airbyte_active_row = 1
);

8
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql
View File

@@ -2,7 +2,7 @@
create table
"integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition"
"normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition"
compound sortkey(_airbyte_emitted_at)
@@ -12,7 +12,7 @@
with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as (
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd"
-- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd"
select
_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid,
"partition"."double_array_data" as double_array_data,
@@ -20,7 +20,7 @@ select
_airbyte_ab_id,
_airbyte_emitted_at,
getdate() as _airbyte_normalized_at
from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" as table_alias
from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" as table_alias
-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition
where 1 = 1
and "partition" is not null
@@ -62,7 +62,7 @@ select
getdate() as _airbyte_normalized_at,
_airbyte_partition_hashid
from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3
-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd"
-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd"
where 1 = 1
);

10
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql
View File

@@ -2,7 +2,7 @@
create table
"integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition_data"
"normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition_data"
compound sortkey(_airbyte_emitted_at)
@@ -12,13 +12,13 @@
with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as (
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition"
-- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition"
with joined as (
select
table_alias._airbyte_partition_hashid as _airbyte_hashid,
_airbyte_nested_data
from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.data as _airbyte_nested_data
from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.data as _airbyte_nested_data
)
select
_airbyte_partition_hashid,
@@ -26,7 +26,7 @@ select
_airbyte_ab_id,
_airbyte_emitted_at,
getdate() as _airbyte_normalized_at
from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias
from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias
-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA
left join joined on _airbyte_partition_hashid = joined._airbyte_hashid
where 1 = 1
@@ -67,7 +67,7 @@ select
getdate() as _airbyte_normalized_at,
_airbyte_data_hashid
from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3
-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition"
-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition"
where 1 = 1
);

10
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql
View File

@@ -2,7 +2,7 @@
create table
"integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data"
"normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data"
compound sortkey(_airbyte_emitted_at)
@@ -12,13 +12,13 @@
with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as (
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition"
-- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition"
with joined as (
select
table_alias._airbyte_partition_hashid as _airbyte_hashid,
_airbyte_nested_data
from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.double_array_data as _airbyte_nested_data
from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.double_array_data as _airbyte_nested_data
)
select
_airbyte_partition_hashid,
@@ -26,7 +26,7 @@ select
_airbyte_ab_id,
_airbyte_emitted_at,
getdate() as _airbyte_normalized_at
from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias
from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias
-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data
left join joined on _airbyte_partition_hashid = joined._airbyte_hashid
where 1 = 1
@@ -67,7 +67,7 @@ select
getdate() as _airbyte_normalized_at,
_airbyte_double_array_data_hashid
from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3
-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition"
-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition"
where 1 = 1
);

6
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql
View File

@@ -1,11 +1,11 @@
{{ config(
sort = "_airbyte_emitted_at",
unique_key = '_airbyte_ab_id',
schema = "_airbyte_test_normalization_xjvlg",
schema = "_airbyte_test_normalization_iprwf",
tags = [ "top-level-intermediate" ]
) }}
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
-- depends_on: {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
-- depends_on: {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
select
{{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id,
{{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date,
@@ -13,7 +13,7 @@ select
_airbyte_ab_id,
_airbyte_emitted_at,
{{ current_timestamp() }} as _airbyte_normalized_at
from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias
from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias
-- nested_stream_with_complex_columns_resulting_into_long_names
where 1 = 1
{{ incremental_clause('_airbyte_emitted_at', this) }}

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql
View File

@@ -1,7 +1,7 @@
{{ config(
sort = "_airbyte_emitted_at",
unique_key = '_airbyte_ab_id',
schema = "_airbyte_test_normalization_xjvlg",
schema = "_airbyte_test_normalization_iprwf",
tags = [ "top-level-intermediate" ]
) }}
-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql
View File

@@ -1,6 +1,6 @@
{{ config(
sort = "_airbyte_emitted_at",
schema = "_airbyte_test_normalization_xjvlg",
schema = "_airbyte_test_normalization_iprwf",
tags = [ "nested-intermediate" ]
) }}
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql
View File

@@ -1,6 +1,6 @@
{{ config(
sort = "_airbyte_emitted_at",
schema = "_airbyte_test_normalization_xjvlg",
schema = "_airbyte_test_normalization_iprwf",
tags = [ "nested-intermediate" ]
) }}
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql
View File

@@ -1,6 +1,6 @@
{{ config(
sort = "_airbyte_emitted_at",
schema = "_airbyte_test_normalization_xjvlg",
schema = "_airbyte_test_normalization_iprwf",
tags = [ "nested-intermediate" ]
) }}
-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema

12
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql
View File

@@ -1,7 +1,7 @@
{{ config(
sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"],
unique_key = "_airbyte_unique_key_scd",
schema = "test_normalization_xjvlg",
schema = "test_normalization_iprwf",
post_hook = ["
{%
set final_table_relation = adapter.get_relation(
@@ -33,12 +33,12 @@
from (
select distinct _airbyte_unique_key as unique_key
from {{ this }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
) recent_records
left join (
select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count
from {{ this }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }}
group by _airbyte_unique_key
) active_counts
on recent_records.unique_key = active_counts.unique_key
@@ -48,7 +48,7 @@
-- We have to have a non-empty query, so just do a noop delete
delete from {{ this }} where 1=0
{% endif %}
","drop view _airbyte_test_normalization_xjvlg.nested_stream_with_complex_columns_resulting_into_long_names_stg"],
","drop view _airbyte_test_normalization_iprwf.nested_stream_with_complex_columns_resulting_into_long_names_stg"],
tags = [ "top-level" ]
) }}
-- depends_on: ref('nested_stream_with_complex_columns_resulting_into_long_names_stg')
@@ -59,7 +59,7 @@ new_data as (
select
*
from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') }}
-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
where 1 = 1
{{ incremental_clause('_airbyte_emitted_at', this) }}
),
@@ -95,7 +95,7 @@ input_data as (
input_data as (
select *
from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') }}
-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
),
{% endif %}
scd_data as (

4
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql
View File

@@ -1,7 +1,7 @@
{{ config(
sort = ["_airbyte_unique_key", "_airbyte_emitted_at"],
unique_key = "_airbyte_unique_key",
schema = "test_normalization_xjvlg",
schema = "test_normalization_iprwf",
tags = [ "top-level" ]
) }}
-- Final base SQL model
@@ -16,7 +16,7 @@ select
{{ current_timestamp() }} as _airbyte_normalized_at,
_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid
from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }}
-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }}
where 1 = 1
and _airbyte_active_row = 1
{{ incremental_clause('_airbyte_emitted_at', this) }}

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql
View File

@@ -1,6 +1,6 @@
{{ config(
sort = "_airbyte_emitted_at",
schema = "test_normalization_xjvlg",
schema = "test_normalization_iprwf",
tags = [ "nested" ]
) }}
-- Final base SQL model

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql
View File

@@ -1,6 +1,6 @@
{{ config(
sort = "_airbyte_emitted_at",
schema = "test_normalization_xjvlg",
schema = "test_normalization_iprwf",
tags = [ "nested" ]
) }}
-- Final base SQL model

2
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql → airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql
View File

@@ -1,6 +1,6 @@
{{ config(
sort = "_airbyte_emitted_at",
schema = "test_normalization_xjvlg",
schema = "test_normalization_iprwf",
tags = [ "nested" ]
) }}
-- Final base SQL model

16
airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml
View File

@@ -1,13 +1,6 @@
version: 2
sources:
- name: test_normalization_namespace
quoting:
database: true
schema: false
identifier: false
tables:
- name: _airbyte_raw_simple_stream_with_namespace_resulting_into_long_names
- name: test_normalization_xjvlg
- name: test_normalization_iprwf
quoting:
database: true
schema: false
@@ -21,3 +14,10 @@ sources:
- name: _airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names
- name: _airbyte_raw_some_stream_that_was_empty
- name: _airbyte_raw_unnest_alias
- name: test_normalization_namespace
quoting:
database: true
schema: false
identifier: false
tables:
- name: _airbyte_raw_simple_stream_with_namespace_resulting_into_long_names

Some files were not shown because too many files have changed in this diff Show More