jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 18:12:08 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-12209: Always include format-version in DESCRIBE FORMATTED and SHOW CREATE TABLE for Iceberg tables

Browse Source 
HiveCatalog does not include format-version for Iceberg tables in the
table's parameters, therefore the output of SHOW CREATE TABLE may not
replicate the original table.
This patch makes sure to add it to both the SHOW CREATE TABLE and
DESCRIBE FORMATTED/EXTENDED output.

Additionally, adds ICEBERG_DEFAULT_FORMAT_VERSION variable to E2E
tests, deducting from IMPALA_ICEBERG_VERSION environment variable.

If Iceberg version is at least 1.4, default format-version is 2, before
1.4 it's 1. This way tests can work with multiple Iceberg versions.

Testing:
 * updated show-create-table.test and show-create-table-with-stats.test
   for Iceberg tables
 * added format-version checks to multiple DESCRIBE FORMATTED tests

Change-Id: I991edf408b24fa73e8a8abe64ac24929aeb8e2f8
Reviewed-on: http://gerrit.cloudera.org:8080/23514
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
Daniel Vanko
2025-10-07 16:07:59 +02:00
committed by Impala Public Jenkins
parent f6ceca2b4d
commit 3d22c7fe05
21 changed files with 243 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
View File

@@ -189,10 +189,10 @@ public class ToSqlUtils {
commonProps.remove(KuduTable.KEY_TABLE_ID);
} else if (table instanceof FeIcebergTable) {
FeIcebergTable feIcebergTable = (FeIcebergTable) table;
if (feIcebergTable.getFormatVersion() == IcebergTable.ICEBERG_FORMAT_V1) {
commonProps.put(TableProperties.FORMAT_VERSION,
String.valueOf(IcebergTable.ICEBERG_FORMAT_V1));
}
// Add "format-version" property if it's not already present.
commonProps.putIfAbsent(IcebergTable.FORMAT_VERSION,
Integer.toString(feIcebergTable.getFormatVersion()));
// Hide Iceberg internal metadata properties
removeHiddenIcebergTableProperties(commonProps);
} else if (table instanceof FePaimonTable) {

3
fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
View File

@@ -164,6 +164,9 @@ public class IcebergTable extends Table implements FeIcebergTable {
// Internal Iceberg table property that specifies the UUID of the table.
public static final String UUID = "uuid";
// Internal Iceberg table property that specifies the table format version.
public static final String FORMAT_VERSION = "format-version";
// Parquet compression codec and compression level table properties.
public static final String PARQUET_COMPRESSION_CODEC =
"write.parquet.compression-codec";

8
fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
View File

@@ -30,6 +30,7 @@ import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.IcebergColumn;
import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.catalog.KuduColumn;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
@@ -236,8 +237,13 @@ public class DescribeResultFactory {
sb.append(MetastoreShim.getAllColumnsInformation(msTable.getSd().getCols(),
msTable.getPartitionKeys(), true, false, true));
if (table instanceof FeIcebergTable) {
FeIcebergTable feIcebergTable = (FeIcebergTable) table;
sb.append(MetastoreShim.getPartitionTransformInformation(
FeIcebergTable.Utils.getPartitionTransformKeys((FeIcebergTable) table)));
FeIcebergTable.Utils.getPartitionTransformKeys(feIcebergTable)));
// msTable is a deep copy hence we can add the "format-version" parameter to it
msTable.getParameters().putIfAbsent(IcebergTable.FORMAT_VERSION,
Integer.toString(feIcebergTable.getFormatVersion()));
}
// Add the extended table metadata information.
sb.append(MetastoreShim.getTableInformation(msTable));

2
testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test vendored
View File

@@ -323,6 +323,7 @@ TBLPROPERTIES (
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','134217728 '
'','write.parquet.compression-codec','zstd '
@@ -342,6 +343,7 @@ ALTER TABLE iceberg_changing_parq_tblprops set TBLPROPERTIES(
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','268435456 '
'','write.parquet.compression-codec','snappy '

2
testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test vendored
View File

@@ -325,6 +325,7 @@ TBLPROPERTIES (
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','1 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','134217728 '
'','write.parquet.compression-codec','zstd '
@@ -344,6 +345,7 @@ ALTER TABLE iceberg_changing_parq_tblprops set TBLPROPERTIES(
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','1 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','268435456 '
'','write.parquet.compression-codec','snappy '

6
testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test vendored
View File

@@ -325,6 +325,7 @@ TBLPROPERTIES (
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','2 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','134217728 '
'','write.parquet.compression-codec','zstd '
@@ -344,6 +345,7 @@ ALTER TABLE iceberg_changing_parq_tblprops set TBLPROPERTIES(
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','2 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','268435456 '
'','write.parquet.compression-codec','snappy '
@@ -379,6 +381,7 @@ string, string, string
ALTER TABLE iceberg_upgrade_v2_no_write_mode SET TBLPROPERTIES('format-version'='2');
DESCRIBE FORMATTED iceberg_upgrade_v2_no_write_mode;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','2 '
'','write.delete.mode ','merge-on-read '
'','write.update.mode ','merge-on-read '
'','write.merge.mode ','merge-on-read '
@@ -392,6 +395,7 @@ ALTER TABLE iceberg_upgrade_v2_delete_mode
SET TBLPROPERTIES('format-version'='2', 'write.delete.mode'='copy-on-write');
DESCRIBE FORMATTED iceberg_upgrade_v2_delete_mode;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','2 '
'','write.delete.mode ','copy-on-write '
---- TYPES
string, string, string
@@ -411,6 +415,7 @@ ALTER TABLE iceberg_upgrade_v2_update_mode
SET TBLPROPERTIES('format-version'='2', 'write.update.mode'='copy-on-write');
DESCRIBE FORMATTED iceberg_upgrade_v2_update_mode;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','2 '
'','write.update.mode ','copy-on-write '
---- TYPES
string, string, string
@@ -430,6 +435,7 @@ ALTER TABLE iceberg_upgrade_v2_merge_mode
SET TBLPROPERTIES('format-version'='2', 'write.merge.mode'='merge-on-read');
DESCRIBE FORMATTED iceberg_upgrade_v2_merge_mode;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','2 '
'','write.merge.mode ','merge-on-read '
---- TYPES
string, string, string

4
testdata/workloads/functional-query/queries/QueryTest/iceberg-catalogs.test vendored
View File

@@ -14,6 +14,7 @@ DESCRIBE FORMATTED iceberg_hadoop_catalogs;
'Location: ','$NAMENODE/test-warehouse/ice_hadoop_cat/$DATABASE/iceberg_hadoop_catalogs','NULL'
'','write.format.default','parquet '
'','iceberg.catalog ','ice_hadoop_cat '
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====
@@ -33,6 +34,7 @@ DESCRIBE FORMATTED iceberg_hadoop_catalogs_with_id;
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','org.$DATABASE.tbl'
'','name ','org.$DATABASE.tbl'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====
@@ -67,6 +69,7 @@ DESCRIBE FORMATTED iceberg_hadoop_cat_with_id_ext;
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','org.$DATABASE.tbl'
'','name ','org.$DATABASE.tbl'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====
@@ -104,6 +107,7 @@ DESCRIBE FORMATTED iceberg_hive_catalogs;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/iceberg_hive_catalogs','NULL'
'','write.format.default','parquet '
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====

8
testdata/workloads/functional-query/queries/QueryTest/iceberg-create-table-like-table.test vendored
View File

@@ -48,6 +48,7 @@ describe formatted ice_hadoop_tbl_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/ice_hadoop_tbl_no_part_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.tables '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -109,6 +110,7 @@ describe formatted ice_hadoop_tbl_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/ice_hadoop_tbl_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.tables '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -171,6 +173,7 @@ describe formatted ice_hadoop_catalog_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/cat_loc/$DATABASE/ice_hadoop_catalog_no_part_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.catalog '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE.db/cat_loc'
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -237,6 +240,7 @@ describe formatted ice_hadoop_catalog_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/cat_loc/$DATABASE/ice_hadoop_catalog_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.catalog '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE.db/cat_loc'
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -299,6 +303,7 @@ describe formatted ice_hive_catalog_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_no_part_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.table_identifier','$DATABASE.ice_hive_catalog_no_part_clone'
row_regex: '','metadata_location ','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_no_part_clone/metadata/.*.metadata.json'
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -365,6 +370,7 @@ describe formatted ice_hive_catalog_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.table_identifier','$DATABASE.ice_hive_catalog_clone'
row_regex: '','metadata_location ','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_clone/metadata/.*.metadata.json'
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -428,6 +434,7 @@ describe formatted ice_hadoop_cat_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location: ','$NAMENODE/test-warehouse/ice_hadoop_cat/$DATABASE/ice_hadoop_cat_no_part_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','$DATABASE.ice_hadoop_cat_no_part_clone'
'','name ','$DATABASE.ice_hadoop_cat_no_part_clone'
@@ -494,6 +501,7 @@ describe formatted ice_hadoop_cat_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location: ','$NAMENODE/test-warehouse/ice_hadoop_cat/$DATABASE/ice_hadoop_cat_clone','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','$DATABASE.ice_hadoop_cat_clone'
'','name ','$DATABASE.ice_hadoop_cat_clone'

15
testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test vendored
View File

@@ -217,6 +217,7 @@ string
DESCRIBE FORMATTED iceberg_hadoop_cat_query;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test/$DATABASE/iceberg_hadoop_cat_query','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test'
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.catalog '
@@ -234,6 +235,7 @@ TBLPROPERTIES('iceberg.catalog'='hadoop.catalog',
DESCRIBE FORMATTED iceberg_hadoop_cat_with_ident;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test/org/db/tbl','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test'
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.catalog '
@@ -256,6 +258,7 @@ TBLPROPERTIES('iceberg.catalog'='hadoop.catalog',
DESCRIBE FORMATTED iceberg_hadoop_cat_with_ident_ext;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test/org/db/tbl','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test'
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.catalog '
@@ -366,6 +369,7 @@ STRING,STRING,STRING,STRING
DESCRIBE FORMATTED iceberg_part_hive_cat;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/iceberg_part_hive_cat','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','metadata_location ','$NAMENODE/test-warehouse/$DATABASE.db/iceberg_part_hive_cat/metadata/.*.metadata.json'
'','table_type ','ICEBERG '
'','iceberg.catalog ','hive.catalog '
@@ -568,6 +572,7 @@ create table iceberg_stored_by (i int, s string, ts timestamp, d date) stored by
---- QUERY
describe formatted iceberg_stored_by;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','table_type ','ICEBERG '
---- TYPES
string, string, string
@@ -580,6 +585,7 @@ create table ice_tbl (i int) stored as iceberg tblproperties('external.table.pur
---- QUERY
describe formatted ice_tbl;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','FALSE '
---- TYPES
string, string, string
@@ -595,6 +601,7 @@ create table ice_hive_cat_tbl (i int) stored as iceberg tblproperties(
---- QUERY
describe formatted ice_hive_cat_tbl;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','FALSE '
---- TYPES
string, string, string
@@ -612,6 +619,7 @@ The table property 'external.table.purge' will be set to 'TRUE' on newly created
---- QUERY
describe formatted ice_hadoop_tbl;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','TRUE '
---- TYPES
string, string, string
@@ -623,6 +631,7 @@ create table ice_tbl_with_pk
stored as iceberg;
describe formatted ice_tbl_with_pk;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','current-schema.*','.*identifier-field-ids\\\\":\[1,3\].*'
---- TYPES
string, string, string
@@ -635,6 +644,7 @@ create table ice_tbl_with_pk_partitioned
stored as iceberg;
describe formatted ice_tbl_with_pk_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','current-schema.*','.*identifier-field-ids\\\\":\[1,2\].*'
---- TYPES
string, string, string
@@ -647,6 +657,7 @@ create table ice_tbl_with_pk_partition_transform
stored as iceberg;
describe formatted ice_tbl_with_pk_partition_transform;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','current-schema.*','.*identifier-field-ids\\\\":\[2,3\].*'
---- TYPES
string, string, string
@@ -659,6 +670,7 @@ create table ctas_pk_in_source_not_used
as select * from ice_tbl_with_pk;
describe formatted ctas_pk_in_source_not_used;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*'
---- RESULTS: VERIFY_IS_NOT_IN
row_regex:'.*','current-schema.*','.*identifier-field-ids.*'
@@ -673,6 +685,7 @@ create table ctas_pk_unpartitioned
as select * from ice_tbl_with_pk;
describe formatted ctas_pk_unpartitioned;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*identifier-field-ids\\\\":\[3\].*'
---- TYPES
STRING,STRING,STRING
@@ -686,6 +699,7 @@ create table ctas_pk_partitioned
as select * from ice_tbl_with_pk;
describe formatted ctas_pk_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*identifier-field-ids\\\\":\[2,3\].*'
---- TYPES
STRING,STRING,STRING
@@ -695,6 +709,7 @@ STRING,STRING,STRING
create table ice_like_pk like ice_tbl_with_pk_partitioned;
describe formatted ice_like_pk;
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*identifier-field-ids\\\\":\[1,2\].*'
---- TYPES
STRING,STRING,STRING

2
testdata/workloads/functional-query/queries/QueryTest/iceberg-external.test vendored
View File

@@ -30,6 +30,7 @@ alter table ice_hive_ext set tblproperties ('external.table.purge'='true');
insert into ice_hive_ext values (6,6);
describe formatted ice_hive_ext
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
---- TYPES
STRING,STRING,STRING
@@ -65,6 +66,7 @@ alter table ice_hadoop_tables_ext set tblproperties ('external.table.purge'='tru
insert into ice_hadoop_tables_ext values (6,6);
describe formatted ice_hadoop_tables_ext
---- RESULTS: VERIFY_IS_SUBSET
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
---- TYPES
STRING,STRING,STRING

4
testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test vendored
View File

@@ -90,6 +90,7 @@ aggregation(SUM, NumRowGroups): 0
describe formatted parquet_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/parquet_partitioned','NULL'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.tables '
'','external.table.purge','true '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -177,6 +178,7 @@ describe formatted parquet_nopartitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/parquet_nopartitioned','NULL'
row_regex: '','metadata_location ','$NAMENODE/test-warehouse/$DATABASE.db/parquet_nopartitioned/metadata/.*.metadata.json'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -237,6 +239,7 @@ describe formatted decimal_tbl;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/decimal_tbl','NULL'
row_regex: '','metadata_location ','$NAMENODE/test-warehouse/$DATABASE.db/decimal_tbl/metadata/.*.metadata.json'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -281,6 +284,7 @@ describe formatted table_at_random_location;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/$DATABASE.db/random_location','NULL'
row_regex: '','metadata_location ','$NAMENODE/test-warehouse/$DATABASE.db/random_location/metadata/.*.metadata.json'
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '

1
testdata/workloads/functional-query/queries/QueryTest/iceberg-missing-data-files.test vendored
View File

@@ -72,6 +72,7 @@ describe formatted missing_files_part
'','default-partition-spec','{\\"spec-id\\":0,\\"fields\\":[{\\"name\\":\\"p\\",\\"transform\\":\\"identity\\",\\"source-id\\":2,\\"field-id\\":1000}]}'
'','engine.hive.enabled ','true '
'','external.table.purge','TRUE '
'','format-version ','2 '
'','numFiles ','2 '
'','numRows ','2 '
'','snapshot-count ','2 '

1
testdata/workloads/functional-query/queries/QueryTest/iceberg-old-fileformat.test vendored
View File

@@ -17,6 +17,7 @@ DESCRIBE FORMATTED iceberg_partitioned_orc_external_old_fileformat;
'','iceberg.table_identifier','functional_parquet.iceberg_partitioned_orc'
'','iceberg.file_format','orc '
'','iceberg.catalog ','hadoop.catalog '
'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- RESULTS: VERIFY_IS_NOT_IN
'','write.format.default','orc '
---- TYPES

5
testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test vendored
View File

@@ -368,6 +368,7 @@ STRING,STRING,STRING,STRING
describe formatted iceberg_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/iceberg_test/iceberg_partitioned','NULL'
'','format-version ','1 '
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.tables '
---- TYPES
@@ -377,6 +378,7 @@ string, string, string
describe formatted iceberg_non_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/iceberg_test/iceberg_non_partitioned','NULL'
'','format-version ','1 '
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.tables '
---- TYPES
@@ -386,6 +388,7 @@ string, string, string
describe formatted hadoop_catalog_test_external;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/hadoop_catalog_test/functional_parquet/hadoop_catalog_test','NULL'
'','format-version ','1 '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/iceberg_test/hadoop_catalog/hadoop_catalog_test'
'','iceberg.table_identifier','functional_parquet.hadoop_catalog_test'
'','write.format.default','parquet '
@@ -397,6 +400,7 @@ string, string, string
describe formatted iceberg_partitioned_orc_external;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc','NULL'
'','format-version ','1 '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc'
'','iceberg.table_identifier','functional_parquet.iceberg_partitioned_orc'
'','write.format.default','orc '
@@ -467,6 +471,7 @@ bigint,bigint
describe formatted iceberg_resolution_test_external;
---- RESULTS: VERIFY_IS_SUBSET
'Location: ','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_resolution_test/functional_parquet/iceberg_resolution_test','NULL'
'','format-version ','1 '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/iceberg_test/hadoop_catalog/iceberg_resolution_test'
'','iceberg.table_identifier','functional_parquet.iceberg_resolution_test'
'','write.format.default','parquet '

1
testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-catalog.test vendored
View File

@@ -64,6 +64,7 @@ row_regex:'Erasure Coding Policy:','$ERASURECODE_POLICY *','NULL'
'','EXTERNAL ','TRUE '
'','bucketing_version ','2 '
'','engine.hive.enabled ','true '
'','format-version ','1 '
'','gc.enabled ','TRUE '
'','numFiles ','1 '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'

1
testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-fgac.test vendored
View File

@@ -75,6 +75,7 @@ row_regex:'Erasure Coding Policy:','$ERASURECODE_POLICY *','NULL'
'','EXTERNAL ','TRUE '
'','bucketing_version ','2 '
'','engine.hive.enabled ','true '
'','format-version ','1 '
'','gc.enabled ','TRUE '
'','numFiles ','1 '
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'

4
testdata/workloads/functional-query/queries/QueryTest/show-create-table-with-stats.test vendored
View File

@@ -65,8 +65,8 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_with_stats (
)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('OBJCAPABILITIES'='EXTREAD,EXTWRITE', 'engine.hive.enabled'='true', 'external.table.purge'='TRUE', 'impala.computeStatsSnapshotIds'='<NUM>', 'impala.lastComputeStatsTime'='<NUM>', 'table_type'='ICEBERG', 'write.delete.mode'='merge-on-read', 'write.format.default'='parquet', 'write.merge.mode'='merge-on-read', 'write.parquet.compression-codec'='snappy', 'write.update.mode'='merge-on-read');
ALTER TABLE show_create_table_test_db.ice_with_stats SET TBLPROPERTIES ('EXTERNAL'='TRUE', 'OBJCAPABILITIES'='EXTREAD,EXTWRITE', 'engine.hive.enabled'='true', 'external.table.purge'='TRUE', 'impala.computeStatsSnapshotIds'='<NUM>', 'impala.events.catalogServiceId'='<NUM>', 'impala.events.catalogVersion'='<NUM>', 'impala.lastComputeStatsTime'='<NUM>', 'numFiles'='<NUM>', 'numRows'='<NUM>', 'table_type'='ICEBERG', 'totalSize'='<NUM>', 'write.delete.mode'='merge-on-read', 'write.format.default'='parquet', 'write.merge.mode'='merge-on-read', 'write.parquet.compression-codec'='snappy', 'write.update.mode'='merge-on-read');
TBLPROPERTIES ('OBJCAPABILITIES'='EXTREAD,EXTWRITE', 'engine.hive.enabled'='true', 'external.table.purge'='TRUE', 'format-version'='$$iceberg_default_format_version$$', 'impala.computeStatsSnapshotIds'='<NUM>', 'impala.lastComputeStatsTime'='<NUM>', 'table_type'='ICEBERG', 'write.delete.mode'='merge-on-read', 'write.format.default'='parquet', 'write.merge.mode'='merge-on-read', 'write.parquet.compression-codec'='snappy', 'write.update.mode'='merge-on-read');
ALTER TABLE show_create_table_test_db.ice_with_stats SET TBLPROPERTIES ('EXTERNAL'='TRUE', 'OBJCAPABILITIES'='EXTREAD,EXTWRITE', 'engine.hive.enabled'='true', 'external.table.purge'='TRUE', 'format-version'='$$iceberg_default_format_version$$', 'impala.computeStatsSnapshotIds'='<NUM>', 'impala.events.catalogServiceId'='<NUM>', 'impala.events.catalogVersion'='<NUM>', 'impala.lastComputeStatsTime'='<NUM>', 'numFiles'='<NUM>', 'numRows'='<NUM>', 'table_type'='ICEBERG', 'totalSize'='<NUM>', 'write.delete.mode'='merge-on-read', 'write.format.default'='parquet', 'write.merge.mode'='merge-on-read', 'write.parquet.compression-codec'='snappy', 'write.update.mode'='merge-on-read');
ALTER TABLE show_create_table_test_db.ice_with_stats SET COLUMN STATS i ('numDVs'='1', 'numNulls'='0', 'numTrues'='-1', 'numFalses'='-1');
ALTER TABLE show_create_table_test_db.ice_with_stats SET COLUMN STATS s ('numDVs'='1', 'numNulls'='0', 'maxSize'='1', 'avgSize'='1', 'numTrues'='-1', 'numFalses'='-1');
====

193
testdata/workloads/functional-query/queries/QueryTest/show-create-table.test vendored
View File

@@ -612,6 +612,7 @@ STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -645,6 +646,7 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_test2 (
STORED AS ICEBERG
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -680,6 +682,7 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_test3 (
STORED AS ICEBERG
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -741,6 +744,7 @@ PARTITIONED BY SPEC (
STORED AS ICEBERG
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -768,10 +772,14 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_test_orc (
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='orc',
'engine.hive.enabled'='true', 'iceberg.catalog'='hadoop.tables',
'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'write.format.default'='orc',
'engine.hive.enabled'='true',
'iceberg.catalog'='hadoop.tables',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
# Default Iceberg table
@@ -785,10 +793,15 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_default_tbl (
)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read',
'write.parquet.compression-codec'='snappy')
TBLPROPERTIES ('write.format.default'='parquet',
'write.parquet.compression-codec'='snappy',
'engine.hive.enabled'='true',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
# Default Iceberg table with ORC format
@@ -805,10 +818,15 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_default_tbl_orc (
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'write.format.default'='orc', 'write.parquet.compression-codec'='snappy',
'engine.hive.enabled'='true', 'external.table.purge'='TRUE', 'table_type'='ICEBERG',
'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read',
'write.delete.mode'='merge-on-read')
'write.format.default'='orc',
'write.parquet.compression-codec'='snappy',
'engine.hive.enabled'='true',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'table_type'='ICEBERG',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
'write.delete.mode'='merge-on-read')
====
---- CREATE_TABLE
# Iceberg table in HiveCatalog
@@ -825,10 +843,16 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_hive_cat_explicit (
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'write.format.default'='parquet', 'iceberg.catalog'='hive.catalog',
'engine.hive.enabled'='true', 'external.table.purge'='TRUE', 'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy', 'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read')
'write.format.default'='parquet',
'iceberg.catalog'='hive.catalog',
'engine.hive.enabled'='true',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_nullable_test (
@@ -855,6 +879,7 @@ STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -880,10 +905,15 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_old_style_partitions (
PARTITIONED BY SPEC (p, d)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
TBLPROPERTIES ('external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_ctas
@@ -899,10 +929,15 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_ctas (
PARTITIONED BY SPEC (BUCKET(5, id))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
TBLPROPERTIES ('external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_ctas_ht
@@ -921,9 +956,14 @@ PARTITIONED BY SPEC (BUCKET(5, id))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'iceberg.catalog'='hadoop.tables', 'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'iceberg.catalog'='hadoop.tables',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_catalogs_hive (i int)
@@ -937,10 +977,15 @@ PARTITIONED BY SPEC (BUCKET(3, i))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_catalogs_hadoop (i int)
@@ -954,9 +999,14 @@ PARTITIONED BY SPEC (BUCKET(3, i))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'iceberg.catalog'='ice_hadoop_cat', 'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'iceberg.catalog'='ice_hadoop_cat',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_void_transform (i int, j int)
@@ -967,10 +1017,15 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_void_transform (i INT NU
PARTITIONED BY SPEC (VOID(i), VOID(j))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
TBLPROPERTIES ('external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='$$iceberg_default_format_version$$',
'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
# Creating V1 tables explicitly should not set 'merge-on-read' write modes if no write mode is
@@ -983,9 +1038,12 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_explicit_v1 (i INT NULL)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'format-version'='1', 'engine.hive.enabled'='true', 'external.table.purge'='TRUE',
'table_type'='ICEBERG',
'write.format.default'='parquet', 'write.parquet.compression-codec'='snappy')
'engine.hive.enabled'='true',
'external.table.purge'='TRUE',
'format-version'='1',
'table_type'='ICEBERG',
'write.format.default'='parquet',
'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should set 'merge-on-read' write modes if no write mode is specified.
@@ -997,10 +1055,15 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_v2 (i INT NULL)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read',
'write.parquet.compression-codec'='snappy')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='2',
'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to any value.
@@ -1013,9 +1076,13 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_v2_explicit_delete (i INT NU
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.delete.mode'='merge-on-read',
'write.parquet.compression-codec'='snappy')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='2',
'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read',
'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to any value.
@@ -1028,9 +1095,13 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_v2_explicit_delete_2 (i INT
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.delete.mode'='copy-on-write',
'write.parquet.compression-codec'='snappy')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='2',
'table_type'='ICEBERG',
'write.delete.mode'='copy-on-write',
'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to any value.
@@ -1043,9 +1114,13 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_v2_explicit_update (i INT NU
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.update.mode'='copy-on-write',
'write.parquet.compression-codec'='snappy')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='2',
'table_type'='ICEBERG',
'write.update.mode'='copy-on-write',
'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to any value.
@@ -1058,9 +1133,13 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_v2_explicit_merge (i INT NUL
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE', 'write.format.default'='parquet',
'engine.hive.enabled'='true', 'table_type'='ICEBERG', 'write.merge.mode'='copy-on-write',
'write.parquet.compression-codec'='snappy')
'external.table.purge'='TRUE',
'write.format.default'='parquet',
'engine.hive.enabled'='true',
'format-version'='2',
'table_type'='ICEBERG',
'write.merge.mode'='copy-on-write',
'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Test create Bucketed Table

18
tests/common/environ.py
View File

@@ -77,6 +77,24 @@ else:
MANAGED_WAREHOUSE_DIR = 'test-warehouse'
EXTERNAL_WAREHOUSE_DIR = 'test-warehouse'
# Set Iceberg's default format version based on version, if <1.4 it's 1, if >=1.4 then 2
impala_iceberg_version = os.environ.get("IMPALA_ICEBERG_VERSION")
ICEBERG_DEFAULT_FORMAT_VERSION = '1'
if impala_iceberg_version:
# Extract the major and minor version numbers
version_parts = impala_iceberg_version.split('.')
if len(version_parts) >= 2:
try:
major = int(version_parts[0])
minor = int(version_parts[1])
# If version is 1.4 or higher, use format version 2
if major > 1 or (major == 1 and minor >= 4):
ICEBERG_DEFAULT_FORMAT_VERSION = '2'
except ValueError:
# If we can't parse the version, default to format version 1
pass
IS_APACHE_HIVE = os.environ.get("USE_APACHE_HIVE", False) == 'true'
# Resolve any symlinks in the path.

10
tests/common/impala_test_suite.py
View File

@@ -46,7 +46,8 @@ from tests.common.base_test_suite import BaseTestSuite
from tests.common.environ import (
HIVE_MAJOR_VERSION,
MANAGED_WAREHOUSE_DIR,
EXTERNAL_WAREHOUSE_DIR)
EXTERNAL_WAREHOUSE_DIR,
ICEBERG_DEFAULT_FORMAT_VERSION)
from tests.common.errors import Timeout
from tests.common.impala_connection import create_connection
from tests.common.impala_service import ImpaladService
@@ -672,7 +673,8 @@ class ImpalaTestSuite(BaseTestSuite):
"INTERNAL_LISTEN_HOST",
"INTERNAL_LISTEN_IP",
"MANAGED_WAREHOUSE_DIR",
"EXTERNAL_WAREHOUSE_DIR"])
"EXTERNAL_WAREHOUSE_DIR",
"ICEBERG_DEFAULT_FORMAT_VERSION"])
repl.update({
'$ERASURECODE_POLICY': os.getenv("ERASURECODE_POLICY", "NONE"),
'$SECONDARY_FILESYSTEM': os.getenv("SECONDARY_FILESYSTEM", ""),
@@ -740,7 +742,9 @@ class ImpalaTestSuite(BaseTestSuite):
.replace('$MANAGED_WAREHOUSE_DIR',
MANAGED_WAREHOUSE_DIR) \
.replace('$EXTERNAL_WAREHOUSE_DIR',
EXTERNAL_WAREHOUSE_DIR)
EXTERNAL_WAREHOUSE_DIR) \
.replace('$ICEBERG_DEFAULT_FORMAT_VERSION',
ICEBERG_DEFAULT_FORMAT_VERSION)
if use_db:
test_section[section_name] = test_section[section_name].replace(
'$DATABASE', use_db)

27
tests/metadata/test_show_create_table.py
View File

@@ -24,7 +24,7 @@ from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIf, SkipIfFS, SkipIfHive2
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.test_file_parser import QueryTestSectionReader, remove_comments
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.environ import HIVE_MAJOR_VERSION, ICEBERG_DEFAULT_FORMAT_VERSION
from tests.util.filesystem_utils import WAREHOUSE
@@ -139,7 +139,7 @@ class TestShowCreateTable(ImpalaTestSuite):
if not test_case.existing_table:
# create table in Impala
self.__exec(self.__replace_warehouse(test_case.create_table_sql))
self.__exec(self.__replace_variables(test_case.create_table_sql))
# execute "SHOW CREATE TABLE ..."
result = self.__exec(test_case.show_create_table_sql)
create_table_result = self.__normalize(result.data[0])
@@ -149,7 +149,7 @@ class TestShowCreateTable(ImpalaTestSuite):
self.__exec(test_case.drop_table_sql)
# check the result matches the expected result
expected_result = self.__normalize(self.__replace_warehouse(self.__replace_uri(
expected_result = self.__normalize(self.__replace_variables(self.__replace_uri(
test_case.expected_result,
self.__get_location_uri(create_table_result))))
self.__compare_result(expected_result, create_table_result)
@@ -176,7 +176,7 @@ class TestShowCreateTable(ImpalaTestSuite):
test_case = ShowCreateTableTestCase(test_section, test_file_name, unique_db_name)
if not test_case.existing_table:
# create table in Impala (support multiple setup statements)
setup_sql = self.__replace_warehouse(test_case.create_table_sql)
setup_sql = self.__replace_variables(test_case.create_table_sql)
for stmt in re.split(r";\s*", setup_sql.strip()):
if not stmt:
continue
@@ -212,7 +212,7 @@ class TestShowCreateTable(ImpalaTestSuite):
self.__exec(test_case.drop_table_sql)
# Build expected statements list and compare per-statement
expected_sql = self.__replace_warehouse(self.__replace_uri(
expected_sql = self.__replace_variables(self.__replace_uri(
test_case.expected_result,
self.__get_location_uri(location_source)
))
@@ -328,18 +328,18 @@ class TestShowCreateTable(ImpalaTestSuite):
"""
s = ' '.join(s.split())
for k in self.FILTER_TBL_PROPERTIES:
kv_regex = "'%s'\s*=\s*'[^\']+'\s*,?" % (k)
kv_regex = r"'%s'\s*=\s*'[^\']+'\s*,?" % (k)
s = re.sub(kv_regex, "", s)
# If we removed the last property, there will be a dangling comma that is not valid
# e.g. 'k1'='v1', ) -> 'k1'='v1')
s = re.sub(",\s*\)", ")", s)
s = re.sub(r",\s*\)", ")", s)
# Need to remove any whitespace after left parens and before right parens
s = re.sub("\(\s+", "(", s)
s = re.sub("\s+\)", ")", s)
s = re.sub(r"\(\s+", "(", s)
s = re.sub(r"\s+\)", ")", s)
# If the only properties were removed, the properties sections may be empty, which
# is not valid
s = re.sub("TBLPROPERTIES\s*\(\s*\)", "", s)
s = re.sub("SERDEPROPERTIES\s*\(\s*\)", "", s)
s = re.sub(r"TBLPROPERTIES\s*\(\s*\)", "", s)
s = re.sub(r"SERDEPROPERTIES\s*\(\s*\)", "", s)
# By removing properties in the middle we might ended up having extra whitespaces,
# let's remove them.
s = ' '.join(s.split())
@@ -357,8 +357,9 @@ class TestShowCreateTable(ImpalaTestSuite):
def __replace_uri(self, s, uri):
return s if uri is None else s.replace("$$location_uri$$", uri)
def __replace_warehouse(self, s):
return s.replace("$$warehouse$$", WAREHOUSE)
def __replace_variables(self, s):
return s.replace("$$warehouse$$", WAREHOUSE)\
.replace("$$iceberg_default_format_version$$", ICEBERG_DEFAULT_FORMAT_VERSION)
# Represents one show-create-table test case. Performs validation of the test sections