mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
3d22c7fe05
HiveCatalog does not include format-version for Iceberg tables in the table's parameters, therefore the output of SHOW CREATE TABLE may not replicate the original table. This patch makes sure to add it to both the SHOW CREATE TABLE and DESCRIBE FORMATTED/EXTENDED output. Additionally, adds ICEBERG_DEFAULT_FORMAT_VERSION variable to E2E tests, deducting from IMPALA_ICEBERG_VERSION environment variable. If Iceberg version is at least 1.4, default format-version is 2, before 1.4 it's 1. This way tests can work with multiple Iceberg versions. Testing: * updated show-create-table.test and show-create-table-with-stats.test for Iceberg tables * added format-version checks to multiple DESCRIBE FORMATTED tests Change-Id: I991edf408b24fa73e8a8abe64ac24929aeb8e2f8 Reviewed-on: http://gerrit.cloudera.org:8080/23514 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
146 lines
3.6 KiB
Plaintext
146 lines
3.6 KiB
Plaintext
====
|
|
---- QUERY
|
|
select * from missing_files_nopart
|
|
---- CATCH
|
|
unavailable files
|
|
====
|
|
---- QUERY
|
|
select * from missing_files_part
|
|
---- CATCH
|
|
unavailable files
|
|
====
|
|
---- QUERY
|
|
select * from missing_files_part
|
|
where p=2
|
|
---- CATCH
|
|
Cannot find file
|
|
====
|
|
---- QUERY
|
|
# Time travel queries should work
|
|
select * from missing_files_nopart
|
|
for system_version as of $NOPART_FIRST_SNAPSHOT
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
select * from missing_files_part
|
|
for system_version as of $PART_FIRST_SNAPSHOT
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
====
|
|
---- QUERY
|
|
# We can still query partitions without missing files
|
|
select * from missing_files_part
|
|
where p=1
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
describe missing_files_part
|
|
---- RESULTS
|
|
'i','int','','true'
|
|
'p','int','','true'
|
|
---- TYPES
|
|
STRING, STRING, STRING, STRING
|
|
====
|
|
---- QUERY
|
|
describe formatted missing_files_part
|
|
---- RESULTS: VERIFY_IS_SUBSET
|
|
'# col_name ','data_type ','comment '
|
|
'','NULL','NULL'
|
|
'i','int','NULL'
|
|
'p','int','NULL'
|
|
'','NULL','NULL'
|
|
'# Partition Transform Information','NULL','NULL'
|
|
'# col_name ','transform_type ','NULL'
|
|
'','NULL','NULL'
|
|
'p','IDENTITY','NULL'
|
|
'','NULL','NULL'
|
|
'# Detailed Table Information','NULL','NULL'
|
|
'OwnerType: ','USER ','NULL'
|
|
'Table Type: ','EXTERNAL_TABLE ','NULL'
|
|
'Table Parameters:','NULL','NULL'
|
|
'','EXTERNAL ','TRUE '
|
|
'','OBJCAPABILITIES ','EXTREAD,EXTWRITE '
|
|
'','default-partition-spec','{\\"spec-id\\":0,\\"fields\\":[{\\"name\\":\\"p\\",\\"transform\\":\\"identity\\",\\"source-id\\":2,\\"field-id\\":1000}]}'
|
|
'','engine.hive.enabled ','true '
|
|
'','external.table.purge','TRUE '
|
|
'','format-version ','2 '
|
|
'','numFiles ','2 '
|
|
'','numRows ','2 '
|
|
'','snapshot-count ','2 '
|
|
'','storage_handler ','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
|
|
'','table_type ','ICEBERG '
|
|
'','write.delete.mode ','merge-on-read '
|
|
'','write.format.default','parquet '
|
|
'','write.merge.mode ','merge-on-read '
|
|
'','write.update.mode ','merge-on-read '
|
|
'','NULL','NULL'
|
|
'# Storage Information','NULL','NULL'
|
|
'SerDe Library: ','org.apache.iceberg.mr.hive.HiveIcebergSerDe','NULL'
|
|
'InputFormat: ','org.apache.iceberg.mr.hive.HiveIcebergInputFormat','NULL'
|
|
'OutputFormat: ','org.apache.iceberg.mr.hive.HiveIcebergOutputFormat','NULL'
|
|
'Compressed: ','No ','NULL'
|
|
'Sort Columns: ','[] ','NULL'
|
|
'','NULL','NULL'
|
|
'# Constraints','NULL','NULL'
|
|
---- TYPES
|
|
STRING, STRING, STRING
|
|
====
|
|
---- QUERY
|
|
# Metadata queries work
|
|
SELECT snapshot_id, operation from $DATABASE.missing_files_part.snapshots;
|
|
---- RESULTS
|
|
regex:\d+,'append'
|
|
regex:\d+,'append'
|
|
---- TYPES
|
|
BIGINT,STRING
|
|
====
|
|
---- QUERY
|
|
ALTER TABLE missing_files_part DROP PARTITION (p=2);
|
|
SELECT * FROM missing_files_part;
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
ALTER TABLE missing_files_nopart EXECUTE ROLLBACK($NOPART_FIRST_SNAPSHOT);
|
|
SELECT * FROM missing_files_nopart;
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
ALTER TABLE missing_files_part EXECUTE ROLLBACK($PART_FIRST_SNAPSHOT);
|
|
SELECT * FROM missing_files_part;
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
INSERT INTO missing_files_nopart VALUES (3,3);
|
|
---- DML_RESULTS: missing_files_nopart
|
|
1,1
|
|
3,3
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
INSERT INTO missing_files_part VALUES (3,3);
|
|
---- DML_RESULTS: missing_files_part
|
|
1,1
|
|
3,3
|
|
---- TYPES
|
|
INT,INT
|
|
====
|