mirror of
https://github.com/apache/impala.git
synced 2026-02-03 00:00:40 -05:00
fb6d96e001
This patch mainly realizes the querying of iceberg table through impala,
we can use the following sql to create an external iceberg table:
CREATE EXTERNAL TABLE default.iceberg_test (
level string,
event_time timestamp,
message string,
)
STORED AS ICEBERG
LOCATION 'hdfs://xxx'
TBLPROPERTIES ('iceberg_file_format'='parquet');
Or just including table name and location like this:
CREATE EXTERNAL TABLE default.iceberg_test
STORED AS ICEBERG
LOCATION 'hdfs://xxx'
TBLPROPERTIES ('iceberg_file_format'='parquet');
'iceberg_file_format' is the file format in iceberg, currently only
support PARQUET, other format would be supported in the future. And
if you don't specify this property in your SQL, default file format
is PARQUET.
We achieved this function by treating the iceberg table as normal
unpartitioned hdfs table. When querying iceberg table, we pushdown
partition column predicates to iceberg to decide which data files
need to be scanned, and then transfer this information to BE to
do the real scan operation.
Testing:
- Unit test for Iceberg in FileMetadataLoaderTest
- Create table tests in functional_schema_template.sql
- Iceberg table query test in test_scanners.py
Change-Id: I856cfee4f3397d1a89cf17650e8d4fbfe1f2b006
Reviewed-on: http://gerrit.cloudera.org:8080/16143
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
130 lines
2.5 KiB
Plaintext
130 lines
2.5 KiB
Plaintext
====
|
|
---- QUERY
|
|
CREATE TABLE iceberg_test1(
|
|
level STRING,
|
|
event_time TIMESTAMP,
|
|
register_time DATE,
|
|
message STRING,
|
|
price DECIMAL(8,1),
|
|
map_test MAP <STRING, array <STRING>>,
|
|
struct_test STRUCT <f1: BIGINT, f2: BIGINT>
|
|
)
|
|
PARTITION BY SPEC
|
|
(
|
|
level IDENTITY,
|
|
event_time IDENTITY,
|
|
event_time HOUR,
|
|
register_time DAY
|
|
)
|
|
STORED AS ICEBERG;
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
DESCRIBE iceberg_test1;
|
|
---- RESULTS
|
|
'level','string',''
|
|
'event_time','timestamp',''
|
|
'register_time','date',''
|
|
'message','string',''
|
|
'price','decimal(8,1)',''
|
|
'map_test','map<string,array<string>>',''
|
|
'struct_test','struct<\n f1:bigint,\n f2:bigint\n>',''
|
|
---- TYPES
|
|
STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
SHOW PARTITIONS iceberg_test1;
|
|
---- RESULTS
|
|
0,1,1000,'level','IDENTITY'
|
|
0,2,1001,'event_time','IDENTITY'
|
|
0,2,1002,'event_time_hour','HOUR'
|
|
0,3,1003,'register_time_day','DAY'
|
|
---- TYPES
|
|
BIGINT,BIGINT,BIGINT,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
DROP TABLE iceberg_test1;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE iceberg_test2(
|
|
level STRING
|
|
)
|
|
STORED AS ICEBERG;
|
|
DESCRIBE iceberg_test2;
|
|
---- RESULTS
|
|
'level','string',''
|
|
---- TYPES
|
|
STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
CREATE EXTERNAL TABLE iceberg_test_external(
|
|
level STRING
|
|
)
|
|
STORED AS ICEBERG;
|
|
---- RESULTS
|
|
'Location is necessary for external iceberg table.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE iceberg_test4(
|
|
level STRING
|
|
)
|
|
PARTITION BY SPEC
|
|
(
|
|
level IDENTITY
|
|
)
|
|
STORED AS ICEBERG
|
|
LOCATION '/$DATABASE.iceberg_test_with_location';
|
|
CREATE EXTERNAL TABLE iceberg_test_external(
|
|
level STRING
|
|
)
|
|
STORED AS ICEBERG
|
|
LOCATION '/$DATABASE.iceberg_test_with_location';
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
DESCRIBE iceberg_test_external;
|
|
---- RESULTS
|
|
'level','string',''
|
|
---- TYPES
|
|
STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
SHOW PARTITIONS iceberg_test_external;
|
|
---- RESULTS
|
|
0,1,1000,'level','IDENTITY'
|
|
---- TYPES
|
|
BIGINT,BIGINT,BIGINT,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
CREATE EXTERNAL TABLE iceberg_test_external_empty_column
|
|
STORED AS ICEBERG
|
|
LOCATION '/$DATABASE.iceberg_test_with_location';
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
DESCRIBE iceberg_test_external_empty_column;
|
|
---- RESULTS
|
|
'level','string',''
|
|
---- TYPES
|
|
STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
SHOW PARTITIONS iceberg_test_external_empty_column;
|
|
---- RESULTS
|
|
0,1,1000,'level','IDENTITY'
|
|
---- TYPES
|
|
BIGINT,BIGINT,BIGINT,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
DROP TABLE iceberg_test4;
|
|
DROP TABLE iceberg_test_external;
|
|
DROP TABLE iceberg_test_external_empty_column;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|