mirror of
https://github.com/apache/impala.git
synced 2025-12-25 02:03:09 -05:00
826c8cf9b0
This patch mainly implement the creation/drop of paimon table
through impala.
Supported impala data types:
- BOOLEAN
- TINYINT
- SMALLINT
- INTEGER
- BIGINT
- FLOAT
- DOUBLE
- STRING
- DECIMAL(P,S)
- TIMESTAMP
- CHAR(N)
- VARCHAR(N)
- BINARY
- DATE
Syntax for creating paimon table:
CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [db_name.]table_name
(
[col_name data_type ,...]
[PRIMARY KEY (col1,col2)]
)
[PARTITIONED BY (col_name data_type [COMMENT 'col_comment'], ...)]
STORED AS PAIMON
[LOCATION 'hdfs_path']
[TBLPROPERTIES (
'primary-key'='col1,col2',
'file.format' = 'orc/parquet',
'bucket' = '2',
'bucket-key' = 'col3',
];
Two types of paimon catalogs are supported.
(1) Create table with hive catalog:
CREATE TABLE paimon_hive_cat(userid INT,movieId INT)
STORED AS PAIMON;
(2) Create table with hadoop catalog:
CREATE [EXTERNAL] TABLE paimon_hadoop_cat
STORED AS PAIMON
TBLPROPERTIES('paimon.catalog'='hadoop',
'paimon.catalog_location'='/path/to/paimon_hadoop_catalog',
'paimon.table_identifier'='paimondb.paimontable');
SHOW TABLE STAT/SHOW COLUMN STAT/SHOW PARTITIONS/SHOW FILES
statements are also supported.
TODO:
- Patches pending submission:
- Query support for paimon data files.
- Partition pruning and predicate push down.
- Query support with time travel.
- Query support for paimon meta tables.
- WIP:
- Complex type query support.
- Virtual Column query support for querying
paimon data table.
- Native paimon table scanner, instead of
jni based.
Testing:
- Add unit test for paimon impala type conversion.
- Add unit test for ToSqlTest.java.
- Add unit test for AnalyzeDDLTest.java.
- Update default_file_format TestEnumCase in
be/src/service/query-options-test.cc.
- Update test case in
testdata/workloads/functional-query/queries/QueryTest/set.test.
- Add test cases in metadata/test_show_create_table.py.
- Add custom test test_paimon.py.
Change-Id: I57e77f28151e4a91353ef77050f9f0cd7d9d05ef
Reviewed-on: http://gerrit.cloudera.org:8080/22914
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Riza Suminto <riza.suminto@cloudera.com>
212 lines
5.6 KiB
Plaintext
212 lines
5.6 KiB
Plaintext
====
|
|
---- QUERY
|
|
CREATE TABLE IF NOT EXISTS test_create_managed_paimon_table (
|
|
user_id BIGINT COMMENT 'The user_id field',
|
|
item_id BIGINT COMMENT 'The item_id field',
|
|
behavior STRING COMMENT 'The behavior field'
|
|
)
|
|
STORED AS PAIMON;
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE IF NOT EXISTS test_create_managed_part_paimon_table (
|
|
user_id BIGINT COMMENT 'The user_id field',
|
|
item_id BIGINT COMMENT 'The item_id field',
|
|
behavior STRING COMMENT 'The behavior field'
|
|
)
|
|
PARTITIONED BY (
|
|
dt STRING COMMENT 'The dt field',
|
|
hh STRING COMMENT 'The hh field'
|
|
)
|
|
STORED AS PAIMON;
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE test_create_managed_part_pk_paimon_table (
|
|
user_id BIGINT COMMENT 'The user_id field',
|
|
item_id BIGINT COMMENT 'The item_id field',
|
|
behavior STRING COMMENT 'The behavior field'
|
|
)
|
|
PARTITIONED BY (
|
|
dt STRING COMMENT 'The dt field',
|
|
hh STRING COMMENT 'The hh field'
|
|
)
|
|
STORED AS PAIMON
|
|
TBLPROPERTIES (
|
|
'primary-key'='user_id'
|
|
);
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE test_create_managed_part_pkstmt_paimon_table (
|
|
user_id BIGINT COMMENT 'The user_id field',
|
|
item_id BIGINT COMMENT 'The item_id field',
|
|
behavior STRING COMMENT 'The behavior field',
|
|
PRIMARY KEY(user_id)
|
|
)
|
|
PARTITIONED BY (
|
|
dt STRING COMMENT 'The dt field',
|
|
hh STRING COMMENT 'The hh field'
|
|
)
|
|
STORED AS PAIMON;
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE test_create_managed_bucket_paimon_table (
|
|
user_id BIGINT COMMENT 'The user_id field',
|
|
item_id BIGINT COMMENT 'The item_id field',
|
|
behavior STRING COMMENT 'The behavior field'
|
|
)
|
|
STORED AS PAIMON
|
|
TBLPROPERTIES (
|
|
'bucket' = '4',
|
|
'bucket-key'='behavior'
|
|
);
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE test_create_managed_location_paimon_table (
|
|
user_id BIGINT COMMENT 'The user_id field',
|
|
item_id BIGINT COMMENT 'The item_id field',
|
|
behavior STRING COMMENT 'The behavior field',
|
|
PRIMARY KEY(user_id)
|
|
)
|
|
PARTITIONED BY (
|
|
dt STRING COMMENT 'The dt field',
|
|
hh STRING COMMENT 'The hh field'
|
|
)
|
|
STORED AS PAIMON
|
|
LOCATION 'hdfs:///test-warehouse/test_create_managed_location_paimon_table';
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE TEST_CREATE_MANAGED_UPPERCASE_PAIMON_TABLE (
|
|
`USER_ID` BIGINT COMMENT 'The user_id field',
|
|
`ITEM_ID` BIGINT COMMENT 'The item_id field',
|
|
BEHAVIOR STRING COMMENT 'The behavior field',
|
|
PRIMARY KEY(`USER_ID`)
|
|
)
|
|
PARTITIONED BY (
|
|
`DT` STRING COMMENT 'The dt field',
|
|
`HH` STRING COMMENT 'The hh field'
|
|
)
|
|
STORED AS PAIMON;
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE EXTERNAL TABLE test_create_external_hivecat_nocol_paimon_table
|
|
STORED AS PAIMON
|
|
LOCATION 'hdfs:///test-warehouse/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned';
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE EXTERNAL TABLE test_create_external_hivecat_nocol_paimon_part_table
|
|
STORED AS PAIMON
|
|
LOCATION 'hdfs:///test-warehouse/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned';
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE EXTERNAL TABLE test_create_external_hadoopcat_nocol_paimon_table
|
|
STORED AS PAIMON
|
|
TBLPROPERTIES('paimon.catalog'='hadoop',
|
|
'paimon.catalog_location'='hdfs:///test-warehouse/paimon_test/paimon_catalog/warehouse',
|
|
'paimon.table_identifier'='functional.paimon_non_partitioned');
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE EXTERNAL TABLE test_create_external_hadoopcat_nocol_paimon_part_table
|
|
STORED AS PAIMON
|
|
TBLPROPERTIES('paimon.catalog'='hadoop',
|
|
'paimon.catalog_location'='hdfs:///test-warehouse/paimon_test/paimon_catalog/warehouse',
|
|
'paimon.table_identifier'='functional.paimon_partitioned');
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE impala_paimon_data_types_test (
|
|
col_boolean BOOLEAN,
|
|
col_tinyint TINYINT,
|
|
col_smallint SMALLINT,
|
|
col_int INT,
|
|
col_integer INTEGER,
|
|
col_bigint BIGINT,
|
|
col_float FLOAT,
|
|
col_double DOUBLE,
|
|
col_decimal DECIMAL(10,2),
|
|
col_string STRING,
|
|
col_char CHAR(20),
|
|
col_varchar VARCHAR(100),
|
|
col_timestamp TIMESTAMP,
|
|
col_date DATE,
|
|
col_binary BINARY
|
|
)
|
|
COMMENT 'Table to test all Impala paimon-supported data types'
|
|
STORED AS PAIMON;
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_managed_paimon_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_managed_part_paimon_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_managed_part_pk_paimon_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_managed_part_pkstmt_paimon_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_managed_bucket_paimon_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_managed_location_paimon_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS TEST_CREATE_MANAGED_UPPERCASE_PAIMON_TABLE;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_external_hivecat_nocol_paimon_part_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_external_hadoopcat_nocol_paimon_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS test_create_external_hadoopcat_nocol_paimon_part_table;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
DROP TABLE IF EXISTS impala_paimon_data_types_test;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|