mirror of
https://github.com/apache/impala.git
synced 2026-02-01 21:00:29 -05:00
8b83726b8e
Adds erasure coding policy to introspection commands SHOW FILES, SHOW PARTITIONS, SHOW TABLE STATS, and DESCRIBE EXTENDED. Remove `throws IOException` for methods that don't throw. Removes null check for getSd because getStorageDescriptorInfo - which is called right after getTableMetaDataInformation - uses it without checking for null. Adds '$ERASURECODE_POLICY' for runtime test substitution. The test suite replaces this with the current erasure code policy - from HDFS_ERASURECODE_POLICY - or NONE to match expected output. Testing: - ran backend, end-to-end, and custom cluster tests with erasure coding - ran backend, end-to-end, and custom cluster tests with exhaustive strategy Change-Id: Idd95f2d18b3980581788c92993b6d2f53504b5e0 Reviewed-on: http://gerrit.cloudera.org:8080/19268 Reviewed-by: Michael Smith <michael.smith@cloudera.com> Tested-by: Michael Smith <michael.smith@cloudera.com>
164 lines
4.3 KiB
Plaintext
164 lines
4.3 KiB
Plaintext
====
|
|
---- HIVE_QUERY
|
|
# Create a table with Hive and run insert, select, and drop from Impala on it.
|
|
use $DATABASE;
|
|
create table tt (x int) tblproperties (
|
|
'transactional'='true',
|
|
'transactional_properties'='insert_only');
|
|
|
|
insert into tt values (1);
|
|
====
|
|
---- QUERY
|
|
invalidate metadata tt;
|
|
select * from tt
|
|
---- RESULTS
|
|
1
|
|
====
|
|
---- HIVE_QUERY
|
|
# Insert from Hive to test refresh table from Impala in the below test.
|
|
use $DATABASE;
|
|
insert into tt values (2);
|
|
====
|
|
---- QUERY
|
|
refresh tt;
|
|
select * from tt order by x;
|
|
---- RESULTS
|
|
1
|
|
2
|
|
====
|
|
---- QUERY
|
|
# Do a second refresh on an already refreshed ACID table.
|
|
refresh tt;
|
|
select * from tt order by x;
|
|
---- RESULTS
|
|
1
|
|
2
|
|
====
|
|
---- QUERY
|
|
insert overwrite table tt values (3);
|
|
insert into tt values (4);
|
|
====
|
|
---- QUERY
|
|
refresh tt;
|
|
select * from tt order by x;
|
|
---- RESULTS
|
|
3
|
|
4
|
|
====
|
|
---- QUERY
|
|
create table upgraded_table (x int);
|
|
insert into upgraded_table values (1);
|
|
====
|
|
---- HIVE_QUERY
|
|
use $DATABASE;
|
|
# Upgrade to the table to insert only acid when there are already values in it.
|
|
alter table upgraded_table set tblproperties
|
|
('transactional' = 'true', 'transactional_properties' = 'insert_only',
|
|
'EXTERNAL' = 'FALSE');
|
|
====
|
|
---- QUERY
|
|
refresh upgraded_table;
|
|
insert into upgraded_table values (2);
|
|
insert into upgraded_table values (3);
|
|
====
|
|
---- QUERY
|
|
select * from upgraded_table;
|
|
---- RESULTS
|
|
1
|
|
2
|
|
3
|
|
====
|
|
---- QUERY
|
|
drop table tt;
|
|
show tables;
|
|
---- RESULTS
|
|
'upgraded_table'
|
|
====
|
|
---- QUERY
|
|
# After dropping the table I re-create and drop it again to check that all the locks
|
|
# are released properly from HMS.
|
|
create table tt (x int) tblproperties (
|
|
'transactional'='true',
|
|
'transactional_properties'='insert_only');
|
|
====
|
|
---- QUERY
|
|
show tables;
|
|
---- RESULTS
|
|
'upgraded_table'
|
|
'tt'
|
|
====
|
|
---- QUERY
|
|
drop table tt;
|
|
show tables;
|
|
---- RESULTS
|
|
'upgraded_table'
|
|
====
|
|
---- QUERY
|
|
create table full_acid (i int) stored as orc
|
|
tblproperties('transactional'='true');
|
|
show tables;
|
|
---- RESULTS
|
|
'full_acid'
|
|
'upgraded_table'
|
|
====
|
|
---- QUERY
|
|
drop table full_acid;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
show tables;
|
|
---- RESULTS
|
|
'upgraded_table'
|
|
====
|
|
---- QUERY
|
|
# Test reading minor-compacted table.
|
|
show files in functional_orc_def.complextypestbl_minor_compacted;
|
|
---- LABELS
|
|
Path,Size,Partition,EC Policy
|
|
---- RESULTS
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_orc_def.db/complextypestbl_minor_compacted_orc_def/delta_0000001_0000008_v\d+/bucket_00000','.+KB','','$ERASURECODE_POLICY'
|
|
---- TYPES
|
|
STRING,STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
select row__id.originaltransaction, id
|
|
from functional_orc_def.complextypestbl_minor_compacted;
|
|
---- RESULTS
|
|
1,1
|
|
2,2
|
|
3,3
|
|
4,4
|
|
5,5
|
|
6,6
|
|
7,7
|
|
8,8
|
|
---- TYPES
|
|
BIGINT,BIGINT
|
|
====
|
|
---- QUERY
|
|
# Test that Impala sees the compacted delta dir in a minor-compacted insert-only table.
|
|
show files in functional_parquet.insert_only_minor_compacted;
|
|
---- LABELS
|
|
Path,Size,Partition,EC Policy
|
|
---- RESULTS
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000001_0000002_v\d+/000000_0','.+B','','$ERASURECODE_POLICY'
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000003_0000003_0000/000000_0','.+B','','$ERASURECODE_POLICY'
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000004_0000004_0000/000000_0','.+B','','$ERASURECODE_POLICY'
|
|
---- TYPES
|
|
STRING,STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
# Test that Impala sees the compacted delta dir in a first major then minor-compacted insert-only table.
|
|
show files in functional_parquet.insert_only_major_and_minor_compacted;
|
|
---- LABELS
|
|
Path,Size,Partition,EC Policy
|
|
---- RESULTS
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/base_0000002_v\d+/000000_0','.+B','','$ERASURECODE_POLICY'
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000003_0000004_v\d+/000000_0','.+B','','$ERASURECODE_POLICY'
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000005_0000005_0000/000000_0','.+B','','$ERASURECODE_POLICY'
|
|
row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000006_0000006_0000/000000_0','.+B','','$ERASURECODE_POLICY'
|
|
---- TYPES
|
|
STRING,STRING,STRING,STRING
|
|
====
|