mirror of
https://github.com/apache/impala.git
synced 2026-01-09 15:00:11 -05:00
6360657cb4
This commit adds INSERT support for insert-only ACID tables. The Frontend opens a transaction for INSERT statements when the target table is transactional. It also allocates a write ID for the target table. The Frontend aborts the transaction if an error occurs during analysis/planning. The Backend gets the transaction id and the write id in TFinalizeParams. The write id is also set the for the HDFS table sinks. The sinks write the files at their final destination which is an ACID base or delta directory. There is no need for finalization of transactional INSERTS. When the sinks finished with writing the data, the Coordinator invokes updateCatalog() on catalogd which also commits the transaction if everything went well, otherwise the Coordinator aborts the transaction. Testing: * added new tables during dataload * added acid-insert.test file with INSERT statements against the new tables * test insertions between ACID and non-ACID tables * test error scenarios via debug actions * added integration test with Hive to test_hms_integration.py. The test inserts data with Impala and reads with Hive. (These integration tests only run with exhaustive exploration strategy) TODO in following commits: * add locks and heartbeats (without heartbeats long-running transactions might be aborted by HMS) * implement TRUNCATE * CTAS creates files in the 'root' directory of the table/partition. It is handled correctly during SELECT, but would be better to create a base directory from the beginning. Hive creates a delta directory for CTAS. Change-Id: Id6c36fa6902676f06b4e38730f737becfc7c06ad Reviewed-on: http://gerrit.cloudera.org:8080/13559 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
86 lines
1.8 KiB
Plaintext
86 lines
1.8 KiB
Plaintext
====
|
|
---- QUERY
|
|
create table insertonly_acid (i int)
|
|
tblproperties('transactional'='true', 'transactional_properties'='insert_only');
|
|
insert into insertonly_acid values (1), (2);
|
|
select * from insertonly_acid;
|
|
---- RESULTS
|
|
1
|
|
2
|
|
---- TYPES
|
|
INT
|
|
====
|
|
---- QUERY
|
|
set DEBUG_ACTION="FIS_FAIL_HDFS_TABLE_SINK_FLUSH_FINAL:FAIL@1.0";
|
|
insert into insertonly_acid values (42);
|
|
---- CATCH
|
|
Query aborted:Debug Action: FIS_FAIL_HDFS_TABLE_SINK_FLUSH_FINAL:FAIL@1.0
|
|
====
|
|
---- QUERY
|
|
select * from insertonly_acid;
|
|
---- RESULTS
|
|
1
|
|
2
|
|
---- TYPES
|
|
INT
|
|
====
|
|
---- QUERY
|
|
set DEBUG_ACTION="CLIENT_REQUEST_UPDATE_CATALOG:FAIL@1.0";
|
|
insert into insertonly_acid values (42);
|
|
---- CATCH
|
|
Query aborted:Debug Action: CLIENT_REQUEST_UPDATE_CATALOG:FAIL@1.0
|
|
====
|
|
---- QUERY
|
|
select * from insertonly_acid;
|
|
---- RESULTS
|
|
1
|
|
2
|
|
---- TYPES
|
|
INT
|
|
====
|
|
---- QUERY
|
|
create table part (n int)
|
|
partitioned by (p int) tblproperties (
|
|
'transactional'='true',
|
|
'transactional_properties'='insert_only');
|
|
insert into table part (p, n) select 1, 10;
|
|
insert into table part (p, n) select 2, 20;
|
|
select p, n from part;
|
|
---- RESULTS
|
|
1,10
|
|
2,20
|
|
----
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
# Dynamic partition insert into existing and non-existing partitions.
|
|
set DEBUG_ACTION="FIS_FAIL_HDFS_TABLE_SINK_FLUSH_FINAL:FAIL@1.0";
|
|
insert into part (p, n) select cast(i + 1 as INT), 11 from insertonly_acid;
|
|
---- CATCH
|
|
Query aborted:Debug Action: FIS_FAIL_HDFS_TABLE_SINK_FLUSH_FINAL:FAIL@1.0
|
|
====
|
|
---- QUERY
|
|
select p, n from part;
|
|
---- RESULTS
|
|
1,10
|
|
2,20
|
|
---- TYPES
|
|
INT,INT
|
|
====
|
|
---- QUERY
|
|
# Dynamic partition insert into existing and non-existing partitions.
|
|
set DEBUG_ACTION="CLIENT_REQUEST_UPDATE_CATALOG:FAIL@1.0";
|
|
insert into part (p, n) select cast(i + 1 as INT), 11 from insertonly_acid;
|
|
---- CATCH
|
|
Query aborted:Debug Action: CLIENT_REQUEST_UPDATE_CATALOG:FAIL@1.0
|
|
====
|
|
---- QUERY
|
|
select p, n from part;
|
|
---- RESULTS
|
|
1,10
|
|
2,20
|
|
---- TYPES
|
|
INT,INT
|
|
====
|