mirror of
https://github.com/apache/impala.git
synced 2026-02-01 12:00:22 -05:00
c579212251
This patch adds support for DELETE statements on unpartitioned Iceberg tables. Impala uses the 'merge-on-read' mode with position delete files. The patch reuses the existing IcebergPositionDeleteTable as the target table of the DELETE statements, because this table already has the same schema as position delete files, even with correct Iceberg field IDs. The patch basically rewrites DELETE statements to INSERT statements, e.g.: from: DELETE FROM ice_t WHERE id = 42; to: INSERT INTO ice_t-POSITION-DELETE SELECT INPUT__FILE__NAME, FILE__POSITION FROM ice_t WHERE id = 42; Position delete files need to be ordered by (file_path, pos), so we add an extra SORT node before the table sink operator. In the backend the patch adds a new table sink operator, the IcebergDeleteSink. It writes the incoming rows (file_path, position) to delete files. It reuses a lot of code from HdfsTableSink, so this patch moves the common code to the new common base class: TableSinkBase. The coordinator then collects the written delete files and invokes UpdateCatalog to finalize the DELETE statement. The Catalog then uses Iceberg APIs to create a new snapshot with the created delete files. It also validates that there was no conflicting data files written since the operation started. Testing: * added planer test * e2e tests * interop test between Impala and Hive Change-Id: Ic933b2295abe54b46d2a736961219988ff42915b Reviewed-on: http://gerrit.cloudera.org:8080/19776 Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Reviewed-by: Gabor Kaszab <gaborkaszab@cloudera.com>
118 lines
2.1 KiB
Plaintext
118 lines
2.1 KiB
Plaintext
====
|
|
---- QUERY
|
|
CREATE TABLE ice_delete (i int, s string)
|
|
STORED BY ICEBERG
|
|
TBLPROPERTIES ('format-version'='2');
|
|
====
|
|
---- QUERY
|
|
# Delete from empty table is no-op.
|
|
DELETE FROM ice_delete where i = 1;
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
INSERT INTO ice_delete VALUES(1, 'one'), (2, 'two'), (3, 'three');
|
|
DELETE FROM ice_delete WHERE i = 2;
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
1,'one'
|
|
3,'three'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
SELECT count(*) FROM ice_delete;
|
|
---- RESULTS
|
|
2
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
INSERT INTO ice_delete VALUES (4, 'four'), (5, 'five'), (6, 'six');
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
1,'one'
|
|
3,'three'
|
|
4,'four'
|
|
5,'five'
|
|
6,'six'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
DELETE FROM ice_delete WHERE s like 'f%' and i > 4;
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
1,'one'
|
|
3,'three'
|
|
4,'four'
|
|
6,'six'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
INSERT INTO ice_delete VALUES (7, 'seven'), (8, 'eight');
|
|
DELETE FROM ice_delete WHERE i in (SELECT i FROM ice_delete where s in ('one', 'three'));
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
4,'four'
|
|
6,'six'
|
|
7,'seven'
|
|
8,'eight'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
DELETE FROM ice_delete WHERE FILE__POSITION = 0;
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
6,'six'
|
|
8,'eight'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
INSERT INTO ice_delete VALUES (9, 'nine'), (10, 'ten');
|
|
DELETE FROM ice_delete WHERE s = (SELECT min(s) FROM ice_delete);
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
6,'six'
|
|
9,'nine'
|
|
10,'ten'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
DELETE FROM ice_delete WHERE i < 10;
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
10,'ten'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
DELETE FROM ice_delete WHERE i = 1000;
|
|
SELECT * FROM ice_delete;
|
|
---- RESULTS
|
|
10,'ten'
|
|
---- TYPES
|
|
INT,STRING
|
|
====
|
|
---- QUERY
|
|
CREATE TABLE ice_lineitem STORED BY ICEBERG
|
|
TBLPROPERTIES ('format-version'='2')
|
|
AS SELECT * FROM tpch_parquet.lineitem;
|
|
DELETE FROM ice_lineitem WHERE l_orderkey % 5 = 1;
|
|
SELECT count(*) FROM ice_lineitem;
|
|
---- RESULTS
|
|
4799418
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
SELECT * FROM ice_lineitem WHERE l_orderkey % 5 = 1;
|
|
---- RESULTS
|
|
====
|