mirror of
https://github.com/apache/impala.git
synced 2026-01-08 12:02:54 -05:00
0eaab69fff
Tuple pointers in the generated row batches may not be initialized if a tuple has byte size 0. There are some codes which compare these uninitialized pointers against nullptr so having them uninitialized may return wrong (and non-deterministic) results, e.g.: impala::TupleIsNullPredicate::GetBooleanVal() The following query produces non-deterministic results currently: SELECT count(v.x) FROM functional.alltypestiny t3 LEFT OUTER JOIN ( SELECT true AS x FROM functional.alltypestiny t1 LEFT OUTER JOIN functional.alltypestiny t2 ON (true)) v ON (v.x = t3.bool_col) WHERE t3.bool_col = true; The alltypestiny table has 8 records, 4 records of them has the true boolean value for bool_col. Therefore, the above query is a fancy way of saying "8 * 8 * 4", i.e. it should return 256. The solution is that scanners initialize tuple pointers to a non-null value when there are no materialized slots. This non-null value is provided by the new static member Tuple::POISON. I extended QueryTest/scanners.test with the above query. This test executes the query against all table formats. This change has the biggest performance impact on count(*) queries on large kudu tables. For my quick benchmark I copied tpch_kudu.lineitem and doubled its data. The resulting table has 12,002,430 rows. Without this patch 'select count(*) from biglineitem' runs for ~0.12s. With the patch applied, the overhead is around a dozens of ms. I measured the query on my desktop PC using a relase build of Impala. On debug builds, the execution time of the patched version is around 160% of the original version. Without this patch: +--------------+--------+----------+----------+--------+------------+-----------+---------------+---------------------+ | Operator | #Hosts | Avg Time | Max Time | #Rows | Est. #Rows | Peak Mem | Est. Peak Mem | Detail | +--------------+--------+----------+----------+--------+------------+-----------+---------------+---------------------+ | 03:AGGREGATE | 1 | 127.50us | 127.50us | 1 | 1 | 28.00 KB | 10.00 MB | FINALIZE | | 02:EXCHANGE | 1 | 22.32ms | 22.32ms | 3 | 1 | 0 B | 0 B | UNPARTITIONED | | 01:AGGREGATE | 3 | 1.78ms | 1.89ms | 3 | 1 | 16.00 KB | 10.00 MB | | | 00:SCAN KUDU | 3 | 8.00ms | 8.28ms | 12.00M | -1 | 512.00 KB | 0 B | default.biglineitem | +--------------+--------+----------+----------+--------+------------+-----------+---------------+---------------------+ With this patch: +--------------+--------+----------+----------+--------+------------+-----------+---------------+---------------------+ | Operator | #Hosts | Avg Time | Max Time | #Rows | Est. #Rows | Peak Mem | Est. Peak Mem | Detail | +--------------+--------+----------+----------+--------+------------+-----------+---------------+---------------------+ | 03:AGGREGATE | 1 | 129.01us | 129.01us | 1 | 1 | 28.00 KB | 10.00 MB | FINALIZE | | 02:EXCHANGE | 1 | 33.00ms | 33.00ms | 3 | 1 | 0 B | 0 B | UNPARTITIONED | | 01:AGGREGATE | 3 | 1.99ms | 2.13ms | 3 | 1 | 16.00 KB | 10.00 MB | | | 00:SCAN KUDU | 3 | 13.13ms | 13.97ms | 12.00M | -1 | 512.00 KB | 0 B | default.biglineitem | +--------------+--------+----------+----------+--------+------------+-----------+---------------+---------------------+ Change-Id: I298122aaaa7e62eb5971508e0698e189519755de Reviewed-on: http://gerrit.cloudera.org:8080/9239 Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com> Tested-by: Impala Public Jenkins
113 lines
2.9 KiB
Plaintext
113 lines
2.9 KiB
Plaintext
====
|
|
---- QUERY
|
|
# This query will do a full table scan, doing a simple aggregation on all cols with
|
|
# a simple predicate
|
|
select count(*),
|
|
sum(id), count(bool_col), sum(tinyint_col), sum(smallint_col),
|
|
sum(int_col), sum(bigint_col), max(float_col), max(double_col),
|
|
max(date_string_col), max(string_col), max(timestamp_col)
|
|
from alltypesagg
|
|
where id % 2 = 0 and day is not null
|
|
---- RESULTS
|
|
5000,24995000,5000,20000,245000,2495000,24950000,1097.800048828125,10079.8,'01/10/10','998',2010-01-10 18:00:55.300000000
|
|
---- TYPES
|
|
BIGINT, BIGINT, BIGINT, BIGINT, BIGINT, BIGINT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIMESTAMP
|
|
====
|
|
---- QUERY
|
|
# This query will do a join, projecting one string col from each table.
|
|
# This is interesting because the join contains string cols which causes the scanners
|
|
# to do different memory handling.
|
|
select sum(t1.id), sum(t1.int_col),max(t1.date_string_col), max(t2.string_col)
|
|
from alltypesagg t1
|
|
inner join alltypesagg t2
|
|
on t1.id = t2.id and t1.day is not null and t2.day is not null
|
|
---- RESULTS
|
|
49995000,4995000,'01/10/10','999'
|
|
---- TYPES
|
|
BIGINT, BIGINT, STRING, STRING
|
|
====
|
|
---- QUERY
|
|
# This query does a top-n on non-string cols. This is different because without
|
|
# string cols, scanners will handle io buffers differently. They don't need to
|
|
# be passed up the execution tree.
|
|
select id, bool_col, int_col
|
|
from alltypesagg where day is not null
|
|
order by 1 desc, 2 desc, 3 desc
|
|
limit 10
|
|
---- RESULTS
|
|
9999,false,999
|
|
9998,true,998
|
|
9997,false,997
|
|
9996,true,996
|
|
9995,false,995
|
|
9994,true,994
|
|
9993,false,993
|
|
9992,true,992
|
|
9991,false,991
|
|
9990,true,990
|
|
---- TYPES
|
|
INT, BOOLEAN, INT
|
|
====
|
|
---- QUERY
|
|
# The next sequence of queries is a regression test for IMPALA-4153
|
|
# verifying the retrieval of empty and NULL string columns
|
|
select count(*)
|
|
from nulltable
|
|
---- RESULTS
|
|
1
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
select count(*)
|
|
from nulltable where b = ''
|
|
---- RESULTS
|
|
1
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
select a,b
|
|
from nulltable where b = ''
|
|
---- RESULTS
|
|
'a',''
|
|
---- TYPES
|
|
STRING, STRING
|
|
====
|
|
---- QUERY
|
|
# The following 3 tests are regression tests for IMPALA-6187. Make sure the conjuncts are
|
|
# evaluated when there are no materialized slots or only partition columns are accessed.
|
|
select count(*) from alltypes where rand() * 10 >= 0.0;
|
|
---- RESULTS
|
|
7300
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
select count(*) from alltypes where rand() * 10 < 0.0;
|
|
---- RESULTS
|
|
0
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
# 'year' and 'month' are partition columns.
|
|
select count(*) from alltypes where rand() - year > month;
|
|
---- RESULTS
|
|
0
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
# IMPALA-6258: Uninitialized tuple pointers in row batch for empty rows
|
|
# The following query was non-deterministic because of this bug
|
|
select count(v.x) from alltypestiny t3 left outer join (
|
|
select true as x from alltypestiny t1 left outer join
|
|
alltypestiny t2 on (true)) v
|
|
on (v.x = t3.bool_col) where t3.bool_col = true
|
|
---- RESULTS
|
|
256
|
|
---- TYPES
|
|
BIGINT
|
|
====
|