mirror of
https://github.com/apache/impala.git
synced 2026-01-05 03:01:02 -05:00
e08ffde009
Small buffers introduced an issue that is exacerbated by the large fanout. A stream can only be appended to forever once it has grabbed the initial io sized buffer. With small buffers, we don't grab that at the beginning anymore and, before this patch, it is grabbed when the stream first needs it. This means when one stream needs it, another stream could have already grabbed it (meaning this stream is pinned with multiple buffers). This patch has all the streams grab an IO buffer as soon as the first stream needs an io buffer. This guarantees that all streams get 1 before any get 2. Change-Id: I1be1219fc5f1fa3ceedd4d5e76ae056c8bb8ff3d
162 lines
3.1 KiB
Plaintext
162 lines
3.1 KiB
Plaintext
====
|
|
---- QUERY
|
|
set num_nodes=1;
|
|
set max_block_mgr_memory=25m;
|
|
select l_orderkey, count(*)
|
|
from lineitem
|
|
group by 1
|
|
order by 1 limit 10;
|
|
---- RESULTS
|
|
1,6
|
|
2,1
|
|
3,6
|
|
4,1
|
|
5,3
|
|
6,1
|
|
7,7
|
|
32,6
|
|
33,4
|
|
34,3
|
|
---- TYPES
|
|
BIGINT, BIGINT
|
|
====
|
|
---- QUERY
|
|
# Test query with string grouping column and string agg columns
|
|
set num_nodes=1;
|
|
set max_block_mgr_memory=30m;
|
|
select l_returnflag, l_orderkey, avg(l_tax), min(l_shipmode)
|
|
from lineitem
|
|
group by 1,2
|
|
order by 1,2 limit 3;
|
|
---- RESULTS
|
|
'A',3,0.05,'RAIL'
|
|
'A',5,0.03,'AIR'
|
|
'A',6,0.03,'TRUCK'
|
|
---- TYPES
|
|
STRING, BIGINT, DECIMAL, STRING
|
|
====
|
|
---- QUERY
|
|
# TODO: the distributed agg tests have a high likely of becoming flaking. The memory is
|
|
# split between the local and merge agg in an uncontrolled way which can lead to
|
|
# pathological cases causing the queries to fail with OOM. These cases are expected
|
|
# to be unlikely with reasonable limits but we are explicitly trying to pick very low
|
|
# limits. (IMPALA-1305)
|
|
set max_block_mgr_memory=45m;
|
|
select l_orderkey, count(*)
|
|
from lineitem
|
|
group by 1
|
|
order by 1 limit 10;
|
|
---- RESULTS
|
|
1,6
|
|
2,1
|
|
3,6
|
|
4,1
|
|
5,3
|
|
6,1
|
|
7,7
|
|
32,6
|
|
33,4
|
|
34,3
|
|
---- TYPES
|
|
BIGINT, BIGINT
|
|
====
|
|
---- QUERY
|
|
# Test query with string intermediate state.
|
|
set num_nodes=0;
|
|
set max_block_mgr_memory=70m;
|
|
select l_orderkey, avg(l_orderkey)
|
|
from lineitem
|
|
group by 1
|
|
order by 1 limit 5;
|
|
---- RESULTS
|
|
1,1
|
|
2,2
|
|
3,3
|
|
4,4
|
|
5,5
|
|
---- TYPES
|
|
BIGINT, DOUBLE
|
|
====
|
|
---- QUERY
|
|
# Test query with string grouping column
|
|
set num_nodes=0;
|
|
set max_block_mgr_memory=160m;
|
|
select l_comment, count(*)
|
|
from lineitem
|
|
group by 1
|
|
order by count(*) desc limit 5;
|
|
---- RESULTS
|
|
' furiously',943
|
|
' carefully',893
|
|
' carefully ',875
|
|
'carefully ',854
|
|
' furiously ',845
|
|
---- TYPES
|
|
STRING, BIGINT
|
|
====
|
|
---- QUERY
|
|
# Test query with string grouping column and string agg columns
|
|
set num_nodes=0;
|
|
set max_block_mgr_memory=80m;
|
|
select l_returnflag, l_orderkey, avg(l_tax), min(l_shipmode)
|
|
from lineitem
|
|
group by 1,2
|
|
order by 1,2 limit 3;
|
|
---- RESULTS
|
|
'A',3,0.05,'RAIL'
|
|
'A',5,0.03,'AIR'
|
|
'A',6,0.03,'TRUCK'
|
|
---- TYPES
|
|
STRING, BIGINT, DECIMAL, STRING
|
|
====
|
|
---- QUERY
|
|
set num_nodes=0;
|
|
set max_block_mgr_memory=100m;
|
|
select count(l1.l_tax)
|
|
from
|
|
lineitem l1,
|
|
lineitem l2,
|
|
lineitem l3
|
|
where
|
|
l1.l_tax < 0.01 and
|
|
l2.l_tax < 0.04 and
|
|
l1.l_orderkey = l2.l_orderkey and
|
|
l1.l_orderkey = l3.l_orderkey and
|
|
l1.l_comment = l3.l_comment and
|
|
l1.l_shipdate = l3.l_shipdate;
|
|
---- RESULTS
|
|
1846743
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
set num_nodes=0;
|
|
set max_block_mgr_memory=45m;
|
|
select max(t1.total_count), max(t1.l_shipinstruct), max(t1.l_comment) from
|
|
(select l_shipinstruct, l_comment, count(*) over () total_count from lineitem) t1
|
|
---- RESULTS
|
|
6001215,'TAKE BACK RETURN','zzle? slyly final platelets sleep quickly. '
|
|
---- TYPES
|
|
BIGINT, STRING, STRING
|
|
---- QUERY
|
|
# Run this query with very low memory. Since the tables are small, the PA/PHJ should be
|
|
# using buffers much smaller than the io buffer.
|
|
set max_block_mgr_memory=10m;
|
|
select a.int_col, count(*)
|
|
from functional.alltypessmall a, functional.alltypessmall b, functional.alltypessmall c
|
|
where a.id = b.id and b.id = c.id group by a.int_col;
|
|
---- RESULTS
|
|
0,12
|
|
1,12
|
|
2,12
|
|
3,12
|
|
4,12
|
|
5,8
|
|
6,8
|
|
7,8
|
|
8,8
|
|
9,8
|
|
---- TYPES
|
|
INT, BIGINT
|
|
====
|