mirror of
https://github.com/apache/impala.git
synced 2026-02-03 09:00:39 -05:00
e0c0fe988a
the coordinator This patch fixes a bug where if an insert or CTAS query has no fragments scheduled on the coordinator and a mem limit is to be enforced on the query (either through query option or automatically through estimates) then the same limit is also applied to the coordinator backend even though it does not execute anything. Highlights: - coord_backend_mem_to_admit_/mem_limit will always refer to the memory to admit/limit for the coordinator regardless of which fragments are scheduled on it. - There will always be a BackendExecParams added for the coordinator because coordinator always spawns a QueryState object with a mem_tracker for tracking runtime filter mem and the result set cache. For the case where this BackendExecParams is empty (no instances scheduled) it would ensure that some minimal amount of memory is accounted for by the admission controller and the right mem limit is applied to the QueryState spawned by the coordinator - added changes to Coordinator and Coordinator::BackendState classes to handle an empty BackendExecParams object Testing: The following cases need to be tested where the kind of fragments schduled on the coordinator backend are: 1. Coordinator fragment + other exec fragments 2. Coordinator fragment only 3. other exec fragments only (eg. insert into values OR insert into select 1) 4. No fragments, but coordinator still creates a QueryState Case 1 is covered by tests working with non-dedicated coordinators. Rest are covered by test_dedicated_coordinator_planner_estimates and test_sanity_checks_dedicated_coordinator in test_admission_controller.py Change-Id: Ic4fba02bb7b20553a20634f8c5989d43ba2c0721 Reviewed-on: http://gerrit.cloudera.org:8080/14058 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
97 lines
3.2 KiB
Plaintext
97 lines
3.2 KiB
Plaintext
====
|
|
---- QUERY
|
|
# CTAS
|
|
create table test as select id from functional.alltypes where id > 1
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=16MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=100MB.*
|
|
row_regex: .*Cluster Memory Admitted: 132.00 MB.*
|
|
====
|
|
---- QUERY
|
|
# Truncate table to run the following inserts.
|
|
truncate table test
|
|
====
|
|
---- QUERY
|
|
# Small insert(i.e. values list, runs on coordinator only).
|
|
insert into test values (1), (2), (3)
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=10MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=100MB.*
|
|
row_regex: .*Cluster Memory Admitted: 10.00 MB.*
|
|
====
|
|
---- QUERY
|
|
# Large insert where it doesn't run on the coordinator.
|
|
insert into test select id from functional.alltypes where id > 3
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=16MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=100MB.*
|
|
row_regex: .*Cluster Memory Admitted: 132.00 MB.*
|
|
====
|
|
---- QUERY
|
|
# SELECT with merging exchange (i.e. order by).
|
|
select * from functional.alltypes order by int_col;
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=28MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=100MB.*
|
|
row_regex: .*Cluster Memory Admitted: 157.47 MB.*
|
|
====
|
|
---- QUERY
|
|
# SELECT with non-merging exchange.
|
|
select * from functional.alltypes;
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=16MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=100MB.*
|
|
row_regex: .*Cluster Memory Admitted: 133.47 MB.*
|
|
====
|
|
---- QUERY
|
|
# SELECT with a non-grouping aggregate in the coordinator fragment.
|
|
select avg(int_col) from functional.alltypes;
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=36MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=110MB.*
|
|
row_regex: .*Cluster Memory Admitted: 182.05 MB.*
|
|
====
|
|
---- QUERY
|
|
# SELECT with num_nodes=1 and a complex plan in the coordinator.
|
|
set num_nodes=1;
|
|
select
|
|
l_returnflag,
|
|
l_linestatus,
|
|
sum(l_quantity) as sum_qty,
|
|
sum(l_extendedprice) as sum_base_price,
|
|
sum(l_extendedprice * (1 - l_discount)) as sum_disc_price,
|
|
sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge,
|
|
avg(l_quantity) as avg_qty,
|
|
avg(l_extendedprice) as avg_price,
|
|
avg(l_discount) as avg_disc,
|
|
count(*) as count_order
|
|
from
|
|
tpch.lineitem
|
|
where
|
|
l_shipdate <= '1998-09-02'
|
|
group by
|
|
l_returnflag,
|
|
l_linestatus
|
|
order by
|
|
l_returnflag,
|
|
l_linestatus
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=98MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=198MB.*
|
|
row_regex: .*Cluster Memory Admitted: 198.00 MB.*
|
|
====
|
|
---- QUERY
|
|
# SELECT with multiple unpartitioned analytic functions to force the sort and analytics
|
|
# into the coordinator fragment.
|
|
select id,
|
|
min(int_col) over (order by year),
|
|
min(int_col) over (order by bigint_col),
|
|
avg(smallint_col) over (order by int_col),
|
|
max(int_col) over (order by smallint_col rows between unbounded preceding and 1 following)
|
|
from functional.alltypes;
|
|
---- RUNTIME_PROFILE
|
|
row_regex: .*Per-Host Resource Estimates: Memory=46MB.*
|
|
row_regex: .*Dedicated Coordinator Resource Estimate: Memory=124MB.*
|
|
row_regex: .*Cluster Memory Admitted: 216.00 MB.*
|
|
====
|