jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-25 02:03:09 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-5838: Improve errors on AC buffer mem rejection

Browse Source 
The error message returned when a query is rejected due to
insufficient buffer memory is misleading. It recommended a
mem_limit which would be high enough, but changing the
mem_limit may result in changing the plan, which may result
in further changes to the buffer memory requirement.

In particular, this can happen when the planner compares the
expected hash table size to the mem_limit, and decides to
choose a partitioned join over a broadcast join.

While we might consider other code changes to improve this,
for now lets just be clear in the error message.

Testing:
* Adds tests that verify the expected behavior with the new
  error message.

Change-Id: I3dc3517195508d86078a8a4b537ae7d2f52fbcb7
Reviewed-on: http://gerrit.cloudera.org:8080/7834
Reviewed-by: Matthew Jacobs <mj@cloudera.com>
Tested-by: Impala Public Jenkins
This commit is contained in:
Matthew Jacobs
2017-08-25 13:41:05 -07:00
committed by Impala Public Jenkins
parent 260a97256f
commit 77e9e262af
3 changed files with 57 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
tests/custom_cluster/test_admission_controller.py
View File

@@ -321,7 +321,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
impalad_args=impalad_admission_ctrl_flags(max_requests=1, max_queued=1,
pool_max_mem=10 * 1024 * 1024, proc_mem_limit=1024 * 1024 * 1024),
statestored_args=_STATESTORED_ARGS)
def test_reject_min_reservation(self):
def test_reject_min_reservation(self, vector):
"""Test that the query will be rejected by admission control if:
a) the largest per-backend min buffer reservation is larger than the query mem
limit
@@ -330,26 +330,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
c) the cluster-wide min-buffer reservation size is larger than the pool memory
resources.
"""
query = "select distinct * from functional_parquet.alltypesagg"
opts = {'mem_limit': '10MB', 'num_nodes': '1'}
ex = self.execute_query_expect_failure(self.client, query, opts)
assert ("minimum memory reservation is greater than memory available to the query "
"for buffer reservations. Mem available for buffer reservations based on "
"mem_limit: 10.00 MB, memory reservation needed: 34.00 MB. "
"Set mem_limit to at least 109.00 MB.") in str(ex)
query = "select distinct * from functional_parquet.alltypesagg"
opts = {'buffer_pool_limit': '10MB', 'num_nodes': '1'}
ex = self.execute_query_expect_failure(self.client, query, opts)
assert ("minimum memory reservation is greater than memory available to the query "
"for buffer reservations. Mem available for buffer reservations based on "
"buffer_pool_limit: 10.00 MB, memory reservation needed: 34.00 MB.") in str(ex)
opts = {'mem_limit': '150MB', 'num_nodes': '1'}
ex = self.execute_query_expect_failure(self.client, query, opts)
assert ("minimum memory reservation needed is greater than pool max mem resources. "
"pool max mem resources: 10.00 MB, cluster-wide memory reservation needed: "
"34.00 MB") in str(ex)
self.run_test_case('QueryTest/admission-reject-min-reservation', vector)
# Process mem_limit used in test_mem_limit_upper_bound
PROC_MEM_TEST_LIMIT = 1024 * 1024 * 1024