jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-25 02:03:09 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-5644,IMPALA-5810: Min reservation improvements

Browse Source 
Rejects queries during admission control if:
* the largest (across all backends) min buffer reservation is
  greater than the query mem_limit or buffer_pool_limit
* the sum of the min buffer reservations across the cluster
  is larger than the pool max mem resources

There are some other interesting cases to consider later:
* every per-backend min buffer reservation is less than the
  associated backend's process mem_limit; the current
  admission control code doesn't know about other backend's
  proc mem_limits.

Also reduces minimum non-reservation memory (IMPALA-5810).
See the JIRA for experimental results that show this
slightly improves min memory requirements for small queries.
One reason to tweak this is to compensate for the fact that
BufferedBlockMgr didn't count small buffers against the
BlockMgr limit, but BufferPool counts all buffers against
it.

Testing:
* Adds new test cases in test_admission_controller.py
* Adds BE tests in reservation-tracker-test for the
  reservation-util code.

Change-Id: Iabe87ce8f460356cfe4d1be4d7092c5900f9d79b
Reviewed-on: http://gerrit.cloudera.org:8080/7678
Reviewed-by: Matthew Jacobs <mj@cloudera.com>
Tested-by: Impala Public Jenkins
This commit is contained in:
Matthew Jacobs
2017-08-15 10:06:25 -07:00
committed by Impala Public Jenkins
parent 5ec404640a
commit 7264c54751
13 changed files with 298 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
tests/query_test/test_mem_usage_scaling.py
View File

@@ -26,8 +26,9 @@ from tests.common.test_vector import ImpalaTestDimension
# Substrings of the expected error messages when the mem limit is too low
MEM_LIMIT_EXCEEDED_MSG = "Memory limit exceeded"
INITIAL_RESERVATION_MSG = "Failed to get minimum memory reservation"
MEM_LIMIT_ERROR_MSGS = [MEM_LIMIT_EXCEEDED_MSG, INITIAL_RESERVATION_MSG]
MEM_LIMIT_TOO_LOW_FOR_RESERVATION = ("minimum memory reservation is greater than memory "
"available to the query for buffer reservations")
MEM_LIMIT_ERROR_MSGS = [MEM_LIMIT_EXCEEDED_MSG, MEM_LIMIT_TOO_LOW_FOR_RESERVATION]
class TestQueryMemLimitScaling(ImpalaTestSuite):
"""Test class to do functional validation of per query memory limits. """
@@ -90,30 +91,6 @@ class TestExprMemUsage(ImpalaTestSuite):
table_format=vector.get_value('table_format'))
class TestInitialReservation(ImpalaTestSuite):
@classmethod
def get_workload(self):
# Note: this workload doesn't run exhaustively. See IMPALA-3947 before trying to move
# this test to exhaustive.
return 'tpch'
@classmethod
def add_test_dimensions(cls):
super(TestInitialReservation, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format in ['parquet'])
def test_initial_reservation(self, vector):
"""Test failure to get the initial reservation."""
exec_options = copy(vector.get_value('exec_option'))
exec_options['mem_limit'] = '20m'
query = """select * from tpch_parquet.lineitem l1
join tpch_parquet.lineitem l2 on l1.l_orderkey = l2.l_orderkey"""
result = self.execute_query_expect_failure(self.client, query, exec_options)
assert (INITIAL_RESERVATION_MSG in str(result))
class TestLowMemoryLimits(ImpalaTestSuite):
'''Super class for the memory limit tests with the TPC-H and TPC-DS queries'''