From caefd86136e7d8d5d3cd390402abf3370dc6cc6f Mon Sep 17 00:00:00 2001 From: Tim Armstrong Date: Thu, 24 Aug 2017 13:26:46 -0700 Subject: [PATCH] IMPALA-5830: SET_DENY_RESERVATION_PROBABILITY test Add a targeted test that confirms that setting the query option will force spilling. Testing: Ran test_spilling locally. Change-Id: Ida6b55b2dee0779b1739af5d75943518ec40d6ce Reviewed-on: http://gerrit.cloudera.org:8080/7809 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins --- .../QueryTest/disable-unsafe-spills.test | 11 ---- .../QueryTest/spilling-query-options.test | 54 +++++++++++++++++++ tests/common/impala_test_suite.py | 2 +- tests/query_test/test_spilling.py | 45 +++++++++++----- 4 files changed, 87 insertions(+), 25 deletions(-) delete mode 100644 testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test create mode 100644 testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test diff --git a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test deleted file mode 100644 index a9b8f359b..000000000 --- a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test +++ /dev/null @@ -1,11 +0,0 @@ -==== ----- QUERY -# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should -# not spill to disk. -set disable_unsafe_spills=true; -set buffer_pool_limit=40m; -select distinct * -from tpch_avro.orders ----- CATCH -Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false -==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test new file mode 100644 index 000000000..5b9a64100 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test @@ -0,0 +1,54 @@ +==== +---- QUERY +# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should +# not spill to disk. +set disable_unsafe_spills=true; +set buffer_pool_limit=40m; +select distinct * +from tpch_avro.orders +---- CATCH +Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false +==== +---- QUERY +# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on PREPARE. +set debug_action="-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0"; +select count(*) from ( + select distinct o_orderdate, o_custkey, o_comment + from tpch_parquet.orders where o_orderkey < 500000) v; +---- TYPES +BIGINT +---- RESULTS +124999 +---- RUNTIME_PROFILE +row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) +row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\) +==== +---- QUERY +# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on OPEN. +set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0"; +select count(*) from ( + select distinct o_orderdate, o_custkey, o_comment + from tpch_parquet.orders where o_orderkey < 500000) v; +---- TYPES +BIGINT +---- RESULTS +124999 +---- RUNTIME_PROFILE +row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) +row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\) +==== +---- QUERY +# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on GETNEXT. +# This won't affect the merge aggregation, which accumulates its memory in Open(), but +# will affect the streaming aggregation. +set debug_action="-1:GETNEXT:SET_DENY_RESERVATION_PROBABILITY@1.0"; +select count(*) from ( + select distinct o_orderdate, o_custkey, o_comment + from tpch_parquet.orders where o_orderkey < 500000) v; +---- TYPES +BIGINT +---- RESULTS +124999 +---- RUNTIME_PROFILE +row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\) +==== diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index d5841b270..1b7d04373 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -210,7 +210,7 @@ class ImpalaTestSuite(BaseTestSuite): if not query_option in self.default_query_options: continue default_val = self.default_query_options[query_option] - query_str = 'SET '+ query_option + '=' + default_val + ';' + query_str = 'SET '+ query_option + '="' + default_val + '"' try: impalad_client.execute(query_str) except Exception as e: diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py index c818b6542..0b3642931 100644 --- a/tests/query_test/test_spilling.py +++ b/tests/query_test/test_spilling.py @@ -30,14 +30,14 @@ DEBUG_ACTION_DIMS = [None, @pytest.mark.xfail(pytest.config.option.testing_remote_cluster, reason='Queries may not spill on larger clusters') -class TestSpilling(ImpalaTestSuite): +class TestSpillingDebugActionDimensions(ImpalaTestSuite): @classmethod def get_workload(self): return 'functional-query' @classmethod def add_test_dimensions(cls): - super(TestSpilling, cls).add_test_dimensions() + super(TestSpillingDebugActionDimensions, cls).add_test_dimensions() cls.ImpalaTestMatrix.clear_constraints() cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch')) # Tests are calibrated so that they can execute and spill with this page size. @@ -60,19 +60,38 @@ class TestSpilling(ImpalaTestSuite): """Test spilling null-aware anti-joins""" self.run_test_case('QueryTest/spilling-naaj', vector) - def test_spilling_naaj_no_deny_reservation(self, vector): - """ - Null-aware anti-join tests that depend on getting more than the minimum reservation - and therefore will not reliably pass with the deny reservation debug action enabled. - """ - if vector.get_value('exec_option')['debug_action'] is None: - self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector) - def test_spilling_sorts_exhaustive(self, vector): if self.exploration_strategy() != 'exhaustive': pytest.skip("only run large sorts on exhaustive") self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector) - def test_disable_unsafe_spills(self, vector): - """Test that the disable_unsafe_spills query options works end-to-end.""" - self.run_test_case('QueryTest/disable-unsafe-spills', vector) + +@pytest.mark.xfail(pytest.config.option.testing_remote_cluster, + reason='Queries may not spill on larger clusters') +class TestSpillingNoDebugActionDimensions(ImpalaTestSuite): + """Spilling tests to which we don't want to apply the debug_action dimension.""" + @classmethod + def get_workload(self): + return 'functional-query' + + @classmethod + def add_test_dimensions(cls): + super(TestSpillingNoDebugActionDimensions, cls).add_test_dimensions() + cls.ImpalaTestMatrix.clear_constraints() + cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch')) + # Tests are calibrated so that they can execute and spill with this page size. + cls.ImpalaTestMatrix.add_dimension( + create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k']})) + + def test_spilling_naaj_no_deny_reservation(self, vector): + """ + Null-aware anti-join tests that depend on getting more than the minimum reservation + and therefore will not reliably pass with the deny reservation debug action enabled. + """ + self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector) + + def test_spilling_query_options(self, vector): + """Test that spilling-related query options work end-to-end. These tests rely on + setting debug_action to alternative values via query options.""" + self.run_test_case('QueryTest/spilling-query-options', vector) +