mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
0767ae065a
When runtime filters arrive after tuple caching has occurred, they can't filter the cached results. This can lead to larger tuple caching result sets than expected, causing correctness check failures in TPC tests. While other solutions may exist, extending RUNTIME_FILTER_WAIT_TIME_MS is a simple fix by ensuring runtime filters are applied before tuple caching. Also set the query option enable_tuple_cache_verification to false by default, as the filter arrival time may affect the correctness check. To avoid flaky tests, change to use a more conservative approach and only enable the correctness check when explicitly specified by the testcase. Tests: Verified TPC tests pass correctness checks with increased runtime filter wait time. Change-Id: Ie70a87344c436ce8e2073575df5c5bf762ef562d Reviewed-on: http://gerrit.cloudera.org:8080/21898 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
84 lines
3.4 KiB
Python
84 lines
3.4 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Functional tests running the TPCH and TPCDS workload twice to test tuple cache.
|
|
from __future__ import absolute_import, division, print_function
|
|
import pytest
|
|
|
|
from tests.common.environ import IS_TUPLE_CACHE_CORRECT_CHECK
|
|
from tests.common.impala_test_suite import ImpalaTestSuite
|
|
from tests.common.skip import SkipIf
|
|
from tests.common.test_dimensions import create_single_exec_option_dimension
|
|
from tests.util.test_file_parser import load_tpc_queries_name_sorted
|
|
|
|
MT_DOP_VALUES = [0, 4]
|
|
|
|
|
|
def run_tuple_cache_test(self, vector, query, mtdop):
|
|
vector.get_value('exec_option')['enable_tuple_cache'] = True
|
|
# Use a long runtime filter wait time (1 minute) to ensure filters arrive before
|
|
# generating the tuple cache for correctness check.
|
|
if IS_TUPLE_CACHE_CORRECT_CHECK:
|
|
vector.get_value('exec_option')['runtime_filter_wait_time_ms'] = 60000
|
|
vector.get_value('exec_option')['enable_tuple_cache_verification'] = True
|
|
vector.get_value('exec_option')['mt_dop'] = mtdop
|
|
# Run twice to test write and read the tuple cache.
|
|
self.run_test_case(query, vector)
|
|
self.run_test_case(query, vector)
|
|
|
|
|
|
@SkipIf.not_tuple_cache
|
|
class TestTupleCacheTpchQuery(ImpalaTestSuite):
|
|
@classmethod
|
|
def get_workload(self):
|
|
return 'tpch'
|
|
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestTupleCacheTpchQuery, cls).add_test_dimensions()
|
|
if cls.exploration_strategy() != 'exhaustive':
|
|
cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
|
|
cls.ImpalaTestMatrix.add_constraint(lambda v:
|
|
v.get_value('table_format').file_format == 'parquet'
|
|
and v.get_value('table_format').compression_codec == 'none')
|
|
|
|
@pytest.mark.parametrize("query", load_tpc_queries_name_sorted('tpch'))
|
|
@pytest.mark.parametrize("mtdop", MT_DOP_VALUES)
|
|
def test_tpch(self, vector, query, mtdop):
|
|
run_tuple_cache_test(self, vector, query, mtdop)
|
|
|
|
|
|
@SkipIf.not_tuple_cache
|
|
class TestTupleCacheTpcdsQuery(ImpalaTestSuite):
|
|
@classmethod
|
|
def get_workload(self):
|
|
return 'tpcds'
|
|
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestTupleCacheTpcdsQuery, cls).add_test_dimensions()
|
|
if cls.exploration_strategy() != 'exhaustive':
|
|
cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
|
|
cls.ImpalaTestMatrix.add_constraint(lambda v:
|
|
v.get_value('table_format').file_format == 'parquet'
|
|
and v.get_value('table_format').compression_codec == 'none')
|
|
|
|
@pytest.mark.parametrize("query", load_tpc_queries_name_sorted('tpcds'))
|
|
@pytest.mark.parametrize("mtdop", MT_DOP_VALUES)
|
|
def test_tpcds(self, vector, query, mtdop):
|
|
run_tuple_cache_test(self, vector, query, mtdop)
|