mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
95f353ac4a
We have plenty of custom_cluster tests that assert against content of
Impala daemon log files while the process is still running using
assert_log_contains() and it's wrappers. The method specifically mention
about disabling glog buffering ('-logbuflevel=-1'), but not all
custom_cluster tests do that. This often result in flaky test that hard
to triage and often neglected if it does not frequently run in core
exploration.
This patch adds boolean param 'disable_log_buffering' into
CustomClusterTestSuite.with_args for test to declare intention to
inspect log files in live minicluster. If it is True, start minicluster
with '-logbuflevel=-1' for all daemons. If it is False, log WARNING on
any calls to assert_log_contains().
There are several complex custom_cluster tests that left unchanged and
print out such WARNING logs, such as:
- TestQueryLive
- TestQueryLogTableBeeswax
- TestQueryLogOtherTable
- TestQueryLogTableHS2
- TestQueryLogTableAll
- TestQueryLogTableBufferPool
- TestStatestoreRpcErrors
- TestWorkloadManagementInitWait
- TestWorkloadManagementSQLDetails
This patch also fixed some small flake8 issues on modified tests.
There is a flakiness sign at test_query_live.py where test query is
submitted to coordinator and fail because sys.impala_query_live table
has not exist yet from coordinator's perspective. This patch modify
test_query_live.py to wait for few seconds until sys.impala_query_live
is queryable.
Testing:
- Pass custom_cluster tests in exhaustive exploration.
Change-Id: I56fb1746b8f3cea9f3db3514a86a526dffb44a61
Reviewed-on: http://gerrit.cloudera.org:8080/22015
Reviewed-by: Jason Fehr <jfehr@cloudera.com>
Reviewed-by: Michael Smith <michael.smith@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
73 lines
3.1 KiB
Python
Executable File
73 lines
3.1 KiB
Python
Executable File
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from __future__ import absolute_import, division, print_function
|
|
import pytest
|
|
|
|
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
|
|
|
|
|
|
class TestRE2MaxMem(CustomClusterTestSuite):
|
|
"""test if re2 max_mem parameter is set using the global flag in imapalad"""
|
|
@classmethod
|
|
def get_workload(cls):
|
|
return 'tpch'
|
|
|
|
def _test_re2_max_mem(self, expect_fail, dfa_out_of_mem):
|
|
""""test to see given an amount of memory (in Bytes) does the re2 print an error for
|
|
DFA run out of memory when compiling and pattern matching a long regexp"""
|
|
|
|
client = self.create_impala_client()
|
|
|
|
query = (
|
|
"select c_comment from tpch.customer where regexp_like(c_comment,"
|
|
"repeat('([a-c].*[t-w]|[t].?[h]|[^xyz]|.*?(\\\\d+))\\\\w', 1000),'i');")
|
|
|
|
# RE2 regex compilation storage is also dependent on max_mem parameter, for small
|
|
# values the regex pattern will fail as invalid pattern although it can be valid
|
|
# given a higher max_mem for RE2.
|
|
# See: https://github.com/google/re2/blob/3e9622e/re2/re2.h#L619-L648
|
|
if expect_fail:
|
|
# if we expect the regex compilation to fail we can ignore
|
|
# DFA out of memory issue at that will be brought up when
|
|
# RE2 does not having enough storage to store compiled regex
|
|
self.execute_query_expect_failure(client, query)
|
|
else:
|
|
self.execute_query_expect_success(client, query)
|
|
self.assert_impalad_log_contains(
|
|
"ERROR", "DFA out of memory", -1 if dfa_out_of_mem else 0)
|
|
|
|
@pytest.mark.execute_serially
|
|
@CustomClusterTestSuite.with_args(cluster_size=1,
|
|
impalad_args="--re2_mem_limit=1KB")
|
|
def test_dfa_out_of_mem(self):
|
|
self._test_re2_max_mem(True, True)
|
|
|
|
@pytest.mark.execute_serially
|
|
@CustomClusterTestSuite.with_args(cluster_size=1,
|
|
disable_log_buffering=True)
|
|
def test_re2_max_mem_not_specified(self):
|
|
# default max_mem set by re2's regex engine is 8 MiB
|
|
self._test_re2_max_mem(False, True)
|
|
|
|
@pytest.mark.execute_serially
|
|
@CustomClusterTestSuite.with_args(cluster_size=1,
|
|
impalad_args="--re2_mem_limit=200MB",
|
|
disable_log_buffering=True)
|
|
def test_dfa_not_out_of_mem(self):
|
|
self._test_re2_max_mem(False, False)
|