jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 18:12:08 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-11249: Fix add_test_dimensions() locations to call super()

Browse Source 
The original issue is that the strict HS2 shell tests
are not running in precommit or nightly jobs, but they
do run in local developer environments. Investigating
this showed that the shell tests were running with a
weird set of test dimensions that includes
table_format_and_file_extension. That dimension is only
used in test_insert.py::TestInsertFileExtension.

What is happening is that the shell tests and other
locations are running add_test_dimensions() without
calling super(..., cls).add_test_dimensions(). The
behavior is unclear, but there is clearly cross-talk
between the different tests that do this.

This changes all add_test_dimensions() locations to
call super(..., cls).add_test_dimensions() if they
don't already. Each location has been tuned to run
the same set of tests as before (except the shell
tests which now run the strict HS2 tests).

As part of this, several shell tests need to be
skipped or fixed for strict HS2.

Testing:
 - Ran core job
 - Ran tests locally to verify the set of tests
   didn't change.

Change-Id: Ib20fd479d3b91ed0ed89a0bc5623cd2a5a458614
Reviewed-on: http://gerrit.cloudera.org:8080/18557
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
Joe McDonnell
2022-05-21 19:10:11 -07:00
committed by Impala Public Jenkins
parent f77f577074
commit 6a199be854
7 changed files with 58 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/authorization/test_ranger.py
View File

@@ -1591,6 +1591,7 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestRangerColumnMaskingComplexTypesInSelectList, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_constraint(lambda v:
@@ -1598,6 +1599,13 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
disable_codegen_options=[True]))
@classmethod
def add_custom_cluster_constraints(cls):
# Do not call the super() implementation, because this class needs to relax
# the set of constraints. The usual constraints only run on uncompressed text.
# This disables that constraint to let us run against only ORC.
return
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args=IMPALAD_ARGS, catalogd_args=CATALOGD_ARGS)

1
tests/custom_cluster/test_client_ssl.py
View File

@@ -140,6 +140,7 @@ class TestClientSsl(CustomClusterTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestClientSsl, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
@pytest.mark.execute_serially

22
tests/query_test/test_insert.py
View File

@@ -29,7 +29,9 @@ from tests.common.skip import SkipIfABFS, SkipIfEC, SkipIfLocal, \
SkipIfHive2, SkipIfNotHdfsMinicluster, SkipIfS3, SkipIfDockerizedCluster
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_uncompressed_text_dimension)
create_uncompressed_text_dimension,
create_single_exec_option_dimension,
is_supported_insert_format)
from tests.common.test_result_verifier import (
QueryTestResult,
parse_result_rows)
@@ -335,20 +337,26 @@ class TestInsertFileExtension(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension(
'table_format_and_file_extension',
*[('parquet', '.parq'), ('textfile', '.txt')]))
super(TestInsertFileExtension, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_constraint(lambda v:
is_supported_insert_format(v.get_value('table_format')))
cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
@classmethod
def setup_class(cls):
super(TestInsertFileExtension, cls).setup_class()
def test_file_extension(self, vector, unique_database):
table_format = vector.get_value('table_format_and_file_extension')[0]
file_extension = vector.get_value('table_format_and_file_extension')[1]
table_format = vector.get_value('table_format').file_format
if table_format == 'parquet':
file_extension = '.parq'
stored_as_format = 'parquet'
else:
file_extension = '.txt'
stored_as_format = 'textfile'
table_name = "{0}_table".format(table_format)
ctas_query = "create table {0}.{1} stored as {2} as select 1".format(
unique_database, table_name, table_format)
unique_database, table_name, stored_as_format)
self.execute_query_expect_success(self.client, ctas_query)
for path in self.filesystem_client.ls("test-warehouse/{0}.db/{1}".format(
unique_database, table_name)):

5
tests/query_test/test_nested_types.py
View File

@@ -36,7 +36,8 @@ from tests.common.skip import (
SkipIfNotHdfsMinicluster
)
from tests.common.test_dimensions import (create_exec_option_dimension,
create_exec_option_dimension_from_dict, create_client_protocol_dimension)
create_exec_option_dimension_from_dict, create_client_protocol_dimension,
create_orc_dimension)
from tests.common.test_vector import ImpalaTestDimension
from tests.util.filesystem_utils import WAREHOUSE, get_fs_path, IS_HDFS
@@ -185,9 +186,11 @@ class TestNestedTypesInSelectListWithBeeswax(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestNestedTypesInSelectListWithBeeswax, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(lambda v:
v.get_value('protocol') == 'beeswax')
cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
disable_codegen_options=[True]))

11
tests/shell/test_shell_client.py
View File

@@ -20,8 +20,9 @@
from shell.impala_client import ImpalaBeeswaxClient, ImpalaHS2Client
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_client_protocol_dimension
from tests.common.test_dimensions import create_client_protocol_no_strict_dimension
from tests.common.test_dimensions import (
create_client_protocol_dimension, create_client_protocol_no_strict_dimension,
create_uncompressed_text_dimension, create_single_exec_option_dimension)
from util import get_impalad_host_port
@@ -34,6 +35,12 @@ class TestShellClient(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestShellClient, cls).add_test_dimensions()
# Limit to uncompressed text with default exec options
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
# Run with beeswax and HS2
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_no_strict_dimension())

10
tests/shell/test_shell_commandline.py
View File

@@ -34,8 +34,9 @@ from tests.common.environ import ImpalaTestClusterProperties
from tests.common.impala_service import ImpaladService
from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT
from tests.common.skip import SkipIf
from tests.common.test_dimensions import create_client_protocol_dimension
from tests.common.test_dimensions import create_client_protocol_strict_dimension
from tests.common.test_dimensions import (
create_client_protocol_dimension, create_client_protocol_strict_dimension,
create_uncompressed_text_dimension, create_single_exec_option_dimension)
from time import sleep, time
from util import (get_impalad_host_port, assert_var_substitution, run_impala_shell_cmd,
ImpalaShell, IMPALA_SHELL_EXECUTABLE, SHELL_IS_PYTHON_2,
@@ -134,6 +135,11 @@ class TestImpalaShell(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestImpalaShell, cls).add_test_dimensions()
# Limit to uncompressed text with default exec options
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
# Run with both beeswax and HS2 to ensure that behaviour is the same.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())

16
tests/shell/test_shell_interactive.py
View File

@@ -40,8 +40,9 @@ from tempfile import NamedTemporaryFile
from tests.common.impala_service import ImpaladService
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfLocal
from tests.common.test_dimensions import create_client_protocol_dimension
from tests.common.test_dimensions import create_client_protocol_strict_dimension
from tests.common.test_dimensions import (
create_client_protocol_dimension, create_client_protocol_strict_dimension,
create_uncompressed_text_dimension, create_single_exec_option_dimension)
from tests.shell.util import get_unused_port
from util import (assert_var_substitution, ImpalaShell, get_impalad_port, get_shell_cmd,
get_open_sessions_metric, IMPALA_SHELL_EXECUTABLE, spawn_shell)
@@ -165,6 +166,11 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestImpalaShellInteractive, cls).add_test_dimensions()
# Limit to uncompressed text with default exec options
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
# Run with both beeswax and HS2 to ensure that behaviour is the same.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())
@@ -681,6 +687,8 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
def test_commandline_flag_disable_live_progress(self, vector):
"""Test the command line flag disable_live_progress with live_progress."""
if vector.get_value('strict_hs2_protocol'):
pytest.skip("Live option not supported in strict hs2 mode.")
# By default, shell option live_progress is set to True in the interactive mode.
cmds = "set all;"
result = run_impala_shell_interactive(vector, cmds)
@@ -704,13 +712,15 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
cmds = "set all;"
# override the default option through command line argument.
args = ['--strict_h2_protocol']
args = ['--strict_hs2_protocol']
result = run_impala_shell_interactive(vector, cmds, shell_args=args)
assert "\tLIVE_PROGRESS: False" in result.stdout
assert "\tLIVE_SUMMARY: False" in result.stdout
def test_live_option_configuration(self, vector):
"""Test the optional configuration file with live_progress and live_summary."""
if vector.get_value('strict_hs2_protocol'):
pytest.skip("Live option not supported in strict hs2 mode.")
# Positive tests
# set live_summary and live_progress as True with config file
rcfile_path = os.path.join(QUERY_FILE_PATH, 'good_impalarc3')