From 6a199be8544f5d76e00829f7493fcaac2f9c274c Mon Sep 17 00:00:00 2001 From: Joe McDonnell Date: Sat, 21 May 2022 19:10:11 -0700 Subject: [PATCH] IMPALA-11249: Fix add_test_dimensions() locations to call super() The original issue is that the strict HS2 shell tests are not running in precommit or nightly jobs, but they do run in local developer environments. Investigating this showed that the shell tests were running with a weird set of test dimensions that includes table_format_and_file_extension. That dimension is only used in test_insert.py::TestInsertFileExtension. What is happening is that the shell tests and other locations are running add_test_dimensions() without calling super(..., cls).add_test_dimensions(). The behavior is unclear, but there is clearly cross-talk between the different tests that do this. This changes all add_test_dimensions() locations to call super(..., cls).add_test_dimensions() if they don't already. Each location has been tuned to run the same set of tests as before (except the shell tests which now run the strict HS2 tests). As part of this, several shell tests need to be skipped or fixed for strict HS2. Testing: - Ran core job - Ran tests locally to verify the set of tests didn't change. Change-Id: Ib20fd479d3b91ed0ed89a0bc5623cd2a5a458614 Reviewed-on: http://gerrit.cloudera.org:8080/18557 Reviewed-by: Impala Public Jenkins Tested-by: Impala Public Jenkins --- tests/authorization/test_ranger.py | 8 ++++++++ tests/custom_cluster/test_client_ssl.py | 1 + tests/query_test/test_insert.py | 22 +++++++++++++++------- tests/query_test/test_nested_types.py | 5 ++++- tests/shell/test_shell_client.py | 11 +++++++++-- tests/shell/test_shell_commandline.py | 10 ++++++++-- tests/shell/test_shell_interactive.py | 16 +++++++++++++--- 7 files changed, 58 insertions(+), 15 deletions(-) diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py index 875886b12..d0caa7b17 100644 --- a/tests/authorization/test_ranger.py +++ b/tests/authorization/test_ranger.py @@ -1591,6 +1591,7 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite): @classmethod def add_test_dimensions(cls): + super(TestRangerColumnMaskingComplexTypesInSelectList, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload())) cls.ImpalaTestMatrix.add_constraint(lambda v: @@ -1598,6 +1599,13 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite): cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension( disable_codegen_options=[True])) + @classmethod + def add_custom_cluster_constraints(cls): + # Do not call the super() implementation, because this class needs to relax + # the set of constraints. The usual constraints only run on uncompressed text. + # This disables that constraint to let us run against only ORC. + return + @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impalad_args=IMPALAD_ARGS, catalogd_args=CATALOGD_ARGS) diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py index 66a3fff78..4add40501 100644 --- a/tests/custom_cluster/test_client_ssl.py +++ b/tests/custom_cluster/test_client_ssl.py @@ -140,6 +140,7 @@ class TestClientSsl(CustomClusterTestSuite): @classmethod def add_test_dimensions(cls): + super(TestClientSsl, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) @pytest.mark.execute_serially diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py index 401d9b960..30d75fdc3 100644 --- a/tests/query_test/test_insert.py +++ b/tests/query_test/test_insert.py @@ -29,7 +29,9 @@ from tests.common.skip import SkipIfABFS, SkipIfEC, SkipIfLocal, \ SkipIfHive2, SkipIfNotHdfsMinicluster, SkipIfS3, SkipIfDockerizedCluster from tests.common.test_dimensions import ( create_exec_option_dimension, - create_uncompressed_text_dimension) + create_uncompressed_text_dimension, + create_single_exec_option_dimension, + is_supported_insert_format) from tests.common.test_result_verifier import ( QueryTestResult, parse_result_rows) @@ -335,20 +337,26 @@ class TestInsertFileExtension(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): - cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension( - 'table_format_and_file_extension', - *[('parquet', '.parq'), ('textfile', '.txt')])) + super(TestInsertFileExtension, cls).add_test_dimensions() + cls.ImpalaTestMatrix.add_constraint(lambda v: + is_supported_insert_format(v.get_value('table_format'))) + cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension()) @classmethod def setup_class(cls): super(TestInsertFileExtension, cls).setup_class() def test_file_extension(self, vector, unique_database): - table_format = vector.get_value('table_format_and_file_extension')[0] - file_extension = vector.get_value('table_format_and_file_extension')[1] + table_format = vector.get_value('table_format').file_format + if table_format == 'parquet': + file_extension = '.parq' + stored_as_format = 'parquet' + else: + file_extension = '.txt' + stored_as_format = 'textfile' table_name = "{0}_table".format(table_format) ctas_query = "create table {0}.{1} stored as {2} as select 1".format( - unique_database, table_name, table_format) + unique_database, table_name, stored_as_format) self.execute_query_expect_success(self.client, ctas_query) for path in self.filesystem_client.ls("test-warehouse/{0}.db/{1}".format( unique_database, table_name)): diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py index 3ca64f885..ef64e1181 100644 --- a/tests/query_test/test_nested_types.py +++ b/tests/query_test/test_nested_types.py @@ -36,7 +36,8 @@ from tests.common.skip import ( SkipIfNotHdfsMinicluster ) from tests.common.test_dimensions import (create_exec_option_dimension, - create_exec_option_dimension_from_dict, create_client_protocol_dimension) + create_exec_option_dimension_from_dict, create_client_protocol_dimension, + create_orc_dimension) from tests.common.test_vector import ImpalaTestDimension from tests.util.filesystem_utils import WAREHOUSE, get_fs_path, IS_HDFS @@ -185,9 +186,11 @@ class TestNestedTypesInSelectListWithBeeswax(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): + super(TestNestedTypesInSelectListWithBeeswax, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) cls.ImpalaTestMatrix.add_constraint(lambda v: v.get_value('protocol') == 'beeswax') + cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload())) cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension( disable_codegen_options=[True])) diff --git a/tests/shell/test_shell_client.py b/tests/shell/test_shell_client.py index 5fa1d5caa..1388b527f 100644 --- a/tests/shell/test_shell_client.py +++ b/tests/shell/test_shell_client.py @@ -20,8 +20,9 @@ from shell.impala_client import ImpalaBeeswaxClient, ImpalaHS2Client from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.test_dimensions import create_client_protocol_dimension -from tests.common.test_dimensions import create_client_protocol_no_strict_dimension +from tests.common.test_dimensions import ( + create_client_protocol_dimension, create_client_protocol_no_strict_dimension, + create_uncompressed_text_dimension, create_single_exec_option_dimension) from util import get_impalad_host_port @@ -34,6 +35,12 @@ class TestShellClient(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): + super(TestShellClient, cls).add_test_dimensions() + # Limit to uncompressed text with default exec options + cls.ImpalaTestMatrix.add_dimension( + create_uncompressed_text_dimension(cls.get_workload())) + cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension()) + # Run with beeswax and HS2 cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) cls.ImpalaTestMatrix.add_dimension(create_client_protocol_no_strict_dimension()) diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py index 3d698fb8a..3c6454d5a 100644 --- a/tests/shell/test_shell_commandline.py +++ b/tests/shell/test_shell_commandline.py @@ -34,8 +34,9 @@ from tests.common.environ import ImpalaTestClusterProperties from tests.common.impala_service import ImpaladService from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT from tests.common.skip import SkipIf -from tests.common.test_dimensions import create_client_protocol_dimension -from tests.common.test_dimensions import create_client_protocol_strict_dimension +from tests.common.test_dimensions import ( + create_client_protocol_dimension, create_client_protocol_strict_dimension, + create_uncompressed_text_dimension, create_single_exec_option_dimension) from time import sleep, time from util import (get_impalad_host_port, assert_var_substitution, run_impala_shell_cmd, ImpalaShell, IMPALA_SHELL_EXECUTABLE, SHELL_IS_PYTHON_2, @@ -134,6 +135,11 @@ class TestImpalaShell(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): + super(TestImpalaShell, cls).add_test_dimensions() + # Limit to uncompressed text with default exec options + cls.ImpalaTestMatrix.add_dimension( + create_uncompressed_text_dimension(cls.get_workload())) + cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension()) # Run with both beeswax and HS2 to ensure that behaviour is the same. cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension()) diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py index c1411f86b..cd2db996b 100755 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -40,8 +40,9 @@ from tempfile import NamedTemporaryFile from tests.common.impala_service import ImpaladService from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfLocal -from tests.common.test_dimensions import create_client_protocol_dimension -from tests.common.test_dimensions import create_client_protocol_strict_dimension +from tests.common.test_dimensions import ( + create_client_protocol_dimension, create_client_protocol_strict_dimension, + create_uncompressed_text_dimension, create_single_exec_option_dimension) from tests.shell.util import get_unused_port from util import (assert_var_substitution, ImpalaShell, get_impalad_port, get_shell_cmd, get_open_sessions_metric, IMPALA_SHELL_EXECUTABLE, spawn_shell) @@ -165,6 +166,11 @@ class TestImpalaShellInteractive(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): + super(TestImpalaShellInteractive, cls).add_test_dimensions() + # Limit to uncompressed text with default exec options + cls.ImpalaTestMatrix.add_dimension( + create_uncompressed_text_dimension(cls.get_workload())) + cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension()) # Run with both beeswax and HS2 to ensure that behaviour is the same. cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension()) @@ -681,6 +687,8 @@ class TestImpalaShellInteractive(ImpalaTestSuite): def test_commandline_flag_disable_live_progress(self, vector): """Test the command line flag disable_live_progress with live_progress.""" + if vector.get_value('strict_hs2_protocol'): + pytest.skip("Live option not supported in strict hs2 mode.") # By default, shell option live_progress is set to True in the interactive mode. cmds = "set all;" result = run_impala_shell_interactive(vector, cmds) @@ -704,13 +712,15 @@ class TestImpalaShellInteractive(ImpalaTestSuite): cmds = "set all;" # override the default option through command line argument. - args = ['--strict_h2_protocol'] + args = ['--strict_hs2_protocol'] result = run_impala_shell_interactive(vector, cmds, shell_args=args) assert "\tLIVE_PROGRESS: False" in result.stdout assert "\tLIVE_SUMMARY: False" in result.stdout def test_live_option_configuration(self, vector): """Test the optional configuration file with live_progress and live_summary.""" + if vector.get_value('strict_hs2_protocol'): + pytest.skip("Live option not supported in strict hs2 mode.") # Positive tests # set live_summary and live_progress as True with config file rcfile_path = os.path.join(QUERY_FILE_PATH, 'good_impalarc3')