IMPALA-7632: fix erasure coding build for custom cluster tests

Fix tests to always pass query options via the query_options
parameter.

Modified the infrastructure to fail on non-erasure-coding builds if
tests pass in default query options in the wrong way.

Skip an restart test that makes assumptions about scheduling that EC
seems to break.

Testing:
Ran core tests with erasure coding enabled.

Change-Id: I4d809faedc0c45417519f13c73559efb6c54154e
Reviewed-on: http://gerrit.cloudera.org:8080/11536
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

tests/common/custom_cluster_test_suite.py

@@ -195,10 +195,11 @@ class CustomClusterTestSuite(ImpalaTestSuite):
       default_query_option_kvs.append(("allow_erasure_coded_files", "true"))
     if default_query_options is not None:
       default_query_option_kvs.extend(default_query_options)
-
-    if default_query_option_kvs:
-      options.append("--impalad_args=--default_query_options={0}".format(
-          ','.join(["{0}={1}".format(k, v) for k, v in default_query_option_kvs])))
+    # Add the default query options after any arguments. This will override any default
+    # options set in --impalad_args by design to force tests to pass default_query_options
+    # into this function directly.
+    options.append("--impalad_args=--default_query_options={0}".format(
+        ','.join(["{0}={1}".format(k, v) for k, v in default_query_option_kvs])))
 
     logging.info("Starting cluster with command: %s" %
                  " ".join(pipes.quote(arg) for arg in cmd + options))

tests/common/skip.py

@@ -147,3 +147,4 @@ class SkipIfEC:
       "features relying on local read do not work.")
   oom = pytest.mark.skipif(IS_EC, reason="Probably broken by HDFS-13540.")
   fix_later = pytest.mark.skipif(IS_EC, reason="It should work but doesn't.")
+  scheduling = pytest.mark.skipif(IS_EC, reason="Scheduling is different on EC")

tests/custom_cluster/test_admission_controller.py

@@ -215,13 +215,9 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
         rhs = re.split(": ", line)[1]
         confs = re.split(",", rhs)
         break
-    assert len(confs) == len(expected_query_options)
-    confs = map(str.lower, confs)
-    for expected in expected_query_options:
-      if expected.lower() not in confs:
-        expected = ",".join(sorted(expected_query_options))
-        actual = ",".join(sorted(confs))
-        assert False, "Expected query options %s, got %s." % (expected, actual)
+    expected_set = set([x.lower() for x in expected_query_options])
+    confs_set = set([x.lower() for x in confs])
+    assert expected_set.issubset(confs_set)
 
   def __check_hs2_query_opts(self, pool_name, mem_limit=None, expected_options=None):
     """ Submits a query via HS2 (optionally with a mem_limit in the confOverlay)
@@ -283,8 +279,8 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
 
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(
-      impalad_args=impalad_admission_ctrl_config_args(\
-          "-default_query_options=mem_limit=200000000"),
+      impalad_args=impalad_admission_ctrl_config_args(),
+      default_query_options=[('mem_limit', 200000000)],
       statestored_args=_STATESTORED_ARGS)
   @needs_session(conf_overlay={'batch_size': '100'})
   def test_set_request_pool(self):

tests/custom_cluster/test_restart_services.py

@@ -30,6 +30,7 @@ from TCLIService import TCLIService
 
 from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.skip import SkipIfEC
 from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
 
 LOG = logging.getLogger(__name__)
@@ -161,6 +162,7 @@ class TestShutdownCommand(CustomClusterTestSuite, HS2TestSuite):
   EXEC_SHUTDOWN_DEADLINE_S = 10
 
   @pytest.mark.execute_serially
+  @SkipIfEC.scheduling
   @CustomClusterTestSuite.with_args(
       impalad_args="--shutdown_grace_period_s={grace_period} \
           --shutdown_deadline_s={deadline} \

tests/custom_cluster/test_set_and_unset.py

@@ -29,7 +29,7 @@ class TestSetAndUnset(CustomClusterTestSuite, HS2TestSuite):
   """
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(
-      impalad_args="--default_query_options=debug_action=custom")
+      default_query_options=[('debug_action', 'custom')])
   @needs_session(TCLIService.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6)
   def test_set_and_unset(self):
     """