IMPALA-7660: Support ECDH ciphers for debug webserver

A recent change (IMPALA-7519) added support for ecdh ciphers for the
beeswax/hs2 server. This patch pulls in a recent change on squeasel to
extend that support to the debug webserver.

It also fixes a bug that prevented start-impala-cluster.py from
completing successfully when the webserver is launched with ssl, due
to it trying to verify the availablitiy of the webserver over http.

Testing:
- Added a custom cluster test that verifies start-impala-cluster.py
  runs successfully with webserver ssl enabled.
- Adds the webserver to an existing test for ecdh ciphers.

Change-Id: I80a6b370d5860812cde13229b5bcb2977814c73c
Reviewed-on: http://gerrit.cloudera.org:8080/11585
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

be/src/thirdparty/squeasel/squeasel.c

@@ -4298,11 +4298,37 @@ static int set_ssl_option(struct sq_context *ctx) {
     (void) SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, pem);
   }
 
-  if (ctx->config[SSL_CIPHERS] != NULL &&
-      (SSL_CTX_set_cipher_list(ctx->ssl_ctx, ctx->config[SSL_CIPHERS]) == 0)) {
-    cry(fc(ctx), "SSL_CTX_set_cipher_list: error setting ciphers (%s): %s",
-        ctx->config[SSL_CIPHERS], ssl_error());
-    return 0;
+  if (ctx->config[SSL_CIPHERS] != NULL) {
+    if (SSL_CTX_set_cipher_list(ctx->ssl_ctx, ctx->config[SSL_CIPHERS]) == 0) {
+      cry(fc(ctx), "SSL_CTX_set_cipher_list: error setting ciphers (%s): %s",
+          ctx->config[SSL_CIPHERS], ssl_error());
+      return 0;
+    }
+#ifndef OPENSSL_NO_ECDH
+#if OPENSSL_VERSION_NUMBER < 0x10002000L
+    // OpenSSL 1.0.1 and below only support setting a single ECDH curve at once.
+    // We choose prime256v1 because it's the first curve listed in the "modern
+    // compatibility" section of the Mozilla Server Side TLS recommendations,
+    // accessed Feb. 2017.
+    EC_KEY* ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
+    if (ecdh == NULL) {
+      cry(fc(ctx), "EC_KEY_new_by_curve_name: %s", ssl_error());
+    }
+
+    int rc = SSL_CTX_set_tmp_ecdh(ctx->ssl_ctx, ecdh);
+    if (rc <= 0) {
+      cry(fc(ctx), "SSL_CTX_set_tmp_ecdh: %s", ssl_error());
+    }
+#elif OPENSSL_VERSION_NUMBER < 0x10100000L
+    // OpenSSL 1.0.2 provides the set_ecdh_auto API which internally figures out
+    // the best curve to use.
+    int rc = SSL_CTX_set_ecdh_auto(ctx->ssl_ctx, 1);
+    if (rc <= 0) {
+      cry(fc(ctx), "SSL_CTX_set_ecdh_auto: %s", ssl_error());
+    }
+#endif
+#endif
+
   }
 
   return 1;

tests/common/impala_cluster.py

@@ -222,6 +222,10 @@ class BaseImpalaProcess(Process):
   def _get_webserver_port(self, default=None):
     return int(self._get_arg_value('webserver_port', default))
 
+  def _get_webserver_ssl(self):
+    """Returns true if the webserver is being run with ssl."""
+    return self._get_arg_value("webserver_certificate_file", "") != ""
+
   def _get_arg_value(self, arg_name, default=None):
     """Gets the argument value for given argument name"""
     for arg in self.cmd:
@@ -239,7 +243,8 @@ class ImpaladProcess(BaseImpalaProcess):
     self.service = ImpaladService(self.hostname, self._get_webserver_port(default=25000),
                                   self.__get_beeswax_port(default=21000),
                                   self.__get_be_port(default=22000),
-                                  self.__get_hs2_port(default=21050))
+                                  self.__get_hs2_port(default=21050),
+                                  self._get_webserver_ssl())
 
   def __get_beeswax_port(self, default=None):
     return int(self._get_arg_value('beeswax_port', default))
@@ -264,16 +269,16 @@ class ImpaladProcess(BaseImpalaProcess):
 class StateStoreProcess(BaseImpalaProcess):
   def __init__(self, cmd):
     super(StateStoreProcess, self).__init__(cmd, socket.gethostname())
-    self.service =\
-        StateStoredService(self.hostname, self._get_webserver_port(default=25010))
+    self.service = StateStoredService(
+        self.hostname, self._get_webserver_port(default=25010), self._get_webserver_ssl())
 
 
 # Represents a catalogd process
 class CatalogdProcess(BaseImpalaProcess):
   def __init__(self, cmd):
     super(CatalogdProcess, self).__init__(cmd, socket.gethostname())
-    self.service = CatalogdService(self.hostname,
-        self._get_webserver_port(default=25020), self.__get_port(default=26000))
+    self.service = CatalogdService(self.hostname, self._get_webserver_port(default=25020),
+        self._get_webserver_ssl(), self.__get_port(default=26000))
 
   def __get_port(self, default=None):
     return int(self._get_arg_value('catalog_service_port', default))

tests/common/impala_service.py

@@ -22,6 +22,7 @@
 import json
 import logging
 import re
+import ssl
 import urllib
 from time import sleep, time
 
@@ -41,17 +42,24 @@ LOG.setLevel(level=logging.DEBUG)
 # Base class for all Impala services
 # TODO: Refactor the retry/timeout logic into a common place.
 class BaseImpalaService(object):
-  def __init__(self, hostname, webserver_port):
+  def __init__(self, hostname, webserver_port, webserver_ssl=False):
     self.hostname = hostname
     self.webserver_port = webserver_port
+    self.webserver_ssl = webserver_ssl
 
   def open_debug_webpage(self, page_name, timeout=10, interval=1):
     start_time = time()
 
     while (time() - start_time < timeout):
       try:
-        return urllib.urlopen("http://%s:%d/%s" %
-            (self.hostname, int(self.webserver_port), page_name))
+        protocol = "http"
+        context = None
+        if self.webserver_ssl:
+          protocol = "https"
+          context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        url = "%s://%s:%d/%s" % \
+            (protocol, self.hostname, int(self.webserver_port), page_name)
+        return urllib.urlopen(url, context=context)
       except Exception:
         LOG.info("Debug webpage not yet available.")
       sleep(interval)
@@ -166,8 +174,8 @@ class BaseImpalaService(object):
 # new connections or accessing the debug webpage.
 class ImpaladService(BaseImpalaService):
   def __init__(self, hostname, webserver_port=25000, beeswax_port=21000, be_port=22000,
-               hs2_port=21050):
-    super(ImpaladService, self).__init__(hostname, webserver_port)
+               hs2_port=21050, webserver_ssl=False):
+    super(ImpaladService, self).__init__(hostname, webserver_port, webserver_ssl)
     self.beeswax_port = beeswax_port
     self.be_port = be_port
     self.hs2_port = hs2_port
@@ -307,8 +315,8 @@ class ImpaladService(BaseImpalaService):
 # Allows for interacting with the StateStore service to perform operations such as
 # accessing the debug webpage.
 class StateStoredService(BaseImpalaService):
-  def __init__(self, hostname, webserver_port):
-    super(StateStoredService, self).__init__(hostname, webserver_port)
+  def __init__(self, hostname, webserver_port, webserver_ssl):
+    super(StateStoredService, self).__init__(hostname, webserver_port, webserver_ssl)
 
   def wait_for_live_subscribers(self, num_subscribers, timeout=15, interval=1):
     self.wait_for_metric_value('statestore.live-backends', num_subscribers,
@@ -318,8 +326,8 @@ class StateStoredService(BaseImpalaService):
 # Allows for interacting with the Catalog service to perform operations such as
 # accessing the debug webpage.
 class CatalogdService(BaseImpalaService):
-  def __init__(self, hostname, webserver_port, service_port):
-    super(CatalogdService, self).__init__(hostname, webserver_port)
+  def __init__(self, hostname, webserver_port, webserver_ssl, service_port):
+    super(CatalogdService, self).__init__(hostname, webserver_port, webserver_ssl)
     self.service_port = service_port
 
   def get_catalog_version(self, timeout=10, interval=1):

tests/custom_cluster/test_client_ssl.py

@@ -19,6 +19,7 @@
 import logging
 import os
 import pytest
+import requests
 import signal
 import ssl
 import socket
@@ -110,13 +111,27 @@ class TestClientSsl(CustomClusterTestSuite):
     assert "Query Status: Cancelled" in result.stdout
     assert impalad.wait_for_num_in_flight_queries(0)
 
+  WEBSERVER_SSL_ARGS = ("--webserver_certificate_file=%(cert_dir)s/server-cert.pem "
+                        "--webserver_private_key_file=%(cert_dir)s/server-key.pem "
+                        % {'cert_dir': CERT_DIR})
+
+  @pytest.mark.execute_serially
+  @CustomClusterTestSuite.with_args(impalad_args=WEBSERVER_SSL_ARGS,
+                                    statestored_args=WEBSERVER_SSL_ARGS,
+                                    catalogd_args=WEBSERVER_SSL_ARGS)
+  def test_webserver_ssl(self):
+    "Tests that the debug web pages are reachable when run with ssl."
+    self._verify_ssl_webserver()
+
   # Test that the shell can connect to a ECDH only cluster.
-  TLS_ECDH_ARGS = ("--ssl_client_ca_certificate=%s/server-cert.pem "
-                  "--ssl_server_certificate=%s/server-cert.pem "
-                  "--ssl_private_key=%s/server-key.pem "
-                  "--hostname=localhost "  # Required to match hostname in certificate"
-                  "--ssl_cipher_list=ECDHE-RSA-AES128-GCM-SHA256 "
-                  % (CERT_DIR, CERT_DIR, CERT_DIR))
+  TLS_ECDH_ARGS = ("--ssl_client_ca_certificate=%(cert_dir)s/server-cert.pem "
+                   "--ssl_server_certificate=%(cert_dir)s/server-cert.pem "
+                   "--ssl_private_key=%(cert_dir)s/server-key.pem "
+                   "--hostname=localhost "  # Required to match hostname in certificate"
+                   "--ssl_cipher_list=ECDHE-RSA-AES128-GCM-SHA256 "
+                   "--webserver_certificate_file=%(cert_dir)s/server-cert.pem "
+                   "--webserver_private_key_file=%(cert_dir)s/server-key.pem "
+                   % {'cert_dir': CERT_DIR})
 
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(impalad_args=TLS_ECDH_ARGS,
@@ -128,6 +143,7 @@ class TestClientSsl(CustomClusterTestSuite):
   def test_tls_ecdh(self, vector):
     self._verify_negative_cases()
     self._validate_positive_cases("%s/server-cert.pem" % self.CERT_DIR)
+    self._verify_ssl_webserver()
 
   # Test that the shell can connect to a TLS1.2 only cluster, and for good measure
   # restrict the cipher suite to just one choice.
@@ -209,3 +225,9 @@ class TestClientSsl(CustomClusterTestSuite):
       result = run_impala_shell_cmd(shell_options)
       for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]:
         assert msg in result.stderr
+
+  def _verify_ssl_webserver(self):
+    for port in ["25000", "25010", "25020"]:
+      url = "https://localhost:%s" % port
+      response = requests.get(url, verify="%s/server-cert.pem" % self.CERT_DIR)
+      assert response.status_code == requests.codes.ok, url