Support for LDAP tests

* Allow Beeswax connections to optionally use LDAP
* Run custom cluster tests from the aux repo, if it exists

Change-Id: I054af64e030ad0cd722ae8dd75afda9c58ea2913
Reviewed-on: http://gerrit.ent.cloudera.com:8080/2547
Reviewed-by: Ishaan Joshi <ishaan@cloudera.com>
Tested-by: jenkins
Reviewed-on: http://gerrit.ent.cloudera.com:8080/2640

tests/beeswax/impala_beeswax.py

@@ -82,12 +82,14 @@ class ImpalaBeeswaxClient(object):
   # Regex applied to all tokens of a query to detect the query type.
   INSERT_REGEX = re.compile("^insert$", re.I)
 
-  def __init__(self, impalad, use_kerberos=False):
+  def __init__(self, impalad, use_kerberos=False, user=None, password=None):
     self.connected = False
     self.impalad = impalad
     self.imp_service = None
     self.transport = None
     self.use_kerberos = use_kerberos
+    self.user, self.password = user, password
+    self.use_ldap = (self.user is not None)
     self.__query_options = {}
     self.query_states = QueryState._NAMES_TO_VALUES
 
@@ -138,8 +140,11 @@ class ImpalaBeeswaxClient(object):
     trans_type = 'buffered'
     if self.use_kerberos:
       trans_type = 'kerberos'
+    elif self.use_ldap:
+      trans_type = 'plain_sasl'
     return create_transport(host=self.impalad[0], port=int(self.impalad[1]),
-                            service='impala', transport_type=trans_type)
+                            service='impala', transport_type=trans_type, user=self.user,
+                            password=self.password)
 
   def execute(self, query_string):
     """Re-directs the query to its appropriate handler, returns QueryResult"""

tests/common/impala_connection.py

@@ -116,8 +116,8 @@ class ImpalaConnection(object):
 
 # Represents a connection to Impala using the Beeswax API.
 class BeeswaxConnection(ImpalaConnection):
-  def __init__(self, host_port, use_kerberos=False):
-    self.__beeswax_client = ImpalaBeeswaxClient(host_port, use_kerberos)
+  def __init__(self, host_port, use_kerberos=False, user=None, password=None):
+    self.__beeswax_client = ImpalaBeeswaxClient(host_port, use_kerberos, user, password)
     self.__host_port = host_port
     self.QUERY_STATES = self.__beeswax_client.query_states
 
@@ -192,3 +192,6 @@ class BeeswaxConnection(ImpalaConnection):
 def create_connection(host_port, use_kerberos=False):
   # TODO: Support HS2 connections.
   return BeeswaxConnection(host_port=host_port, use_kerberos=use_kerberos)
+
+def create_ldap_connection(host_port, user, password):
+  return BeeswaxConnection(host_port=host_port, user=user, password=password)

tests/common/impala_service.py

@@ -26,6 +26,7 @@ import urllib
 from collections import defaultdict
 from HTMLParser import HTMLParser
 from tests.common.impala_connection import ImpalaConnection, create_connection
+from tests.common.impala_connection import create_ldap_connection
 from time import sleep, time
 
 logging.basicConfig(level=logging.ERROR, format='%(threadName)s: %(message)s')
@@ -170,6 +171,12 @@ class ImpaladService(BaseImpalaService):
     client.connect()
     return client
 
+  def create_ldap_beeswax_client(self, user, password):
+    client = create_ldap_connection('%s:%d' % (self.hostname, self.beeswax_port),
+                                    user=user, password=password)
+    client.connect()
+    return client
+
   def get_catalog_object_dump(self, object_type, object_name):
     return self.read_debug_webpage('catalog_objects?object_type=%s&object_name=%s' %\
         (object_type, object_name))

tests/run-custom-cluster-tests.sh

@@ -26,11 +26,16 @@ mkdir -p ${RESULTS_DIR}
 LOG_DIR=${IMPALA_TEST_CLUSTER_LOG_DIR}/custom_cluster/
 mkdir -p ${LOG_DIR}
 
+AUX_CUSTOM_DIR=""
+if [ -n ${IMPALA_AUX_TEST_HOME} ]; then
+    AUX_CUSTOM_DIR=${IMPALA_AUX_TEST_HOME}/tests/aux_custom_cluster_tests/
+fi
+
 export LOG_DIR
 
 cd ${IMPALA_HOME}/tests
 . ${IMPALA_HOME}/bin/set-classpath.sh
-py.test custom_cluster/ authorization/ \
+py.test custom_cluster/ authorization/ ${AUX_CUSTOM_DIR} \
     --junitxml="${RESULTS_DIR}/TEST-impala-custom-cluster.xml" \
     --resultlog="${RESULTS_DIR}/TEST-impala-custom-cluster.log" "$@"
 EXIT_CODE=$?

tests/util/thrift_util.py

@@ -19,7 +19,8 @@ from thrift.transport.TSocket import TSocket
 from thrift.transport.TTransport import TBufferedTransport
 import getpass
 
-def create_transport(host, port, service, transport_type="buffered"):
+def create_transport(host, port, service, transport_type="buffered", user=None,
+                     password=None):
   """
   Create a new Thrift Transport based on the requested type.
   Supported transport types:
@@ -31,6 +32,11 @@ def create_transport(host, port, service, transport_type="buffered"):
   if transport_type.lower() == "buffered":
     return TBufferedTransport(sock)
 
+  # Set defaults for LDAP connections
+  if transport_type.lower() == "plain_sasl":
+    if user is None: user = getpass.getuser()
+    if password is None: password = ""
+
   # Initializes a sasl client
   from shell.thrift_sasl import TSaslClientTransport
   def sasl_factory():
@@ -43,8 +49,8 @@ def create_transport(host, port, service, transport_type="buffered"):
     sasl_client.setAttr("host", host)
     sasl_client.setAttr("service", service)
     if transport_type.lower() == "plain_sasl":
-      sasl_client.setAttr("username", getpass.getuser())
-      sasl_client.setAttr("password", getpass.getuser())
+      sasl_client.setAttr("username", user)
+      sasl_client.setAttr("password", password)
     sasl_client.init()
     return sasl_client
   if transport_type.lower() == "plain_sasl":