mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
28cff4022d
Running exhaustive tests with env var IMPALA_USE_PYTHON3_TESTS=true reveals some tests that require adjustment. This patch made such adjustment, which mostly revolves around encoding differences and string vs bytes type in Python3. This patch also switch the default to run pytest with Python3 by setting IMPALA_USE_PYTHON3_TESTS=true. The following are the details: Change hash() function in conftest.py to crc32() to produce deterministic hash. Hash randomization is enabled by default since Python 3.3 (see https://docs.python.org/3/reference/datamodel.html#object.__hash__). This cause test sharding (like --shard_tests=1/2) produce inconsistent set of tests per shard. Always restart minicluster during custom cluster tests if --shard_tests argument is set, because test order may change and affect test correctness, depending on whether running on fresh minicluster or not. Moved one test case from delimited-latin-text.test to test_delimited_text.py for easier binary comparison. Add bytes_to_str() as a utility function to decode bytes in Python3. This is often needed when inspecting the return value of subprocess.check_output() as a string. Implement DataTypeMetaclass.__lt__ to substitute DataTypeMetaclass.__cmp__ that is ignored in Python3 (see https://peps.python.org/pep-0207/). Fix WEB_CERT_ERR difference in test_ipv6.py. Fix trivial integer parsing in test_restart_services.py. Fix various encoding issues in test_saml2_sso.py, test_shell_commandline.py, and test_shell_interactive.py. Change timeout in Impala.for_each_impalad() from sys.maxsize to 2^31-1. Switch to binary comparison in test_iceberg.py where needed. Specify text mode when calling tempfile.NamedTemporaryFile(). Simplify create_impala_shell_executable_dimension to skip testing dev and python2 impala-shell when IMPALA_USE_PYTHON3_TESTS=true. The reason is that several UTF-8 related tests in test_shell_commandline.py break in Python3 pytest + Python2 impala-shell combo. This skipping already happen automatically in build OS without system Python2 available like RHEL9 (IMPALA_SYSTEM_PYTHON2 env var is empty). Removed unused vector argument and fixed some trivial flake8 issues. Several test logic require modification due to intermittent issue in Python3 pytest. These include: Add _run_query_with_client() in test_ranger.py to allow reusing a single Impala client for running several queries. Ensure clients are closed when the test is done. Mark several tests in test_ranger.py with SkipIfFS.hive because they run queries through beeline + HiveServer2, but Ozone and S3 build environment does not start HiveServer2 by default. Increase the sleep period from 0.1 to 0.5 seconds per iteration in test_statestore.py and mark TestStatestore to execute serially. This is because TServer appears to shut down more slowly when run concurrently with other tests. Handle the deprecation of Thread.setDaemon() as well. Always force_restart=True each test method in TestLoggingCore, TestShellInteractiveReconnect, and TestQueryRetries to prevent them from reusing minicluster from previous test method. Some of these tests destruct minicluster (kill impalad) and will produce minidump if metrics verifier for next tests fail to detect healthy minicluster state. Testing: Pass exhaustive tests with IMPALA_USE_PYTHON3_TESTS=true. Change-Id: I401a93b6cc7bcd17f41d24e7a310e0c882a550d4 Reviewed-on: http://gerrit.cloudera.org:8080/23319 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
254 lines
10 KiB
Python
254 lines
10 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
from __future__ import absolute_import, division, print_function
|
|
import json
|
|
import logging
|
|
import os
|
|
import pytest
|
|
import requests
|
|
import sys
|
|
|
|
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
|
|
from tests.common.network import SKIP_SSL_MSG
|
|
from tests.common.test_dimensions import create_client_protocol_dimension
|
|
from tests.shell.util import run_impala_shell_cmd, \
|
|
create_impala_shell_executable_dimension
|
|
from tests.common.impala_connection import create_connection
|
|
|
|
LOG = logging.getLogger('impala_test_suite')
|
|
|
|
CERT_DIR = "%s/be/src/testutil" % os.environ['IMPALA_HOME']
|
|
|
|
# Use wildcard san cert to be flexible about host name.
|
|
SSL_WILDCARD_SAN_ARGS = ("--ssl_client_ca_certificate={0}/wildcardCA.pem "
|
|
"--ssl_server_certificate={0}/wildcard-san-cert.pem "
|
|
"--ssl_private_key={0}/wildcard-san-cert.key "
|
|
"--hostname={1} "
|
|
"--state_store_host={1} "
|
|
"--catalog_service_host={1} "
|
|
"--webserver_certificate_file={0}/wildcard-san-cert.pem "
|
|
"--webserver_private_key_file={0}/wildcard-san-cert.key "
|
|
).format(CERT_DIR, "ip4.impala.test")
|
|
|
|
WILDCARD_CA_CERT_PATH = "%s/wildcardCA.pem" % CERT_DIR
|
|
|
|
IPV6_ONLY_IP_WEBSERBER_ARG = "--webserver_interface=::1 "
|
|
IPV6_DUAL_IP_WEBSERBER_ARG = "--webserver_interface=:: "
|
|
IPV6_ONLY_IP_QUERY_ARG = "--external_interface=::1 "
|
|
IPV6_DUAL_IP_QUERY_ARG = "--external_interface=:: "
|
|
IPV6_ONLY_IP_COORDINATOR_ARG = \
|
|
"%s %s" % (IPV6_ONLY_IP_WEBSERBER_ARG, IPV6_ONLY_IP_QUERY_ARG)
|
|
IPV6_DUAL_IP_COORDINATOR_ARG = \
|
|
"%s %s" % (IPV6_DUAL_IP_WEBSERBER_ARG, IPV6_DUAL_IP_QUERY_ARG)
|
|
|
|
IPV6_ONLY_HOSTNAME_WEBSERBER_ARG = "--webserver_interface=ip6.impala.test "
|
|
IPV6_DUAL_HOSTNAME_WEBSERBER_ARG = "--webserver_interface=ip46.impala.test "
|
|
IPV6_ONLY_HOSTNAME_QUERY_ARG = "--external_interface=::1 "
|
|
IPV6_DUAL_HOSTNAME_QUERY_ARG = "--external_interface=:: "
|
|
|
|
WEBUI_PORTS = [25000, 25010, 25020]
|
|
|
|
# Error text can depend on both protocol and python version.
|
|
CONN_ERR = ["Could not connect", "Connection refused"]
|
|
CERT_ERR = ["doesn't match", "certificate verify failed"]
|
|
WEB_CERT_ERR = ("CertificateError" if sys.version_info.major < 3
|
|
else "SSLCertVerificationError")
|
|
|
|
|
|
class TestIPv6Base(CustomClusterTestSuite):
|
|
ca_cert = None
|
|
|
|
@classmethod
|
|
def setup_class(cls):
|
|
super(TestIPv6Base, cls).setup_class()
|
|
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestIPv6Base, cls).add_test_dimensions()
|
|
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
|
|
cls.ImpalaTestMatrix.add_dimension(create_impala_shell_executable_dimension())
|
|
|
|
def _smoke(self, host, vector, expected_errors=[]):
|
|
proto = vector.get_value('protocol')
|
|
try:
|
|
port = self._get_default_port(proto)
|
|
host_port = "%s:%d" % (host, port)
|
|
use_ssl = self.ca_cert is not None
|
|
conn = create_connection(host_port, protocol=proto, use_ssl=use_ssl)
|
|
conn.connect()
|
|
assert not expected_errors
|
|
res = conn.execute("select 1")
|
|
assert res.data == ["1"]
|
|
except Exception as ex:
|
|
for err in expected_errors:
|
|
if err in str(ex): return
|
|
raise ex
|
|
|
|
def _webui_smoke(self, url, err=None):
|
|
"""Tests to check glibc version and locale is available"""
|
|
try:
|
|
if self.ca_cert:
|
|
other_info_page = requests.get(url + "/?json", verify=self.ca_cert).text
|
|
else:
|
|
other_info_page = requests.get(url + "/?json", verify=False).text
|
|
assert err is None
|
|
other_info = json.loads(other_info_page)
|
|
assert "glibc_version" in other_info
|
|
except Exception as ex:
|
|
if not err: raise ex
|
|
assert err in str(ex)
|
|
|
|
def _shell_smoke(self, host, vector, expected_errors=[]):
|
|
proto = vector.get_value('protocol')
|
|
port = self._get_default_port(proto)
|
|
host_port = "%s:%d" % (host, port)
|
|
try:
|
|
shell_options = ["-i", host_port, "-q", "select 1"]
|
|
if self.ca_cert:
|
|
shell_options.append("--ssl")
|
|
shell_options.append("--ca_cert=" + self.ca_cert)
|
|
result = run_impala_shell_cmd(vector, shell_options)
|
|
assert not expected_errors
|
|
assert "Fetched 1 row" in result.stderr
|
|
except Exception as ex:
|
|
for err in expected_errors:
|
|
if err in str(ex): return
|
|
raise ex
|
|
|
|
|
|
@CustomClusterTestSuite.with_args(impalad_args=IPV6_DUAL_IP_WEBSERBER_ARG
|
|
+ IPV6_DUAL_IP_QUERY_ARG,
|
|
statestored_args=IPV6_DUAL_IP_WEBSERBER_ARG,
|
|
catalogd_args=IPV6_DUAL_IP_WEBSERBER_ARG)
|
|
class TestIPv6DualNoSsl(TestIPv6Base):
|
|
|
|
def test_ipv6_dual_no_ssl(self, vector):
|
|
for port in WEBUI_PORTS:
|
|
self._webui_smoke("http://127.0.0.1:%d" % port)
|
|
self._webui_smoke("http://[::1]:%d" % port)
|
|
self._webui_smoke("http://ip4.impala.test:%d" % port)
|
|
self._webui_smoke("http://ip6.impala.test:%d" % port)
|
|
self._webui_smoke("http://ip46.impala.test:%d" % port)
|
|
|
|
self._smoke("[::1]", vector)
|
|
self._smoke("127.0.0.1", vector)
|
|
self._smoke("ip4.impala.test", vector)
|
|
self._smoke("ip6.impala.test", vector)
|
|
self._smoke("ip46.impala.test", vector)
|
|
|
|
self._shell_smoke("[::1]", vector)
|
|
self._shell_smoke("127.0.0.1", vector)
|
|
self._shell_smoke("ip4.impala.test", vector)
|
|
self._shell_smoke("ip6.impala.test", vector)
|
|
self._shell_smoke("ip46.impala.test", vector)
|
|
|
|
|
|
@CustomClusterTestSuite.with_args(impalad_args=IPV6_ONLY_IP_WEBSERBER_ARG
|
|
+ IPV6_ONLY_IP_QUERY_ARG,
|
|
statestored_args=IPV6_ONLY_IP_WEBSERBER_ARG,
|
|
catalogd_args=IPV6_ONLY_IP_WEBSERBER_ARG)
|
|
class TestIPv6OnlyNoSsl(TestIPv6Base):
|
|
|
|
def test_ipv6_only_no_ssl(self, vector):
|
|
self.check_connections()
|
|
for port in WEBUI_PORTS:
|
|
self._webui_smoke("http://127.0.0.1:%d" % port, err="Connection refused")
|
|
self._webui_smoke("http://[::1]:%d" % port)
|
|
self._webui_smoke("http://ip4.impala.test:%d" % port, err="Connection refused")
|
|
self._webui_smoke("http://ip6.impala.test:%d" % port)
|
|
self._webui_smoke("http://ip46.impala.test:%d" % port)
|
|
|
|
self._smoke("[::1]", vector)
|
|
self._smoke("127.0.0.1", vector, CONN_ERR)
|
|
self._smoke("ip4.impala.test", vector, CONN_ERR)
|
|
self._smoke("ip6.impala.test", vector)
|
|
self._smoke("ip46.impala.test", vector)
|
|
|
|
self._shell_smoke("[::1]", vector)
|
|
self._shell_smoke("127.0.0.1", vector, CONN_ERR)
|
|
self._shell_smoke("ip4.impala.test", vector, CONN_ERR)
|
|
self._shell_smoke("ip6.impala.test", vector)
|
|
self._shell_smoke("ip46.impala.test", vector)
|
|
|
|
|
|
@CustomClusterTestSuite.with_args(impalad_args=IPV6_DUAL_HOSTNAME_WEBSERBER_ARG
|
|
+ IPV6_DUAL_HOSTNAME_QUERY_ARG
|
|
+ SSL_WILDCARD_SAN_ARGS,
|
|
statestored_args=IPV6_DUAL_HOSTNAME_WEBSERBER_ARG
|
|
+ SSL_WILDCARD_SAN_ARGS,
|
|
catalogd_args=IPV6_DUAL_HOSTNAME_WEBSERBER_ARG
|
|
+ SSL_WILDCARD_SAN_ARGS)
|
|
class TestIPv6DualSsl(TestIPv6Base):
|
|
ca_cert = WILDCARD_CA_CERT_PATH
|
|
|
|
@pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG)
|
|
def test_ipv6_dual_ssl(self, vector):
|
|
self.check_connections()
|
|
for port in WEBUI_PORTS:
|
|
self._webui_smoke("https://127.0.0.1:%d" % port, WEB_CERT_ERR)
|
|
self._webui_smoke("https://[::1]:%d" % port, WEB_CERT_ERR)
|
|
self._webui_smoke("https://ip4.impala.test:%d" % port)
|
|
self._webui_smoke("https://ip6.impala.test:%d" % port)
|
|
self._webui_smoke("https://ip46.impala.test:%d" % port)
|
|
|
|
self._smoke("[::1]", vector, CONN_ERR)
|
|
self._smoke("127.0.0.1", vector, CONN_ERR)
|
|
self._smoke("ip4.impala.test", vector)
|
|
self._smoke("ip6.impala.test", vector)
|
|
self._smoke("ip46.impala.test", vector)
|
|
|
|
self._shell_smoke("[::1]", vector, CERT_ERR)
|
|
self._shell_smoke("127.0.0.1", vector, CERT_ERR)
|
|
self._shell_smoke("ip4.impala.test", vector)
|
|
self._shell_smoke("ip6.impala.test", vector)
|
|
self._shell_smoke("ip46.impala.test", vector)
|
|
|
|
|
|
@CustomClusterTestSuite.with_args(impalad_args=IPV6_ONLY_HOSTNAME_WEBSERBER_ARG
|
|
+ IPV6_ONLY_HOSTNAME_QUERY_ARG
|
|
+ SSL_WILDCARD_SAN_ARGS,
|
|
statestored_args=IPV6_ONLY_HOSTNAME_WEBSERBER_ARG
|
|
+ SSL_WILDCARD_SAN_ARGS,
|
|
catalogd_args=IPV6_ONLY_HOSTNAME_WEBSERBER_ARG
|
|
+ SSL_WILDCARD_SAN_ARGS)
|
|
class TestIPv6OnlySsl(TestIPv6Base):
|
|
ca_cert = WILDCARD_CA_CERT_PATH
|
|
|
|
@pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG)
|
|
def test_ipv6_only_ssl(self, vector):
|
|
self.check_connections()
|
|
for port in WEBUI_PORTS:
|
|
self._webui_smoke("https://127.0.0.1:%d" % port, WEB_CERT_ERR)
|
|
self._webui_smoke("https://[::1]:%d" % port, WEB_CERT_ERR)
|
|
self._webui_smoke("https://ip4.impala.test:%d" % port, "Connection refused")
|
|
self._webui_smoke("https://ip6.impala.test:%d" % port)
|
|
self._webui_smoke("https://ip46.impala.test:%d" % port)
|
|
|
|
self._smoke("[::1]", vector, CONN_ERR)
|
|
self._smoke("127.0.0.1", vector, CONN_ERR)
|
|
self._smoke("ip4.impala.test", vector, CONN_ERR)
|
|
self._smoke("ip6.impala.test", vector)
|
|
self._smoke("ip46.impala.test", vector)
|
|
|
|
self._shell_smoke("[::1]", vector, CERT_ERR)
|
|
self._shell_smoke("127.0.0.1", vector, CONN_ERR)
|
|
self._shell_smoke("ip4.impala.test", vector, CONN_ERR)
|
|
self._shell_smoke("ip6.impala.test", vector)
|
|
self._shell_smoke("ip46.impala.test", vector)
|