mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
ea0969a772
Python 3 changed the behavior of imports with PEP328. Existing imports become absolute unless they use the new relative import syntax. This adapts the impala-shell code to use absolute imports, fixing issues where it is imported from our test code. There are several parts to this: 1. It moves impala shell code into shell/impala_shell. This matches the directory structure of the PyPi package. 2. It changes the imports in the shell code to be absolute paths (i.e. impala_shell.foo rather than foo). This fixes issues with Python 3 absolute imports. It also eliminates the need for ugly hacks in the PyPi package's __init__.py. 3. This changes Thrift generation to put it directly in $IMPALA_HOME/shell rather than $IMPALA_HOME/shell/gen-py. This means that the generated Thrift code is rooted in the same directory as the shell code. 4. This changes the PYTHONPATH to include $IMPALA_HOME/shell and not $IMPALA_HOME/shell/gen-py. This means that the test code is using the same import paths as the pypi package. With all of these changes, the source code is very close to the directory structure of the PyPi package. As long as CMake has generated the thrift files and the Python version file, only a few differences remain. This removes those differences by moving the setup.py / MANIFEST.in and other files from the packaging directory to the top-level shell/ directory. This means that one can pip install directly from the source code. i.e. pip install $IMPALA_HOME/shell This also moves the shell tarball generation script to the packaging directory and changes bin/impala-shell.sh to use Python 3. This sorts the imports using isort for the affected Python files. Testing: - Ran a regular core job with Python 2 - Ran a core job with Python 3 and verified that the absolute import issues are gone. Change-Id: Ica75a24fa6bcb78999b9b6f4f4356951b81c3124 Reviewed-on: http://gerrit.cloudera.org:8080/22330 Reviewed-by: Riza Suminto <riza.suminto@cloudera.com> Reviewed-by: Michael Smith <michael.smith@cloudera.com> Tested-by: Riza Suminto <riza.suminto@cloudera.com>
123 lines
5.5 KiB
Python
123 lines
5.5 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from __future__ import absolute_import, division, print_function
|
|
import os
|
|
import ssl
|
|
import sys
|
|
import time
|
|
|
|
import pytest
|
|
|
|
# This import is the actual ImpalaShell class from impala_shell.py.
|
|
# We rename it to ImpalaShellClass here because we later import another
|
|
# class called ImpalaShell from tests/shell/util.py, and we don't want
|
|
# to mask it.
|
|
from impala_shell.impala_shell import ImpalaShell as ImpalaShellClass
|
|
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
|
|
from tests.common.environ import IS_REDHAT_DERIVATIVE
|
|
from tests.common.test_dimensions import create_client_protocol_dimension
|
|
from tests.common.test_vector import ImpalaTestVector
|
|
from tests.shell.util import ImpalaShell
|
|
|
|
REQUIRED_MIN_OPENSSL_VERSION = 0x10001000
|
|
# Python supports TLSv1.2 from 2.7.9 officially but on Red Hat/CentOS Python2.7.5
|
|
# with newer python-libs (eg python-libs-2.7.5-77) supports TLSv1.2 already
|
|
if IS_REDHAT_DERIVATIVE:
|
|
REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 = (2, 7, 5)
|
|
else:
|
|
REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 = (2, 7, 9)
|
|
_openssl_version_number = getattr(ssl, "OPENSSL_VERSION_NUMBER", None)
|
|
if _openssl_version_number is None:
|
|
SKIP_SSL_MSG = "Legacy OpenSSL module detected"
|
|
elif _openssl_version_number < REQUIRED_MIN_OPENSSL_VERSION:
|
|
SKIP_SSL_MSG = "Only have OpenSSL version %X, but test requires %X" % (
|
|
ssl.OPENSSL_VERSION_NUMBER, REQUIRED_MIN_OPENSSL_VERSION)
|
|
else:
|
|
SKIP_SSL_MSG = None
|
|
CERT_DIR = "%s/be/src/testutil" % os.environ['IMPALA_HOME']
|
|
|
|
SSL_ARGS = ("--ssl_client_ca_certificate=%s/server-cert.pem "
|
|
"--ssl_server_certificate=%s/server-cert.pem "
|
|
"--ssl_private_key=%s/server-key.pem "
|
|
"--hostname=localhost " # Required to match hostname in certificate
|
|
% (CERT_DIR, CERT_DIR, CERT_DIR))
|
|
|
|
# IMPALA-12114 was an issue that occurred when the number of idle polls exceeded a limit
|
|
# and lead to disconnection. This uses a very short poll period to make these tests
|
|
# do more polls and verify the fix for this issue.
|
|
IDLE_ARGS = " --idle_client_poll_period_s=1 -v=2"
|
|
|
|
|
|
class TestThriftSocket(CustomClusterTestSuite):
|
|
""" Check if thrift timeout errors are detected properly """
|
|
|
|
@classmethod
|
|
def setup_class(cls):
|
|
if sys.version_info < REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12:
|
|
pytest.skip("Python version does not support tls 1.2")
|
|
super(TestThriftSocket, cls).setup_class()
|
|
|
|
@pytest.mark.execute_serially
|
|
@CustomClusterTestSuite.with_args(
|
|
impalad_args=IDLE_ARGS, cluster_size=1, disable_log_buffering=True)
|
|
def test_peek_timeout_no_ssl(self):
|
|
# Iterate over test vector within test function to avoid restarting cluster.
|
|
for protocol_dim in create_client_protocol_dimension():
|
|
for vector in [ImpalaTestVector([protocol_dim])]:
|
|
shell_args = ["-Q", "idle_session_timeout=1800"]
|
|
# This uses a longer idle time to verify IMPALA-12114, see comment above.
|
|
self._run_idle_shell(vector, shell_args, 12)
|
|
self.assert_impalad_log_contains('INFO',
|
|
r'Socket read or peek timeout encountered.*THRIFT_EAGAIN \(timed out\)',
|
|
expected_count=-1)
|
|
|
|
@pytest.mark.execute_serially
|
|
@CustomClusterTestSuite.with_args(statestored_args=SSL_ARGS,
|
|
catalogd_args=SSL_ARGS,
|
|
impalad_args=(SSL_ARGS + IDLE_ARGS),
|
|
cluster_size=1,
|
|
disable_log_buffering=True)
|
|
def test_peek_timeout_ssl(self):
|
|
# Iterate over test vector within test function to avoid restarting cluster.
|
|
for protocol_dim in create_client_protocol_dimension():
|
|
for vector in [ImpalaTestVector([protocol_dim])]:
|
|
shell_args = ["-Q", "idle_session_timeout=1800", "--ssl"]
|
|
# This uses a longer idle time to verify IMPALA-12114, see comment above.
|
|
self._run_idle_shell(vector, shell_args, 12)
|
|
self.assert_impalad_log_contains('INFO',
|
|
r'Socket read or peek timeout encountered.*THRIFT_POLL \(timed out\)',
|
|
expected_count=-1)
|
|
|
|
def _run_idle_shell(self, vector, args, idle_time):
|
|
p = ImpalaShell(vector, args)
|
|
p.send_cmd("USE functional")
|
|
# Running different statements before and after idle allows for more
|
|
# precise asserts that can distinguish output from before vs after.
|
|
p.send_cmd("SHOW DATABASES")
|
|
time.sleep(idle_time)
|
|
p.send_cmd("SHOW TABLES")
|
|
|
|
result = p.get_result()
|
|
# Output from before the idle period
|
|
assert "functional_parquet" in result.stdout, result.stdout
|
|
# Output from after the idle period
|
|
assert "alltypesaggmultifilesnopart" in result.stdout, result.stdout
|
|
# Impala shell reconnects automatically, so we need to be sure that
|
|
# it didn't lose the connection.
|
|
assert ImpalaShellClass.CONNECTION_LOST_MESSAGE not in result.stderr, result.stderr
|