From 3ed2a82a95aa08c093e784cdf1b43c8b6069fdf8 Mon Sep 17 00:00:00 2001 From: Riza Suminto Date: Thu, 4 Dec 2025 18:06:48 -0800 Subject: [PATCH] IMPALA-14606: Stop building impala-shell for Python 2 This patch stop setting up and building impala-shell for Python 2. A more thorough clean up will be done in the future. Testing: Pass build and test/shell/ in RHEL8. Change-Id: Ic7d59b283f4e2f011880ff6221d550b52714a538 Reviewed-on: http://gerrit.cloudera.org:8080/23750 Reviewed-by: Impala Public Jenkins Tested-by: Impala Public Jenkins --- CMakeLists.txt | 9 +++++---- bin/bootstrap_system.sh | 4 +++- bin/impala-config.sh | 3 ++- shell/CMakeLists.txt | 13 ++++++++----- tests/shell/util.py | 11 ++++++----- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cfd75ddc5..f19d443fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -530,9 +530,10 @@ add_custom_target(cscope ALL DEPENDS gen-deps COMMAND "${CMAKE_SOURCE_DIR}/bin/gen-cscope.sh" ) -add_custom_target(impala_python ALL - COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" -) +# IMPALA-14606: Stop building impala_python (Python 2) by default. +# add_custom_target(impala_python ALL +# COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" +# ) add_custom_target(impala_python3 ALL COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" "-python3" @@ -548,7 +549,7 @@ endif() add_custom_target(impala_shell_pypi ALL DEPENDS ${IMPALA_PYTHON_INSTALLS}) add_custom_target(notests_independent_targets DEPENDS - java cscope tarballs impala_python impala_python3 impala_shell_pypi + java cscope tarballs impala_python3 impala_shell_pypi ) add_custom_target(notests_regular_targets DEPENDS impalad statestored catalogd admissiond fesupport loggingsupport ImpalaUdf udasample udfsample impala-profile-tool diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index ec5dc7ac5..17fdfd2aa 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -344,7 +344,9 @@ function setup_python2() { sudo dnf -y install python2-devel } -redhat8 setup_python2 +# IMPALA-14606: Stop building using Python 2 and always run with +# IMPALA_USE_PYTHON3_TESTS=true. +# redhat8 setup_python2 redhat8 pip install --user argparse # Point Python to Python 3 for Redhat 9 and Ubuntu 22, or newer diff --git a/bin/impala-config.sh b/bin/impala-config.sh index ffe94fa7c..63a1a21f9 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -224,7 +224,8 @@ unset IMPALA_MOLD_URL export IMPALA_SIMBA_JDBC_DRIVER_VERSION=42-2.6.32.1041 # Find system python versions for testing -export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-$(command -v python2)}" +# IMPALA-14606: Stop building impala_python (Python 2) by default. +export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-}" export IMPALA_SYSTEM_PYTHON3="${IMPALA_SYSTEM_PYTHON3_OVERRIDE-$(command -v python3)}" # Additional Python versions to use when building the impala-shell prebuilt tarball diff --git a/shell/CMakeLists.txt b/shell/CMakeLists.txt index c0c378d97..c60afae1b 100644 --- a/shell/CMakeLists.txt +++ b/shell/CMakeLists.txt @@ -60,7 +60,9 @@ foreach(PYTHON_EXE IN LISTS PYTHON_EXES) # different distributions add_custom_target(${PYTHON_NAME}_venv BYPRODUCTS "${VENV}" - DEPENDS impala_python + # IMPALA-14606: create_virtualenv.sh may invoke impala-virtualenv, which require + # python2_venv. Only uncomment following line if building python2_venv. + # DEPENDS impala_python COMMAND "${CMAKE_SOURCE_DIR}/bin/cmake_aux/create_virtualenv.sh" "${PYTHON_EXE}" "${VENV}" COMMAND "${VENV}/bin/pip" install --cache-dir "${PIP_CACHE}" --upgrade pip @@ -96,10 +98,11 @@ add_custom_target(shell_pypi_test_package ) # Tests expect to find venvs at 'python2_venv' and 'python3_venv' in tests/shell/util.py. -set(PYTHON2_VENV "${VENV_LOC}/python2_venv") -add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package - COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}" -) +# IMPALA-14606: Stop building 'python2_venv' and always run with IMPALA_USE_PYTHON3_TESTS=true. +# set(PYTHON2_VENV "${VENV_LOC}/python2_venv") +# add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package +# COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}" +# ) set(PYTHON3_VENV "${VENV_LOC}/python3_venv") add_custom_target(shell_python3_install DEPENDS python3_venv shell_pypi_test_package diff --git a/tests/shell/util.py b/tests/shell/util.py index 2770d11db..0d4bd0b0b 100755 --- a/tests/shell/util.py +++ b/tests/shell/util.py @@ -381,10 +381,10 @@ def create_impala_shell_executable_dimension(dev_only=False): _, include_pypi = get_dev_impala_shell_executable() dimensions = [] python3_pytest = (os.getenv("IMPALA_USE_PYTHON3_TESTS", "false") == "true") + assert os.getenv("IMPALA_SYSTEM_PYTHON3"), "Must set env var IMPALA_SYSTEM_PYTHON3!" + dimensions.append('dev3') if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest: dimensions.append('dev') - if os.getenv("IMPALA_SYSTEM_PYTHON3"): - dimensions.append('dev3') if include_pypi and not dev_only: if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest: dimensions.append('python2') @@ -395,16 +395,17 @@ def create_impala_shell_executable_dimension(dev_only=False): def get_impala_shell_executable(vector): # impala-shell is invoked some places where adding a test vector may not make sense; - # use 'dev' as the default. + # use 'dev3' as the default. impala_shell_executable, _ = get_dev_impala_shell_executable() - return { + executable_map = { 'dev': ['env', 'IMPALA_PYTHON_EXECUTABLE=python', 'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable], 'dev3': ['env', 'IMPALA_PYTHON_EXECUTABLE=python3', 'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable], 'python2': [os.path.join(IMPALA_HOME, 'shell/build/python2_venv/bin/impala-shell')], 'python3': [os.path.join(IMPALA_HOME, 'shell/build/python3_venv/bin/impala-shell')] - }[vector.get_value_with_default('impala_shell', 'dev')] + } + return executable_map[vector.get_value_with_default('impala_shell', 'dev3')] def stderr_get_first_error_msg(stderr):