jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 09:58:28 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-14606: Stop building impala-shell for Python 2

Browse Source 
This patch stop setting up and building impala-shell for Python 2.
A more thorough clean up will be done in the future.

Testing:
Pass build and test/shell/ in RHEL8.

Change-Id: Ic7d59b283f4e2f011880ff6221d550b52714a538
Reviewed-on: http://gerrit.cloudera.org:8080/23750
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
Riza Suminto
2025-12-04 18:06:48 -08:00
committed by Impala Public Jenkins
parent 1bddbefb2d
commit 3ed2a82a95
5 changed files with 24 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
CMakeLists.txt
View File

@@ -530,9 +530,10 @@ add_custom_target(cscope ALL DEPENDS gen-deps
COMMAND "${CMAKE_SOURCE_DIR}/bin/gen-cscope.sh"
)
add_custom_target(impala_python ALL
COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
)
# IMPALA-14606: Stop building impala_python (Python 2) by default.
# add_custom_target(impala_python ALL
# COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
# )
add_custom_target(impala_python3 ALL
COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" "-python3"
@@ -548,7 +549,7 @@ endif()
add_custom_target(impala_shell_pypi ALL DEPENDS ${IMPALA_PYTHON_INSTALLS})
add_custom_target(notests_independent_targets DEPENDS
java cscope tarballs impala_python impala_python3 impala_shell_pypi
java cscope tarballs impala_python3 impala_shell_pypi
)
add_custom_target(notests_regular_targets DEPENDS
impalad statestored catalogd admissiond fesupport loggingsupport ImpalaUdf udasample udfsample impala-profile-tool

4
bin/bootstrap_system.sh
View File

@@ -344,7 +344,9 @@ function setup_python2() {
sudo dnf -y install python2-devel
}
redhat8 setup_python2
# IMPALA-14606: Stop building using Python 2 and always run with
# IMPALA_USE_PYTHON3_TESTS=true.
# redhat8 setup_python2
redhat8 pip install --user argparse
# Point Python to Python 3 for Redhat 9 and Ubuntu 22, or newer

3
bin/impala-config.sh
View File

@@ -224,7 +224,8 @@ unset IMPALA_MOLD_URL
export IMPALA_SIMBA_JDBC_DRIVER_VERSION=42-2.6.32.1041
# Find system python versions for testing
export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-$(command -v python2)}"
# IMPALA-14606: Stop building impala_python (Python 2) by default.
export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-}"
export IMPALA_SYSTEM_PYTHON3="${IMPALA_SYSTEM_PYTHON3_OVERRIDE-$(command -v python3)}"
# Additional Python versions to use when building the impala-shell prebuilt tarball

13
shell/CMakeLists.txt
View File

@@ -60,7 +60,9 @@ foreach(PYTHON_EXE IN LISTS PYTHON_EXES)
# different distributions
add_custom_target(${PYTHON_NAME}_venv
BYPRODUCTS "${VENV}"
DEPENDS impala_python
# IMPALA-14606: create_virtualenv.sh may invoke impala-virtualenv, which require
# python2_venv. Only uncomment following line if building python2_venv.
# DEPENDS impala_python
COMMAND "${CMAKE_SOURCE_DIR}/bin/cmake_aux/create_virtualenv.sh"
"${PYTHON_EXE}" "${VENV}"
COMMAND "${VENV}/bin/pip" install --cache-dir "${PIP_CACHE}" --upgrade pip
@@ -96,10 +98,11 @@ add_custom_target(shell_pypi_test_package
)
# Tests expect to find venvs at 'python2_venv' and 'python3_venv' in tests/shell/util.py.
set(PYTHON2_VENV "${VENV_LOC}/python2_venv")
add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package
COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}"
)
# IMPALA-14606: Stop building 'python2_venv' and always run with IMPALA_USE_PYTHON3_TESTS=true.
# set(PYTHON2_VENV "${VENV_LOC}/python2_venv")
# add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package
# COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}"
# )
set(PYTHON3_VENV "${VENV_LOC}/python3_venv")
add_custom_target(shell_python3_install DEPENDS python3_venv shell_pypi_test_package

11
tests/shell/util.py
View File

@@ -381,10 +381,10 @@ def create_impala_shell_executable_dimension(dev_only=False):
_, include_pypi = get_dev_impala_shell_executable()
dimensions = []
python3_pytest = (os.getenv("IMPALA_USE_PYTHON3_TESTS", "false") == "true")
assert os.getenv("IMPALA_SYSTEM_PYTHON3"), "Must set env var IMPALA_SYSTEM_PYTHON3!"
dimensions.append('dev3')
if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest:
dimensions.append('dev')
if os.getenv("IMPALA_SYSTEM_PYTHON3"):
dimensions.append('dev3')
if include_pypi and not dev_only:
if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest:
dimensions.append('python2')
@@ -395,16 +395,17 @@ def create_impala_shell_executable_dimension(dev_only=False):
def get_impala_shell_executable(vector):
# impala-shell is invoked some places where adding a test vector may not make sense;
# use 'dev' as the default.
# use 'dev3' as the default.
impala_shell_executable, _ = get_dev_impala_shell_executable()
return {
executable_map = {
'dev': ['env', 'IMPALA_PYTHON_EXECUTABLE=python',
'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable],
'dev3': ['env', 'IMPALA_PYTHON_EXECUTABLE=python3',
'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable],
'python2': [os.path.join(IMPALA_HOME, 'shell/build/python2_venv/bin/impala-shell')],
'python3': [os.path.join(IMPALA_HOME, 'shell/build/python3_venv/bin/impala-shell')]
}[vector.get_value_with_default('impala_shell', 'dev')]
}
return executable_map[vector.get_value_with_default('impala_shell', 'dev3')]
def stderr_get_first_error_msg(stderr):