IMPALA-14606: Stop building impala-shell for Python 2

This patch stop setting up and building impala-shell for Python 2.
A more thorough clean up will be done in the future.

Testing:
Pass build and test/shell/ in RHEL8.

Change-Id: Ic7d59b283f4e2f011880ff6221d550b52714a538
Reviewed-on: http://gerrit.cloudera.org:8080/23750
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

CMakeLists.txt

@@ -530,9 +530,10 @@ add_custom_target(cscope ALL DEPENDS gen-deps
   COMMAND "${CMAKE_SOURCE_DIR}/bin/gen-cscope.sh"
 )
 
-add_custom_target(impala_python ALL
+# IMPALA-14606: Stop building impala_python (Python 2) by default.
-  COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
+# add_custom_target(impala_python ALL
-)
+#   COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
+# )
 
 add_custom_target(impala_python3 ALL
   COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" "-python3"
@@ -548,7 +549,7 @@ endif()
 add_custom_target(impala_shell_pypi ALL DEPENDS ${IMPALA_PYTHON_INSTALLS})
 
 add_custom_target(notests_independent_targets DEPENDS
-  java cscope tarballs impala_python impala_python3 impala_shell_pypi
+  java cscope tarballs impala_python3 impala_shell_pypi
 )
 add_custom_target(notests_regular_targets DEPENDS
   impalad statestored catalogd admissiond fesupport loggingsupport ImpalaUdf udasample udfsample impala-profile-tool

bin/bootstrap_system.sh

@@ -344,7 +344,9 @@ function setup_python2() {
   sudo dnf -y install python2-devel
 }
 
-redhat8 setup_python2
+# IMPALA-14606: Stop building using Python 2 and always run with
+# IMPALA_USE_PYTHON3_TESTS=true.
+# redhat8 setup_python2
 redhat8 pip install --user argparse
 
 # Point Python to Python 3 for Redhat 9 and Ubuntu 22, or newer

bin/impala-config.sh

@@ -224,7 +224,8 @@ unset IMPALA_MOLD_URL
 export IMPALA_SIMBA_JDBC_DRIVER_VERSION=42-2.6.32.1041
 
 # Find system python versions for testing
-export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-$(command -v python2)}"
+# IMPALA-14606: Stop building impala_python (Python 2) by default.
+export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-}"
 export IMPALA_SYSTEM_PYTHON3="${IMPALA_SYSTEM_PYTHON3_OVERRIDE-$(command -v python3)}"
 
 # Additional Python versions to use when building the impala-shell prebuilt tarball

shell/CMakeLists.txt

@@ -60,7 +60,9 @@ foreach(PYTHON_EXE IN LISTS PYTHON_EXES)
   # different distributions
   add_custom_target(${PYTHON_NAME}_venv
     BYPRODUCTS "${VENV}"
-    DEPENDS impala_python
+    # IMPALA-14606: create_virtualenv.sh may invoke impala-virtualenv, which require
+    # python2_venv. Only uncomment following line if building python2_venv.
+    # DEPENDS impala_python
     COMMAND "${CMAKE_SOURCE_DIR}/bin/cmake_aux/create_virtualenv.sh"
             "${PYTHON_EXE}" "${VENV}"
     COMMAND "${VENV}/bin/pip" install --cache-dir "${PIP_CACHE}" --upgrade pip
@@ -96,10 +98,11 @@ add_custom_target(shell_pypi_test_package
 )
 
 # Tests expect to find venvs at 'python2_venv' and 'python3_venv' in tests/shell/util.py.
-set(PYTHON2_VENV "${VENV_LOC}/python2_venv")
+# IMPALA-14606: Stop building 'python2_venv' and always run with IMPALA_USE_PYTHON3_TESTS=true.
-add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package
+# set(PYTHON2_VENV "${VENV_LOC}/python2_venv")
-  COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}"
+# add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package
-)
+#   COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}"
+# )
 
 set(PYTHON3_VENV "${VENV_LOC}/python3_venv")
 add_custom_target(shell_python3_install DEPENDS python3_venv shell_pypi_test_package

tests/shell/util.py

@@ -381,10 +381,10 @@ def create_impala_shell_executable_dimension(dev_only=False):
   _, include_pypi = get_dev_impala_shell_executable()
   dimensions = []
   python3_pytest = (os.getenv("IMPALA_USE_PYTHON3_TESTS", "false") == "true")
+  assert os.getenv("IMPALA_SYSTEM_PYTHON3"), "Must set env var IMPALA_SYSTEM_PYTHON3!"
+  dimensions.append('dev3')
   if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest:
     dimensions.append('dev')
-  if os.getenv("IMPALA_SYSTEM_PYTHON3"):
-    dimensions.append('dev3')
   if include_pypi and not dev_only:
     if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest:
       dimensions.append('python2')
@@ -395,16 +395,17 @@ def create_impala_shell_executable_dimension(dev_only=False):
 
 def get_impala_shell_executable(vector):
   # impala-shell is invoked some places where adding a test vector may not make sense;
-  # use 'dev' as the default.
+  # use 'dev3' as the default.
   impala_shell_executable, _ = get_dev_impala_shell_executable()
-  return {
+  executable_map = {
     'dev': ['env', 'IMPALA_PYTHON_EXECUTABLE=python',
             'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable],
     'dev3': ['env', 'IMPALA_PYTHON_EXECUTABLE=python3',
              'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable],
     'python2': [os.path.join(IMPALA_HOME, 'shell/build/python2_venv/bin/impala-shell')],
     'python3': [os.path.join(IMPALA_HOME, 'shell/build/python3_venv/bin/impala-shell')]
-  }[vector.get_value_with_default('impala_shell', 'dev')]
+  }
+  return executable_map[vector.get_value_with_default('impala_shell', 'dev3')]
 
 
 def stderr_get_first_error_msg(stderr):