mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
1913ab46ed
To remove the dependency on Python 2, existing scripts need to use python3 rather than python. These commands find those locations (for impala-python and regular python): git grep impala-python | grep -v impala-python3 | grep -v impala-python-common | grep -v init-impala-python git grep bin/python | grep -v python3 This removes or switches most of these locations by various means: 1. If a python file has a #!/bin/env impala-python (or python) but doesn't have a main function, it removes the hash-bang and makes sure that the file is not executable. 2. Most scripts can simply switch from impala-python to impala-python3 (or python to python3) with minimal changes. 3. The cm-api pypi package (which doesn't support Python 3) has been replaced by the cm-client pypi package and interfaces have changed. Rather than migrating the code (which hasn't been used in years), this deletes the old code and stops installing cm-api into the virtualenv. The code can be restored and revamped if there is any interest in interacting with CM clusters. 4. This switches tests/comparison over to impala-python3, but this code has bit-rotted. Some pieces can be run manually, but it can't be fully verified with Python 3. It shouldn't hold back the migration on its own. 5. This also replaces locations of impala-python in comments / documentation / READMEs. 6. kazoo (used for interacting with HBase) needed to be upgraded to a version that supports Python 3. The newest version of kazoo requires upgrades of other component versions, so this uses kazoo 2.8.0 to avoid needing other upgrades. The two remaining uses of impala-python are: - bin/cmake_aux/create_virtualenv.sh - bin/impala-env-versioned-python These will be removed separately when we drop Python 2 support completely. In particular, these are useful for testing impala-shell with Python 2 until we stop supporting Python 2 for impala-shell. The docker-based tests still use /usr/bin/python, but this can be switched over independently (and doesn't impact impala-python) Testing: - Ran core job - Ran build + dataload on Centos 7, Redhat 8 - Manual testing of individual scripts (except some bitrotted areas like the random query generator) Change-Id: If209b761290bc7e7c716c312ea757da3e3bca6dc Reviewed-on: http://gerrit.cloudera.org:8080/23468 Reviewed-by: Michael Smith <michael.smith@cloudera.com> Tested-by: Michael Smith <michael.smith@cloudera.com>
112 lines
5.2 KiB
CMake
112 lines
5.2 KiB
CMake
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# The shell tarball build only needs the build virtualenvs for the system
|
|
# pythons that are installed.
|
|
set(IMPALA_PYTHON_BUILD_VENVS "")
|
|
|
|
set(VENV_LOC "${CMAKE_SOURCE_DIR}/shell/build")
|
|
set(PIP_LOC "~/.cache/impala_pip")
|
|
|
|
# Tests depend on installing system pythons to specific locations, so we error if they
|
|
# won't match what's expected in this config and make_shell_tarball.sh.
|
|
set(PYTHON_EXES $ENV{IMPALA_EXTRA_PACKAGE_PYTHONS})
|
|
if (NOT $ENV{IMPALA_SYSTEM_PYTHON2} STREQUAL "")
|
|
get_filename_component(PYTHON_NAME $ENV{IMPALA_SYSTEM_PYTHON2} NAME)
|
|
if (NOT ${PYTHON_NAME} STREQUAL "python2")
|
|
message(FATAL_ERROR "IMPALA_SYSTEM_PYTHON2 must be a binary named python2")
|
|
endif()
|
|
list(APPEND PYTHON_EXES $ENV{IMPALA_SYSTEM_PYTHON2})
|
|
endif()
|
|
if (NOT $ENV{IMPALA_SYSTEM_PYTHON3} STREQUAL "")
|
|
get_filename_component(PYTHON_NAME $ENV{IMPALA_SYSTEM_PYTHON3} NAME)
|
|
if (NOT ${PYTHON_NAME} STREQUAL "python3")
|
|
message(FATAL_ERROR "IMPALA_SYSTEM_PYTHON3 must be a binary named python3")
|
|
endif()
|
|
list(APPEND PYTHON_EXES $ENV{IMPALA_SYSTEM_PYTHON3})
|
|
endif()
|
|
message(STATUS "Packaging for ${PYTHON_EXES}")
|
|
|
|
foreach(PYTHON_EXE IN LISTS PYTHON_EXES)
|
|
get_filename_component(PYTHON_NAME "${PYTHON_EXE}" NAME)
|
|
# These virtualenvs serve two purposes:
|
|
# 1. They have system python with wheel installed, and they can be used to produce
|
|
# wheels for external dependencies for the shell tarball build.
|
|
# 2. We pip install impala-shell into them for use in tests.
|
|
# The initial virtualenv creation includes the "pip install wheel" command to
|
|
# satisfy #1. #2 is a separate step and has no interaction with #1.
|
|
set(VENV "${VENV_LOC}/${PYTHON_NAME}_venv")
|
|
# IMPALA-12117: Use separate pip cache directories to avoid concurrency
|
|
# issues. The standard location is in ~/.cache/pip, so this uses directories
|
|
# inside ~/.cache. These typical consume a couple MB each.
|
|
set(PIP_CACHE "${PIP_LOC}/${PYTHON_NAME}")
|
|
|
|
# Supports fallback to impala-virtualenv for older Python versions.
|
|
# This upgrades pip in the virtualenv to make the behavior more consistent across
|
|
# different distributions
|
|
add_custom_target(${PYTHON_NAME}_venv
|
|
BYPRODUCTS "${VENV}"
|
|
DEPENDS impala_python
|
|
COMMAND "${CMAKE_SOURCE_DIR}/bin/cmake_aux/create_virtualenv.sh"
|
|
"${PYTHON_EXE}" "${VENV}"
|
|
COMMAND "${VENV}/bin/pip" install --cache-dir "${PIP_CACHE}" --upgrade pip
|
|
COMMAND "${VENV}/bin/pip" install --cache-dir "${PIP_CACHE}" wheel
|
|
)
|
|
|
|
list(APPEND IMPALA_PYTHON_BUILD_VENVS ${PYTHON_NAME}_venv)
|
|
endforeach()
|
|
|
|
add_custom_target(shell_impala_build_version
|
|
DEPENDS gen-deps "${CMAKE_SOURCE_DIR}/bin/version.info"
|
|
COMMAND "${CMAKE_SOURCE_DIR}/shell/gen_impala_build_version.sh"
|
|
)
|
|
|
|
add_custom_target(shell_pypi_package
|
|
DEPENDS gen-deps shell_impala_build_version impala_python3
|
|
COMMAND "${CMAKE_SOURCE_DIR}/shell/packaging/make_python_package.sh"
|
|
)
|
|
|
|
# A separate package target is needed because without OFFICIAL the file name is
|
|
# non-deterministic. Uses a custom target to synchronize for multiple dependents.
|
|
# Derive version from IMPALA_VERSION (drops everything after '-' because PEP 440 requires
|
|
# '+' but setup.py doesn't treat it consistently when generating the file name).
|
|
string(REGEX REPLACE "-.*" "" PKG_VERSION $ENV{IMPALA_VERSION})
|
|
set(SHELL_TEST_PKG
|
|
"${CMAKE_SOURCE_DIR}/shell/build/dist/impala_shell-${PKG_VERSION}.tar.gz")
|
|
get_filename_component(SHELL_TEST_PKG_DIR "${SHELL_TEST_PKG}" DIRECTORY)
|
|
# Generates SHELL_TEST_PKG
|
|
add_custom_target(shell_pypi_test_package
|
|
DEPENDS gen-deps shell_impala_build_version impala_python3
|
|
COMMAND env BUILD_VERSION=${PKG_VERSION} OFFICIAL=true DIST_DIR="${SHELL_TEST_PKG_DIR}"
|
|
"${CMAKE_SOURCE_DIR}/shell/packaging/make_python_package.sh"
|
|
)
|
|
|
|
# Tests expect to find venvs at 'python2_venv' and 'python3_venv' in tests/shell/util.py.
|
|
set(PYTHON2_VENV "${VENV_LOC}/python2_venv")
|
|
add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package
|
|
COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}"
|
|
)
|
|
|
|
set(PYTHON3_VENV "${VENV_LOC}/python3_venv")
|
|
add_custom_target(shell_python3_install DEPENDS python3_venv shell_pypi_test_package
|
|
COMMAND "${PYTHON3_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python3" "${SHELL_TEST_PKG}"
|
|
)
|
|
|
|
add_custom_target(shell_tarball DEPENDS gen-deps shell_pypi_test_package "${IMPALA_PYTHON_BUILD_VENVS}"
|
|
COMMAND "${CMAKE_SOURCE_DIR}/shell/packaging/make_shell_tarball.sh" "${SHELL_TEST_PKG}" ${PYTHON_EXES}
|
|
)
|