mirror of
https://github.com/apache/impala.git
synced 2026-01-10 09:00:16 -05:00
aefd1b0920
Currently, the shell tarball maintains its own packaging code
and directory layout. This is very complicated and currently has
several Python packages directly checked into our repository.
To simplify it, this changes the shell tarball to be based on
pip installing the pypi package. Specifically, the new directory
structure for an unpack shell tarball is:
impala-shell-4.5.0-SNAPSHOT/
impala-shell
install_py${PYTHON_VERSION}/
install_py${ANOTHER_PYTHON_VERSION}/
For example, install_py2.7 is the Python 2.7 pip install of impala-shell.
install_py3.8 is a Python 3.8 pip install of impala-shell. This means
that the impala-shell script simply picks the install for the
specified version of python and uses that pip install directory.
To make this more consistent across different Linux distributions, this
upgrades pip in the virtualenv to the latest.
With this, ext-py and pkg_resources.py can be removed.
This requires rearranging the shell build code. Specifically, this splits
out the code that generates impala_build_version.py so that it can run
before generating the pypi package. The shell tarball now has a dependency
on the pypi package and must run after it.
This builds on Michael Smith's work from IMPALA-11399.
Testing:
- Ran shell tests locally
- Built on Centos 7, Redhat 8 & 9, Ubuntu 20 & 22, SLES 15
Change-Id: Ifbb66ab2c5bc7180221f98d9bf5e38d62f4ac036
Reviewed-on: http://gerrit.cloudera.org:8080/20171
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
80 lines
2.9 KiB
Bash
Executable File
80 lines
2.9 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
|
|
# This script makes a tarball of the Python-based shell that can be unzipped and
|
|
# run out-of-the-box with no configuration. The final tarball is left in
|
|
# ${IMPALA_HOME}/shell/build.
|
|
|
|
set -euo pipefail
|
|
. $IMPALA_HOME/bin/report_build_error.sh
|
|
setup_report_build_error
|
|
|
|
if [ "x${IMPALA_HOME}" == "x" ]; then
|
|
echo "\$IMPALA_HOME must be set"
|
|
exit 1
|
|
fi
|
|
|
|
if [ $# -lt 2 ]; then
|
|
echo "Must provide pypi package and at least one python interpreter"
|
|
exit 1
|
|
fi
|
|
|
|
PYPI_PACKAGE=$1
|
|
shift
|
|
SHELL_HOME=${IMPALA_HOME}/shell
|
|
BUILD_DIR=${SHELL_HOME}/build
|
|
TARBALL_ROOT=${BUILD_DIR}/impala-shell-${IMPALA_VERSION}
|
|
|
|
for PYTHON_EXE in $*; do
|
|
PYTHON_NAME=$(basename ${PYTHON_EXE})
|
|
PYTHON_VERSION=$(${PYTHON_EXE} -c 'import sys; \
|
|
print("{}.{}".format(sys.version_info.major, sys.version_info.minor))')
|
|
PYTHON_MAJOR_VERSION=$(${PYTHON_EXE} -c 'import sys; print(sys.version_info.major)')
|
|
# pip install the wheel into the external dependencies directory
|
|
PIP_CACHE="~/.cache/impala_pip/${PYTHON_NAME}"
|
|
BUILD_TMP_DIR="$(mktemp -d)"
|
|
|
|
echo "Deleting all files in ${TARBALL_ROOT}/install_py${PYTHON_VERSION}"
|
|
rm -rf ${TARBALL_ROOT}/install_py${PYTHON_VERSION} 2>&1 > /dev/null
|
|
echo "Installing for python ${PYTHON_VERSION}"
|
|
# Use pip that matches the major version
|
|
if [[ $PYTHON_MAJOR_VERSION == 2 ]]; then
|
|
source ${IMPALA_HOME}/shell/build/python2_venv/bin/activate
|
|
else
|
|
source ${IMPALA_HOME}/shell/build/python3_venv/bin/activate
|
|
fi
|
|
mkdir -p ${TARBALL_ROOT}/install_py${PYTHON_VERSION}
|
|
pip install --cache ${PIP_CACHE} \
|
|
--target ${TARBALL_ROOT}/install_py${PYTHON_VERSION} ${PYPI_PACKAGE}
|
|
# We don't need the impala-shell binary for the installation. It contains
|
|
# a weird shebang from the virtualenv, so it is worth removing it.
|
|
rm ${TARBALL_ROOT}/install_py${PYTHON_VERSION}/bin/impala-shell
|
|
# Cleanup temp build directory
|
|
rm -rf ${BUILD_TMP_DIR}
|
|
done
|
|
|
|
# Copy the impala-shell driver script into the tarball root
|
|
cp ${SHELL_HOME}/impala-shell ${TARBALL_ROOT}
|
|
|
|
pushd ${BUILD_DIR} > /dev/null
|
|
echo "Making tarball in ${BUILD_DIR}"
|
|
tar czf ${BUILD_DIR}/impala-shell-${IMPALA_VERSION}.tar.gz --exclude="*.pyc" \
|
|
./impala-shell-${IMPALA_VERSION}/ || popd 2>&1 > /dev/null
|