IMPALA-1071: Distributable python package for impala-shell

The patch adds a set of scripts for converting the impala-shell
into a true distributable python package. The package can be
installed using familiar python commands, e.g.:

  $ python setup.py (install|develop)

or

  $ pip install -e /path/to/dist/dir

The entry point script, make_python_package.sh, will run as a
part of the standard sequence of steps that results from calling
buildall.sh, and will produce a gzipped tarball inside of
Impala/shell/dist as an artifact. Thereafter, make_python_package.sh
can be run manually any time.

The expectation is that an official maintainer would need to manually
upload official releases to the Python Package Index as appropriate.

Change-Id: Ib8c745bddddf6a16f0c039430152745a2f00e044
Reviewed-on: http://gerrit.cloudera.org:8080/14181
Reviewed-by: David Knupp <dknupp@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

shell/packaging/make_python_package.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ----------------------------------------------------------------------
+# This script is invoked during the Impala build process, and creates
+# a distributable python package of the Impala shell. The resulting
+# archive will be saved to:
+#
+#   ${IMPALA_HOME}/shell/dist/impala_shell-<version>.tar.gz
+#
+# Until the thrift-generated python files in ${IMPALA_HOME}/shell/gen-py
+# have been created by the build process, this script will not work.
+# It also relies upon the impala_build_version.py file created by the
+# parent packaging script, ${IMPALA_HOME}/shell/make_shell_tarball.sh,
+# which needs to be run before this script will work.
+#
+# After those files exist, however, this script can be run again at will.
+
+set -eu -o pipefail
+
+WORKING_DIR="$(cd "$(dirname "$0")" ; pwd -P )"
+SHELL_HOME="${IMPALA_HOME}"/shell
+STAGING_DIR="${WORKING_DIR}"/staging
+DIST_DIR="${DIST_DIR:-$WORKING_DIR/dist}"
+PACKAGE_DIR="${STAGING_DIR}"/impala_shell_package
+MODULE_LIB_DIR="${PACKAGE_DIR}"/impala_shell
+NO_CLEAN_DIST="${NO_CLEAN_DIST:-}"
+
+assemble_package_files() {
+  mkdir -p "${MODULE_LIB_DIR}"
+
+  cp -r "${SHELL_HOME}/gen-py"/* "${MODULE_LIB_DIR}"
+  cp -r "${THRIFT_HOME}/python/lib/python2.7/site-packages/thrift" "${MODULE_LIB_DIR}"
+
+  cp "${WORKING_DIR}/__init__.py" "${MODULE_LIB_DIR}"
+  cp "${SHELL_HOME}/impala_shell.py" "${MODULE_LIB_DIR}"
+  cp "${SHELL_HOME}/impala_client.py" "${MODULE_LIB_DIR}"
+  cp "${SHELL_HOME}/option_parser.py" "${MODULE_LIB_DIR}"
+  cp "${SHELL_HOME}/shell_output.py" "${MODULE_LIB_DIR}"
+  cp "${SHELL_HOME}/impala_shell_config_defaults.py" "${MODULE_LIB_DIR}"
+  cp "${SHELL_HOME}/TSSLSocketWithWildcardSAN.py" "${MODULE_LIB_DIR}"
+
+  cp "${SHELL_HOME}/packaging/README.md" "${PACKAGE_DIR}"
+  cp "${SHELL_HOME}/packaging/MANIFEST.in" "${PACKAGE_DIR}"
+  cp "${SHELL_HOME}/packaging/requirements.txt" "${PACKAGE_DIR}"
+  cp "${SHELL_HOME}/packaging/setup.py" "${PACKAGE_DIR}"
+
+  cp "${IMPALA_HOME}/LICENSE.txt" "${PACKAGE_DIR}"
+}
+
+create_distributable_python_package() {
+  # Generate a new python package tarball in ${IMPALA_HOME}/shell/dist
+  if [[ "${NO_CLEAN_DIST}" != "true" ]]; then
+    rm -rf "${DIST_DIR}"
+  fi
+
+  mkdir -p "${DIST_DIR}"
+
+  pushd "${PACKAGE_DIR}"
+  echo "Building package..."
+  PACKAGE_TYPE="${PACKAGE_TYPE:-}" OFFICIAL="${OFFICIAL:-}" \
+    python setup.py sdist --dist-dir "${DIST_DIR}"
+  popd
+
+  if [[ "${NO_CLEAN_DIST}" != "true" ]]; then
+    rm -rf "${STAGING_DIR}"
+  fi
+}
+
+assemble_package_files
+create_distributable_python_package