IMPALA-10274: Initialize impala-python as part of the CMake build

Initializing the impala-python virtualenv takes a couple minutes,
so it is useful to do that in parallel to the rest of the build.
This moves the impala-python initialization to its own step
in the CMake build. It stops using impala-python for commands
invoked from buildall.sh or the CMake build to avoid premature
or concurrent initializations of impala-python. Then, it adds
a dedicated step to initialize impala-python.

Testing:
 - Ran a core job and a couple builds
 - Rebuilt and verified that impala-python is not reinitialized
   if it is already initialized

Change-Id: Ieff51263c55bd234028fed7101c94b4a928590f0
Reviewed-on: http://gerrit.cloudera.org:8080/16607
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

CMakeLists.txt

@@ -448,8 +448,12 @@ add_custom_target(cscope ALL DEPENDS gen-deps
   COMMAND "${CMAKE_SOURCE_DIR}/bin/gen-cscope.sh"
 )
 
+add_custom_target(impala_python ALL
+  COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
+)
+
 add_custom_target(notests_independent_targets DEPENDS
-  java cscope tarballs
+  java cscope tarballs impala_python
 )
 add_custom_target(notests_regular_targets DEPENDS
   impalad statestored catalogd admissiond fesupport loggingsupport ImpalaUdf udasample udfsample

be/src/codegen/gen_ir_descriptions.py

@@ -1,4 +1,6 @@
-#!/usr/bin/env impala-python
+#!/usr/bin/env python
+# This uses system python to avoid a dependency on impala-python,
+# because this runs during the build.
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/gen_build_version.py

@@ -1,4 +1,6 @@
-#!/usr/bin/env impala-python
+#!/usr/bin/env python
+# This uses system python to avoid a dependency on impala-python,
+# because this runs during the build.
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/init-impala-python.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This is called during the build to initialize the impala-python
+# virtualenv (which involves installing various packages and
+# compiling things). This is not directly in CMake, because
+# this depends on knowing IMPALA_HOME and other environment
+# variables.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/impala-config.sh
+
+cd $IMPALA_HOME
+bin/impala-python -c 'print("Initialized impala-python")'

infra/python/bootstrap_virtualenv.py

@@ -207,7 +207,8 @@ def download_toolchain_python():
         "$IMPALA_TOOLCHAIN_PACKAGES_HOME is set.")
 
   package = ToolchainPackage("python")
-  if not (os.environ.get(SKIP_TOOLCHAIN_BOOTSTRAP) == 'true'):
+  if package.needs_download() and \
+     not (os.environ.get(SKIP_TOOLCHAIN_BOOTSTRAP) == 'true'):
     package.download()
   python_cmd = os.path.join(package.pkg_directory(), "bin/python")
   if not os.path.exists(python_cmd):