IMPALA-13384: Only install gcovr deps for coverage builds

IMPALA-13279 upgraded gcovr to 7.2 and moved it from python 2 to
python 3.8. gcovr has several dependencies that require native
compilation, and this increased the cost of initializing the
Python 3 virtualenv substantially:

Without gcovr: 1m43.279s
With gcovr and deps: 6m35.107s

This moves gcovr to its own requirements file and only installs
gcovr if this is a coverage build (detected from the
.cmake_buid_type file).

Testing:
 - Verified that a coverage build does install gcovr and
   produce a report

Change-Id: I1d0fd6d21273053aaf2acee39fcb83d9093d49a2
Reviewed-on: http://gerrit.cloudera.org:8080/21849
Reviewed-by: Laszlo Gaal <laszlo.gaal@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

infra/python/bootstrap_virtualenv.py

@@ -28,6 +28,7 @@
 # 2. Install most packages (including ones that require C/C++ compilation)
 # 3. Install Kudu package (which uses the toolchain GCC and the installed Cython)
 # 4. Install ADLS packages if applicable
+# 5. Install GCOVR packages if this is a code coverage build
 #
 # This module can be run with python >= 2.7. It makes no guarantees about usage on
 # python < 2.7.
@@ -54,6 +55,8 @@ SKIP_TOOLCHAIN_BOOTSTRAP = "SKIP_TOOLCHAIN_BOOTSTRAP"
 
 GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]
 
+IMPALA_HOME = os.environ["IMPALA_HOME"]
+
 DEPS_DIR = os.path.join(os.path.dirname(__file__), "deps")
 ENV_DIR_PY2 = os.path.join(os.path.dirname(__file__),
                            "env-gcc{0}".format(GCC_VERSION))
@@ -80,6 +83,11 @@ KUDU_REQS_PATH = os.path.join(DEPS_DIR, "kudu-requirements.txt")
 # Interface) being installed by the requirements step.
 ADLS_REQS_PATH = os.path.join(DEPS_DIR, "adls-requirements.txt")
 
+# Requirements for the gcovr utility. These add several minutes to initializing the
+# virtualenv, so they are split off into their own step that only runs when coverage
+# is enabled.
+GCOVR_REQS_PATH = os.path.join(DEPS_DIR, "gcovr-requirements.txt")
+
 # Extra packages specific to python 3
 PY3_REQS_PATH = os.path.join(DEPS_DIR, "py3-requirements.txt")
 
@@ -318,6 +326,35 @@ def install_adls_deps(venv_dir, is_py3):
     mark_reqs_installed(venv_dir, ADLS_REQS_PATH)
 
 
+def install_gcovr_deps(venv_dir, is_py3):
+  # Gcovr is only installed in the python3 virtualenv
+  if not is_py3:
+    return
+  if not reqs_are_installed(venv_dir, GCOVR_REQS_PATH):
+    # Gcovr takes several minutes to install, so we only install it if this is a coverage
+    # build. We detect a coverage build by reading ${IMPALA_HOME}/.cmake_build_type.
+    # The python virtualenv is typically initialized during the main build, and CMake
+    # writes .cmake_build_type before the build starts. If that file doesn't exist
+    # (usually because impala-python3 is being run manually), don't install gcovr. Future
+    # invocations will check again and can install it if needed.
+    cmake_build_type_file = os.path.join(IMPALA_HOME, ".cmake_build_type")
+    if not os.path.isfile(cmake_build_type_file):
+      return
+    coverage_enabled = False
+    with open(cmake_build_type_file) as f:
+      for line in f:
+        if line.find("COVERAGE") != -1:
+          coverage_enabled = True
+          break
+
+    if coverage_enabled:
+      cc = select_cc()
+      assert cc is not None
+      LOG.info("Installing gcovr packages into the python3 virtualenv")
+      exec_pip_install(venv_dir, is_py3, ["-r", GCOVR_REQS_PATH], cc=cc)
+      mark_reqs_installed(venv_dir, GCOVR_REQS_PATH)
+
+
 def install_py_version_deps(venv_dir, is_py3):
   cc = select_cc()
   assert cc is not None
@@ -485,3 +522,4 @@ if __name__ == "__main__":
   install_kudu_client_if_possible(venv_dir, options.python3)
   install_adls_deps(venv_dir, options.python3)
   install_py_version_deps(venv_dir, options.python3)
+  install_gcovr_deps(venv_dir, options.python3)

infra/python/deps/gcovr-requirements.txt

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Requirements for gcovr, which are only needed for coverage runs
+
+gcovr == 7.2
+  Jinja2 == 3.1.4
+    flit-core == 3.9.0
+  lxml == 5.2.2
+    Cython == 3.0.10
+  colorlog == 6.8.2
+  Pygments == 2.13.0
+    MarkupSafe == 2.1.5
+  tomli == 2.0.1
+    packaging == 24.1

infra/python/deps/pip_download.py

@@ -39,7 +39,8 @@ PYPI_MIRROR = os.environ.get('PYPI_MIRROR', 'https://pypi.python.org')
 # The requirement files that list all of the required packages and versions.
 REQUIREMENTS_FILES = ['requirements.txt', 'setuptools-requirements.txt',
                       'kudu-requirements.txt', 'adls-requirements.txt',
-                      'py2-requirements.txt', 'py3-requirements.txt']
+                      'py2-requirements.txt', 'py3-requirements.txt',
+                      'gcovr-requirements.txt']
 
 
 def check_digest(filename, algorithm, expected_digest):

infra/python/deps/py3-requirements.txt

@@ -30,13 +30,3 @@ pylint == 2.10.2
   platformdirs == 2.4.1
   typing-extensions == 3.10.0.2
 k5test==0.10.3
-gcovr == 7.2
-  Jinja2 == 3.1.4
-    flit-core == 3.9.0
-  lxml == 5.2.2
-    Cython == 3.0.10
-  colorlog == 6.8.2
-  Pygments == 2.13.0
-    MarkupSafe == 2.1.5
-  tomli == 2.0.1
-    packaging == 24.1