mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
c3ec9272c5
Updates the test environment to default to the CDP build of Ozone, as the latest build of CDP Hive depends on pre-release features unavailable in Ozone 1.2.1. Apache Ozone 1.2 can still be used by setting USE_APACHE_OZONE=true. The latest CDP build also includes a version of Ozone based on ozone#master with a candidate version of 1.3.0. Both Apache and CDP therefore have builds of Ozone we can test with that use the new artifact names introduced in Ozone 1.2, so this patch cleans up setup that was only needed for Ozone versions prior to 1.2. Change-Id: I1177a1b820fe21adca9f8c1cc51ff73ee001d3f2 Reviewed-on: http://gerrit.cloudera.org:8080/19247 Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com> Tested-by: Michael Smith <michael.smith@cloudera.com>
600 lines
27 KiB
Python
Executable File
600 lines
27 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
# The purpose of this script is to download prebuilt binaries and jar files to satisfy
|
|
# the third-party dependencies for Impala. The script expects bin/impala-config.sh to be
|
|
# sourced to initialize various environment variables (including environment variables
|
|
# specifying the versions for components). It verifies that bin/impala-config.sh
|
|
# has been sourced by verifying that IMPALA_HOME is set. This script will fail if an
|
|
# expected environment variable is not present.
|
|
#
|
|
# To share a toolchain directory between multiple checkouts of Impala (or to use a
|
|
# cached copy to avoid downloading a new one), it is best to override IMPALA_TOOLCHAIN
|
|
# in bin/impala-config-local.sh or in the environment prior to sourcing
|
|
# bin/impala-config.sh. This sets IMPALA_TOOLCHAIN_PACKAGES_HOME as well as
|
|
# CDP_COMPONENTS_HOME.
|
|
#
|
|
# The following environment variables control the behavior of this script:
|
|
# IMPALA_TOOLCHAIN_PACKAGES_HOME - Directory in which to place the native-toolchain
|
|
# packages.
|
|
# IMPALA_TOOLCHAIN_HOST - The host to use for downloading the artifacts
|
|
# CDP_COMPONENTS_HOME - Directory to store CDP Hadoop component artifacts
|
|
# CDP_BUILD_NUMBER - CDP Hadoop components are built with consistent versions so that
|
|
# Hadoop, Hive, Kudu, etc are all built with versions that are compatible with each
|
|
# other. The way to specify a single consistent set of components is via a build
|
|
# number. This determines the location in s3 to get the artifacts.
|
|
# DOWNLOAD_CDH_COMPONENTS - When set to true, this script will also download and extract
|
|
# the CDP Hadoop components (i.e. Hadoop, Hive, HBase, Ranger, Ozone, etc) into
|
|
# CDP_COMPONENTS_HOME as appropriate.
|
|
# IMPALA_<PACKAGE>_VERSION - The version expected for <PACKAGE>. This is typically
|
|
# configured in bin/impala-config.sh and must exist for every package. This is used
|
|
# to construct an appropriate URL and expected archive name.
|
|
# IMPALA_<PACKAGE>_URL - This overrides the download URL for <PACKAGE>. The URL must
|
|
# end with the expected archive name for this package. This is usually used in
|
|
# bin/impala-config-branch.sh or bin/impala-config-local.sh when using custom
|
|
# versions or packages. When this is not specified, packages are downloaded from
|
|
# an S3 bucket named native-toolchain, and the exact URL is based on
|
|
# IMPALA_<PACKAGE>_VERSION as well as the OS version being built on.
|
|
#
|
|
# The script is directly executable, and it takes no parameters:
|
|
# ./bootstrap_toolchain.py
|
|
import logging
|
|
import glob
|
|
import multiprocessing.pool
|
|
import os
|
|
import platform
|
|
import random
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
from collections import namedtuple
|
|
from string import Template
|
|
|
|
# Maps return values from 'lsb_release -irs' to the corresponding OS labels for both the
|
|
# toolchain and the CDP components.
|
|
OsMapping = namedtuple('OsMapping', ['lsb_release', 'toolchain', 'cdh'])
|
|
OS_MAPPING = [
|
|
OsMapping("centos5", "ec2-package-centos-5", None),
|
|
OsMapping("centos6", "ec2-package-centos-6", "redhat6"),
|
|
OsMapping("centos7", "ec2-package-centos-7", "redhat7"),
|
|
OsMapping("centos8", "ec2-package-centos-8", "redhat8"),
|
|
OsMapping("rocky8", "ec2-package-centos-8", "redhat8"),
|
|
OsMapping("almalinux8", "ec2-package-centos-8", "redhat8"),
|
|
OsMapping("redhatenterpriseserver5", "ec2-package-centos-5", None),
|
|
OsMapping("redhatenterpriseserver6", "ec2-package-centos-6", "redhat6"),
|
|
OsMapping("redhatenterpriseserver7", "ec2-package-centos-7", "redhat7"),
|
|
OsMapping("redhatenterprise8", "ec2-package-centos-8", "redhat8"),
|
|
OsMapping("redhatenterpriseserver8", "ec2-package-centos-8", "redhat8"),
|
|
OsMapping("debian6", "ec2-package-debian-6", None),
|
|
OsMapping("debian7", "ec2-package-debian-7", None),
|
|
OsMapping("debian8", "ec2-package-debian-8", "debian8"),
|
|
OsMapping("suselinux11", "ec2-package-sles-11", None),
|
|
OsMapping("suselinux12", "ec2-package-sles-12", "sles12"),
|
|
OsMapping("suse12", "ec2-package-sles-12", "sles12"),
|
|
OsMapping("ubuntu12.04", "ec2-package-ubuntu-12-04", None),
|
|
OsMapping("ubuntu14.04", "ec2-package-ubuntu-14-04", None),
|
|
OsMapping("ubuntu15.04", "ec2-package-ubuntu-14-04", None),
|
|
OsMapping("ubuntu15.10", "ec2-package-ubuntu-14-04", None),
|
|
OsMapping('ubuntu16.04', "ec2-package-ubuntu-16-04", "ubuntu1604"),
|
|
OsMapping('ubuntu18.04', "ec2-package-ubuntu-18-04", "ubuntu1804"),
|
|
OsMapping('ubuntu20.04', "ec2-package-ubuntu-20-04", "ubuntu2004")
|
|
]
|
|
|
|
|
|
def check_output(cmd_args):
|
|
"""Run the command and return the output. Raise an exception if the command returns
|
|
a non-zero return code. Similar to subprocess.check_output() which is only provided
|
|
in python 2.7.
|
|
"""
|
|
process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
stdout, _ = process.communicate()
|
|
if process.wait() != 0:
|
|
raise Exception("Command with args '%s' failed with exit code %s:\n%s"
|
|
% (cmd_args, process.returncode, stdout))
|
|
return stdout
|
|
|
|
|
|
def get_toolchain_compiler():
|
|
"""Return the <name>-<version> string for the compiler package to use for the
|
|
toolchain."""
|
|
# Currently we always use GCC.
|
|
return "gcc-{0}".format(os.environ["IMPALA_GCC_VERSION"])
|
|
|
|
|
|
def wget_and_unpack_package(download_path, file_name, destination, wget_no_clobber):
|
|
if not download_path.endswith("/" + file_name):
|
|
raise Exception("URL {0} does not match with expected file_name {1}"
|
|
.format(download_path, file_name))
|
|
if "closer.cgi" in download_path:
|
|
download_path += "?action=download"
|
|
NUM_ATTEMPTS = 3
|
|
for attempt in range(1, NUM_ATTEMPTS + 1):
|
|
logging.info("Downloading {0} to {1}/{2} (attempt {3})".format(
|
|
download_path, destination, file_name, attempt))
|
|
# --no-clobber avoids downloading the file if a file with the name already exists
|
|
try:
|
|
cmd = ["wget", download_path,
|
|
"--output-document={0}/{1}".format(destination, file_name)]
|
|
if wget_no_clobber:
|
|
cmd.append("--no-clobber")
|
|
check_output(cmd)
|
|
break
|
|
except Exception, e:
|
|
if attempt == NUM_ATTEMPTS:
|
|
raise
|
|
logging.error("Download failed; retrying after sleep: " + str(e))
|
|
time.sleep(10 + random.random() * 5) # Sleep between 10 and 15 seconds.
|
|
logging.info("Extracting {0}".format(file_name))
|
|
check_output(["tar", "xzf", os.path.join(destination, file_name),
|
|
"--directory={0}".format(destination)])
|
|
os.unlink(os.path.join(destination, file_name))
|
|
|
|
|
|
class DownloadUnpackTarball(object):
|
|
"""
|
|
The basic unit of work for bootstrapping the toolchain is:
|
|
- check if a package is already present (via the needs_download() method)
|
|
- if it is not, download a tarball and unpack it into the appropriate directory
|
|
(via the download() method)
|
|
In this base case, everything is known: the url to download from, the archive to
|
|
unpack, and the destination directory.
|
|
"""
|
|
def __init__(self, url, archive_name, destination_basedir, directory_name, makedir):
|
|
self.url = url
|
|
self.archive_name = archive_name
|
|
assert self.archive_name.endswith(".tar.gz")
|
|
self.archive_basename = self.archive_name.replace(".tar.gz", "")
|
|
self.destination_basedir = destination_basedir
|
|
# destination base directory must exist
|
|
assert os.path.isdir(self.destination_basedir)
|
|
self.directory_name = directory_name
|
|
self.makedir = makedir
|
|
|
|
def pkg_directory(self):
|
|
return os.path.join(self.destination_basedir, self.directory_name)
|
|
|
|
def needs_download(self):
|
|
if os.path.isdir(self.pkg_directory()): return False
|
|
return True
|
|
|
|
def download(self):
|
|
unpack_dir = self.pkg_directory()
|
|
if self.makedir:
|
|
# Download and unpack in a temp directory, which we'll later move into place
|
|
download_dir = tempfile.mkdtemp(dir=self.destination_basedir)
|
|
else:
|
|
download_dir = self.destination_basedir
|
|
try:
|
|
wget_and_unpack_package(self.url, self.archive_name, download_dir, False)
|
|
except: # noqa
|
|
# Clean up any partially-unpacked result.
|
|
if os.path.isdir(unpack_dir):
|
|
shutil.rmtree(unpack_dir)
|
|
# Only delete the download directory if it is a temporary directory
|
|
if download_dir != self.destination_basedir and os.path.isdir(download_dir):
|
|
shutil.rmtree(download_dir)
|
|
raise
|
|
if self.makedir:
|
|
os.rename(download_dir, unpack_dir)
|
|
|
|
|
|
class TemplatedDownloadUnpackTarball(DownloadUnpackTarball):
|
|
def __init__(self, url_tmpl, archive_name_tmpl, destination_basedir_tmpl,
|
|
directory_name_tmpl, makedir, template_subs):
|
|
url = self.__do_substitution(url_tmpl, template_subs)
|
|
archive_name = self.__do_substitution(archive_name_tmpl, template_subs)
|
|
destination_basedir = self.__do_substitution(destination_basedir_tmpl, template_subs)
|
|
directory_name = self.__do_substitution(directory_name_tmpl, template_subs)
|
|
super(TemplatedDownloadUnpackTarball, self).__init__(url, archive_name,
|
|
destination_basedir, directory_name, makedir)
|
|
|
|
def __do_substitution(self, template, template_subs):
|
|
return Template(template).substitute(**template_subs)
|
|
|
|
|
|
class EnvVersionedPackage(TemplatedDownloadUnpackTarball):
|
|
def __init__(self, name, url_prefix_tmpl, destination_basedir, explicit_version=None,
|
|
archive_basename_tmpl=None, unpack_directory_tmpl=None, makedir=False,
|
|
template_subs_in={}, target_comp=None):
|
|
template_subs = template_subs_in
|
|
template_subs["name"] = name
|
|
template_subs["version"] = self.__compute_version(name, explicit_version,
|
|
target_comp)
|
|
# The common case is that X.tar.gz unpacks to X directory. archive_basename_tmpl
|
|
# allows overriding the value of X (which defaults to ${name}-${version}).
|
|
# If X.tar.gz unpacks to Y directory, then unpack_directory_tmpl allows overriding Y.
|
|
if archive_basename_tmpl is None:
|
|
archive_basename_tmpl = "${name}-${version}"
|
|
archive_name_tmpl = archive_basename_tmpl + ".tar.gz"
|
|
if unpack_directory_tmpl is None:
|
|
unpack_directory_tmpl = archive_basename_tmpl
|
|
url_tmpl = self.__compute_url(name, archive_name_tmpl, url_prefix_tmpl, target_comp)
|
|
super(EnvVersionedPackage, self).__init__(url_tmpl, archive_name_tmpl,
|
|
destination_basedir, unpack_directory_tmpl, makedir, template_subs)
|
|
|
|
def __compute_version(self, name, explicit_version, target_comp=None):
|
|
if explicit_version is not None:
|
|
return explicit_version
|
|
else:
|
|
# When getting the version from the environment, we need to standardize the name
|
|
# to match expected environment variables.
|
|
std_env_name = name.replace("-", "_").upper()
|
|
if target_comp:
|
|
std_env_name += '_' + target_comp.upper()
|
|
version_env_var = "IMPALA_{0}_VERSION".format(std_env_name)
|
|
env_version = os.environ.get(version_env_var)
|
|
if not env_version:
|
|
raise Exception("Could not find version for {0} in environment var {1}".format(
|
|
name, version_env_var))
|
|
return env_version
|
|
|
|
def __compute_url(self, name, archive_name_tmpl, url_prefix_tmpl, target_comp=None):
|
|
# The URL defined in the environment (IMPALA_*_URL) takes precedence. If that is
|
|
# not defined, use the standard URL (url_prefix + archive_name)
|
|
std_env_name = name.replace("-", "_").upper()
|
|
if target_comp:
|
|
std_env_name += '_' + target_comp.upper()
|
|
url_env_var = "IMPALA_{0}_URL".format(std_env_name)
|
|
url_tmpl = os.environ.get(url_env_var)
|
|
if not url_tmpl:
|
|
url_tmpl = os.path.join(url_prefix_tmpl, archive_name_tmpl)
|
|
return url_tmpl
|
|
|
|
|
|
class ToolchainPackage(EnvVersionedPackage):
|
|
def __init__(self, name, explicit_version=None, platform_release=None):
|
|
toolchain_packages_home = os.environ.get("IMPALA_TOOLCHAIN_PACKAGES_HOME")
|
|
if not toolchain_packages_home:
|
|
logging.error("Impala environment not set up correctly, make sure "
|
|
"$IMPALA_TOOLCHAIN_PACKAGES_HOME is set.")
|
|
sys.exit(1)
|
|
target_comp = None
|
|
if ":" in name:
|
|
parts = name.split(':')
|
|
name = parts[0]
|
|
target_comp = parts[1]
|
|
compiler = get_toolchain_compiler()
|
|
label = get_platform_release_label(release=platform_release).toolchain
|
|
toolchain_build_id = os.environ["IMPALA_TOOLCHAIN_BUILD_ID"]
|
|
toolchain_host = os.environ["IMPALA_TOOLCHAIN_HOST"]
|
|
template_subs = {'compiler': compiler, 'label': label,
|
|
'toolchain_build_id': toolchain_build_id,
|
|
'toolchain_host': toolchain_host}
|
|
archive_basename_tmpl = "${name}-${version}-${compiler}-${label}"
|
|
url_prefix_tmpl = "https://${toolchain_host}/build/${toolchain_build_id}/" + \
|
|
"${name}/${version}-${compiler}/"
|
|
unpack_directory_tmpl = "${name}-${version}"
|
|
super(ToolchainPackage, self).__init__(name, url_prefix_tmpl,
|
|
toolchain_packages_home,
|
|
explicit_version=explicit_version,
|
|
archive_basename_tmpl=archive_basename_tmpl,
|
|
unpack_directory_tmpl=unpack_directory_tmpl,
|
|
template_subs_in=template_subs,
|
|
target_comp=target_comp)
|
|
|
|
def needs_download(self):
|
|
# If the directory doesn't exist, we need the download
|
|
unpack_dir = self.pkg_directory()
|
|
if not os.path.isdir(unpack_dir): return True
|
|
version_file = os.path.join(unpack_dir, "toolchain_package_version.txt")
|
|
if not os.path.exists(version_file): return True
|
|
with open(version_file, "r") as f:
|
|
return f.read().strip() != self.archive_basename
|
|
|
|
def download(self):
|
|
# Remove the existing package directory if it exists (since this has additional
|
|
# conditions as part of needs_download())
|
|
unpack_dir = self.pkg_directory()
|
|
if os.path.exists(unpack_dir):
|
|
logging.info("Removing existing package directory {0}".format(unpack_dir))
|
|
shutil.rmtree(unpack_dir)
|
|
super(ToolchainPackage, self).download()
|
|
# Write the toolchain_package_version.txt file
|
|
version_file = os.path.join(unpack_dir, "toolchain_package_version.txt")
|
|
with open(version_file, "w") as f:
|
|
f.write(self.archive_basename)
|
|
|
|
|
|
class CdpComponent(EnvVersionedPackage):
|
|
def __init__(self, name, explicit_version=None, archive_basename_tmpl=None,
|
|
unpack_directory_tmpl=None, makedir=False):
|
|
# Compute the CDP base URL (based on the IMPALA_TOOLCHAIN_HOST and CDP_BUILD_NUMBER)
|
|
if "IMPALA_TOOLCHAIN_HOST" not in os.environ or "CDP_BUILD_NUMBER" not in os.environ:
|
|
logging.error("Impala environment not set up correctly, make sure "
|
|
"impala-config.sh is sourced.")
|
|
sys.exit(1)
|
|
template_subs = {"toolchain_host": os.environ["IMPALA_TOOLCHAIN_HOST"],
|
|
"cdp_build_number": os.environ["CDP_BUILD_NUMBER"]}
|
|
url_prefix_tmpl = "https://${toolchain_host}/build/cdp_components/" + \
|
|
"${cdp_build_number}/tarballs/"
|
|
|
|
# Get the output base directory from CDP_COMPONENTS_HOME
|
|
destination_basedir = os.environ["CDP_COMPONENTS_HOME"]
|
|
super(CdpComponent, self).__init__(name, url_prefix_tmpl, destination_basedir,
|
|
explicit_version=explicit_version,
|
|
archive_basename_tmpl=archive_basename_tmpl,
|
|
unpack_directory_tmpl=unpack_directory_tmpl,
|
|
makedir=makedir, template_subs_in=template_subs)
|
|
|
|
|
|
class ApacheComponent(EnvVersionedPackage):
|
|
def __init__(self, name, explicit_version=None, archive_basename_tmpl=None,
|
|
unpack_directory_tmpl=None, makedir=False, component_path_tmpl=None):
|
|
# Compute the apache base URL (based on the APACHE_MIRROR)
|
|
if "APACHE_COMPONENTS_HOME" not in os.environ:
|
|
logging.error("Impala environment not set up correctly, make sure "
|
|
"impala-config.sh is sourced.")
|
|
sys.exit(1)
|
|
template_subs = {"apache_mirror": os.environ["APACHE_MIRROR"]}
|
|
# Different components have different sub-paths. For example, hive is hive/hive-xxx,
|
|
# hadoop is hadoop/common/hadoop-xxx. The default is hive format.
|
|
if component_path_tmpl is None:
|
|
component_path_tmpl = "${name}/${name}-${version}/"
|
|
url_prefix_tmpl = "${apache_mirror}/" + component_path_tmpl
|
|
|
|
# Get the output base directory from APACHE_COMPONENTS_HOME
|
|
destination_basedir = os.environ["APACHE_COMPONENTS_HOME"]
|
|
super(ApacheComponent, self).__init__(name, url_prefix_tmpl, destination_basedir,
|
|
explicit_version=explicit_version,
|
|
archive_basename_tmpl=archive_basename_tmpl,
|
|
unpack_directory_tmpl=unpack_directory_tmpl,
|
|
makedir=makedir, template_subs_in=template_subs)
|
|
|
|
class ToolchainKudu(ToolchainPackage):
|
|
def __init__(self, platform_label=None):
|
|
super(ToolchainKudu, self).__init__('kudu', platform_release=platform_label)
|
|
|
|
def needs_download(self):
|
|
# This verifies that the unpack directory exists
|
|
if super(ToolchainKudu, self).needs_download():
|
|
return True
|
|
# Additional check to distinguish this from the Kudu Java package
|
|
# Regardless of the actual build type, the 'kudu' tarball will always contain a
|
|
# 'debug' and a 'release' directory.
|
|
if not os.path.exists(os.path.join(self.pkg_directory(), "debug")):
|
|
return True
|
|
# Both the pkg_directory and the debug directory exist
|
|
return False
|
|
|
|
|
|
def try_get_platform_release_label():
|
|
"""Gets the right package label from the OS version. Returns an OsMapping with both
|
|
'toolchain' and 'cdh' labels. Return None if not found.
|
|
"""
|
|
try:
|
|
return get_platform_release_label()
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
# Cache "lsb_release -irs" to avoid excessive logging from sh, and
|
|
# to shave a little bit of time.
|
|
lsb_release_cache = None
|
|
|
|
|
|
def get_platform_release_label(release=None):
|
|
"""Gets the right package label from the OS version. Raise exception if not found.
|
|
'release' can be provided to override the underlying OS version.
|
|
"""
|
|
global lsb_release_cache
|
|
if not release:
|
|
if lsb_release_cache:
|
|
release = lsb_release_cache
|
|
else:
|
|
lsb_release = check_output(["lsb_release", "-irs"])
|
|
release = "".join(map(lambda x: x.lower(), lsb_release.split()))
|
|
# Only need to check against the major release if RHEL, CentOS or Suse
|
|
for distro in ['centos', 'rocky', 'almalinux', 'redhatenterprise',
|
|
'redhatenterpriseserver', 'suse']:
|
|
if distro in release:
|
|
release = release.split('.')[0]
|
|
break
|
|
lsb_release_cache = release
|
|
for mapping in OS_MAPPING:
|
|
if re.search(mapping.lsb_release, release):
|
|
return mapping
|
|
raise Exception("Could not find package label for OS version: {0}.".format(release))
|
|
|
|
|
|
def check_output(cmd_args):
|
|
"""Run the command and return the output. Raise an exception if the command returns
|
|
a non-zero return code. Similar to subprocess.check_output() which is only provided
|
|
in python 2.7.
|
|
"""
|
|
process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
stdout, _ = process.communicate()
|
|
if process.wait() != 0:
|
|
raise Exception("Command with args '%s' failed with exit code %s:\n%s"
|
|
% (cmd_args, process.returncode, stdout))
|
|
return stdout
|
|
|
|
|
|
def check_custom_toolchain(toolchain_packages_home, packages):
|
|
missing = []
|
|
for p in packages:
|
|
if not os.path.isdir(p.pkg_directory()):
|
|
missing.append((p, p.pkg_directory()))
|
|
|
|
if missing:
|
|
msg = "The following packages are not in their expected locations.\n"
|
|
for p, pkg_dir in missing:
|
|
msg += " %s (expected directory %s to exist)\n" % (p, pkg_dir)
|
|
msg += "Pre-built toolchain archives not available for your platform.\n"
|
|
msg += "Clone and build native toolchain from source using this repository:\n"
|
|
msg += " https://github.com/cloudera/native-toolchain\n"
|
|
logging.error(msg)
|
|
raise Exception("Toolchain bootstrap failed: required packages were missing")
|
|
|
|
|
|
def execute_many(f, args):
|
|
"""
|
|
Executes f(a) for a in args using a threadpool to execute in parallel.
|
|
The pool uses the smaller of 4 and the number of CPUs in the system
|
|
as the pool size.
|
|
"""
|
|
pool = multiprocessing.pool.ThreadPool(processes=min(multiprocessing.cpu_count(), 4))
|
|
return pool.map(f, args, 1)
|
|
|
|
|
|
def create_directory_from_env_var(env_var):
|
|
dir_name = os.environ.get(env_var)
|
|
if not dir_name:
|
|
logging.error("Impala environment not set up correctly, make sure "
|
|
"{0} is set.".format(env_var))
|
|
sys.exit(1)
|
|
if not os.path.exists(dir_name):
|
|
os.makedirs(dir_name)
|
|
|
|
|
|
def get_unique_toolchain_downloads(packages):
|
|
toolchain_packages = map(ToolchainPackage, packages)
|
|
unique_pkg_directories = set()
|
|
unique_packages = []
|
|
for p in toolchain_packages:
|
|
if p.pkg_directory() not in unique_pkg_directories:
|
|
unique_packages.append(p)
|
|
unique_pkg_directories.add(p.pkg_directory())
|
|
return unique_packages
|
|
|
|
|
|
def get_toolchain_downloads():
|
|
toolchain_packages = []
|
|
# The LLVM and GCC packages are the largest packages in the toolchain (Kudu is handled
|
|
# separately). Sort them first so their downloads start as soon as possible.
|
|
llvm_package = ToolchainPackage("llvm")
|
|
llvm_package_asserts = ToolchainPackage(
|
|
"llvm", explicit_version=os.environ.get("IMPALA_LLVM_DEBUG_VERSION"))
|
|
gcc_package = ToolchainPackage("gcc")
|
|
toolchain_packages += [llvm_package, llvm_package_asserts, gcc_package]
|
|
toolchain_packages += map(ToolchainPackage,
|
|
["avro", "binutils", "boost", "breakpad", "bzip2", "calloncehack", "cctz", "cmake",
|
|
"crcutil", "curl", "flatbuffers", "gdb", "gflags", "glog", "gperftools", "gtest",
|
|
"jwt-cpp", "libev", "libunwind", "lz4", "openldap", "openssl", "orc", "protobuf",
|
|
"python", "rapidjson", "re2", "snappy", "tpc-h", "tpc-ds", "zlib", "zstd"])
|
|
toolchain_packages += get_unique_toolchain_downloads(
|
|
["thrift:cpp", "thrift:java", "thrift:py"])
|
|
protobuf_package_clang = ToolchainPackage(
|
|
"protobuf", explicit_version=os.environ.get("IMPALA_PROTOBUF_CLANG_VERSION"))
|
|
toolchain_packages += [protobuf_package_clang]
|
|
# Check whether this platform is supported (or whether a valid custom toolchain
|
|
# has been provided).
|
|
if not try_get_platform_release_label() \
|
|
or not try_get_platform_release_label().toolchain:
|
|
toolchain_packages_home = os.environ.get("IMPALA_TOOLCHAIN_PACKAGES_HOME")
|
|
# This would throw an exception if the custom toolchain were not valid
|
|
check_custom_toolchain(toolchain_packages_home, toolchain_packages)
|
|
# Nothing to download
|
|
return []
|
|
return toolchain_packages
|
|
|
|
|
|
def get_hadoop_downloads():
|
|
cluster_components = []
|
|
hadoop = CdpComponent("hadoop")
|
|
hbase = CdpComponent("hbase", archive_basename_tmpl="hbase-${version}-bin",
|
|
unpack_directory_tmpl="hbase-${version}")
|
|
|
|
use_apache_ozone = os.environ["USE_APACHE_OZONE"] == "true"
|
|
if use_apache_ozone:
|
|
ozone = ApacheComponent("ozone", component_path_tmpl="ozone/${version}")
|
|
else:
|
|
ozone = CdpComponent("ozone")
|
|
|
|
use_apache_hive = os.environ["USE_APACHE_HIVE"] == "true"
|
|
if use_apache_hive:
|
|
hive = ApacheComponent("hive", archive_basename_tmpl="apache-hive-${version}-bin")
|
|
hive_src = ApacheComponent("hive", archive_basename_tmpl="apache-hive-${version}-src")
|
|
else:
|
|
hive = CdpComponent("hive", archive_basename_tmpl="apache-hive-${version}-bin")
|
|
hive_src = CdpComponent("hive-source",
|
|
explicit_version=os.environ.get("IMPALA_HIVE_VERSION"),
|
|
archive_basename_tmpl="hive-${version}-source",
|
|
unpack_directory_tmpl="hive-${version}")
|
|
|
|
tez = CdpComponent("tez", archive_basename_tmpl="tez-${version}-minimal", makedir=True)
|
|
ranger = CdpComponent("ranger", archive_basename_tmpl="ranger-${version}-admin")
|
|
use_override_hive = \
|
|
"HIVE_VERSION_OVERRIDE" in os.environ and os.environ["HIVE_VERSION_OVERRIDE"] != ""
|
|
# If we are using a locally built Hive we do not have a need to pull hive as a
|
|
# dependency
|
|
cluster_components.extend([hadoop, hbase, ozone])
|
|
if not use_override_hive:
|
|
cluster_components.extend([hive, hive_src])
|
|
cluster_components.extend([tez, ranger])
|
|
return cluster_components
|
|
|
|
|
|
def get_kudu_downloads():
|
|
# Toolchain Kudu includes Java artifacts.
|
|
return [ToolchainKudu()]
|
|
|
|
|
|
def main():
|
|
"""
|
|
Validates that bin/impala-config.sh has been sourced by verifying that $IMPALA_HOME
|
|
and $IMPALA_TOOLCHAIN_PACKAGES_HOME are in the environment. We assume that if these
|
|
are set, then IMPALA_<PACKAGE>_VERSION environment variables are also set. This will
|
|
create the directory specified by $IMPALA_TOOLCHAIN_PACKAGES_HOME if it does not
|
|
already exist. Then, it will compute what packages need to be downloaded. Packages are
|
|
only downloaded if they are not already present. There are two main categories of
|
|
packages. Toolchain packages are native packages built using the native toolchain.
|
|
These are always downloaded. Hadoop component packages are the CDP builds of Hadoop
|
|
components such as Hadoop, Hive, HBase, etc. Hadoop component packages are organized as
|
|
a consistent set of compatible version via a build number (i.e. CDP_BUILD_NUMBER).
|
|
Hadoop component packages are only downloaded if $DOWNLOAD_CDH_COMPONENTS is true. The
|
|
versions used for Hadoop components come from the CDP versions based on the
|
|
$CDP_BUILD_NUMBER. CDP Hadoop packages are downloaded into $CDP_COMPONENTS_HOME.
|
|
"""
|
|
logging.basicConfig(level=logging.INFO,
|
|
format='%(asctime)s %(threadName)s %(levelname)s: %(message)s')
|
|
# 'sh' module logs at every execution, which is too noisy
|
|
logging.getLogger("sh").setLevel(logging.WARNING)
|
|
|
|
if not os.environ.get("IMPALA_HOME"):
|
|
logging.error("Impala environment not set up correctly, make sure "
|
|
"impala-config.sh is sourced.")
|
|
sys.exit(1)
|
|
|
|
# Create the toolchain directory if necessary
|
|
create_directory_from_env_var("IMPALA_TOOLCHAIN_PACKAGES_HOME")
|
|
|
|
downloads = []
|
|
if os.getenv("SKIP_TOOLCHAIN_BOOTSTRAP", "false") != "true":
|
|
downloads += get_toolchain_downloads()
|
|
kudu_download = None
|
|
if os.getenv("DOWNLOAD_CDH_COMPONENTS", "false") == "true":
|
|
create_directory_from_env_var("CDP_COMPONENTS_HOME")
|
|
create_directory_from_env_var("APACHE_COMPONENTS_HOME")
|
|
if platform.processor() != "aarch64":
|
|
downloads += get_kudu_downloads()
|
|
downloads += get_hadoop_downloads()
|
|
|
|
components_needing_download = [d for d in downloads if d.needs_download()]
|
|
|
|
def download(component):
|
|
component.download()
|
|
|
|
execute_many(download, components_needing_download)
|
|
|
|
|
|
if __name__ == "__main__": main()
|