mirror of
https://github.com/apache/impala.git
synced 2026-01-15 15:00:36 -05:00
56ee90c598
The locations for native-toolchain packages in IMPALA_TOOLCHAIN currently do not include the compiler version. This means that the toolchain can't distinguish between native-toolchain packages built with gcc 4.9.2 versus gcc 7.5.0. The collisions can cause issues when switching back and forth between branches. This introduces the IMPALA_TOOLCHAIN_PACKAGES_HOME environment variable, which is a location inside IMPALA_TOOLCHAIN that would hold native-toolchain packages. Currently, it is set to the same as IMPALA_TOOLCHAIN, so there is no difference in behavior. This lays the groundwork to add the compiler version to this path when switching to GCC7. Testing: - The only impediment to building with IMPALA_TOOLCHAIN_PACKAGES_HOME=$IMPALA_TOOLCHAIN/test is Impala-lzo. With a custom Impala-lzo, compilation succeeds. Either Impala-lzo will be fixed or it will be removed. - Core tests Change-Id: I1ff641e503b2161baf415355452f86b6c8bfb15b Reviewed-on: http://gerrit.cloudera.org:8080/15991 Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
106 lines
4.0 KiB
Python
Executable File
106 lines
4.0 KiB
Python
Executable File
#!/usr/bin/env impala-python
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Assembles the artifacts required to build docker containers into a single directory.
|
|
# Most artifacts are symlinked so need to be dereferenced (e.g. with tar -h) before
|
|
# being used as a build context.
|
|
|
|
import argparse
|
|
import glob
|
|
import os
|
|
import shutil
|
|
from subprocess import check_call
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--debug-build", help="Setup build context for debug build",
|
|
action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
IMPALA_HOME = os.environ["IMPALA_HOME"]
|
|
if args.debug_build:
|
|
BUILD_TYPE = "debug"
|
|
else:
|
|
BUILD_TYPE = "release"
|
|
OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context", BUILD_TYPE)
|
|
DOCKERFILE = os.path.join(IMPALA_HOME, "docker/impala_base/Dockerfile")
|
|
|
|
IMPALA_TOOLCHAIN_PACKAGES_HOME = os.environ["IMPALA_TOOLCHAIN_PACKAGES_HOME"]
|
|
IMPALA_GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]
|
|
IMPALA_BINUTILS_VERSION = os.environ["IMPALA_BINUTILS_VERSION"]
|
|
GCC_HOME = os.path.join(IMPALA_TOOLCHAIN_PACKAGES_HOME,
|
|
"gcc-{0}".format(IMPALA_GCC_VERSION))
|
|
BINUTILS_HOME = os.path.join(
|
|
IMPALA_TOOLCHAIN_PACKAGES_HOME, "binutils-{0}".format(IMPALA_BINUTILS_VERSION))
|
|
STRIP = os.path.join(BINUTILS_HOME, "bin/strip")
|
|
KUDU_HOME = os.environ["IMPALA_KUDU_HOME"]
|
|
KUDU_LIB_DIR = os.path.join(KUDU_HOME, "release/lib")
|
|
|
|
# Ensure the output directory exists and is empty.
|
|
if os.path.exists(OUTPUT_DIR):
|
|
shutil.rmtree(OUTPUT_DIR)
|
|
os.makedirs(OUTPUT_DIR)
|
|
|
|
os.symlink(os.path.relpath(DOCKERFILE, OUTPUT_DIR),
|
|
os.path.join(OUTPUT_DIR, "Dockerfile"))
|
|
|
|
BIN_DIR = os.path.join(OUTPUT_DIR, "bin")
|
|
LIB_DIR = os.path.join(OUTPUT_DIR, "lib")
|
|
os.mkdir(BIN_DIR)
|
|
os.mkdir(LIB_DIR)
|
|
|
|
|
|
def symlink_file_into_dir(src_file, dst_dir):
|
|
"""Helper to symlink 'src_file' into 'dst_dir'."""
|
|
os.symlink(src_file, os.path.join(dst_dir, os.path.basename(src_file)))
|
|
|
|
|
|
# Impala binaries and native dependencies.
|
|
|
|
# Strip debug symbols from release build to reduce image size. Keep them for
|
|
# debug build.
|
|
IMPALAD_BINARY = os.path.join(IMPALA_HOME, "be/build", BUILD_TYPE, "service/impalad")
|
|
if args.debug_build:
|
|
symlink_file_into_dir(IMPALAD_BINARY, BIN_DIR)
|
|
else:
|
|
check_call([STRIP, "--strip-debug", IMPALAD_BINARY,
|
|
"-o", os.path.join(BIN_DIR, "impalad")])
|
|
for lib in ["libstdc++", "libgcc"]:
|
|
for so in glob.glob(os.path.join(GCC_HOME, "lib64/{0}*.so*".format(lib))):
|
|
symlink_file_into_dir(so, LIB_DIR)
|
|
|
|
for so in glob.glob(os.path.join(KUDU_LIB_DIR, "libkudu_client.so*")):
|
|
symlink_file_into_dir(so, LIB_DIR)
|
|
|
|
# Impala dependencies.
|
|
dep_classpath = file(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()
|
|
for jar in dep_classpath.split(":"):
|
|
assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
|
|
symlink_file_into_dir(jar, LIB_DIR)
|
|
|
|
# Impala jars.
|
|
for jar in glob.glob(
|
|
os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*-SNAPSHOT.jar")):
|
|
symlink_file_into_dir(jar, LIB_DIR)
|
|
|
|
# Templates for debug web pages.
|
|
os.symlink(os.path.join(IMPALA_HOME, "www"), os.path.join(OUTPUT_DIR, "www"))
|
|
# Scripts
|
|
symlink_file_into_dir(os.path.join(IMPALA_HOME, "docker/daemon_entrypoint.sh"), BIN_DIR)
|
|
symlink_file_into_dir(os.path.join(IMPALA_HOME, "bin/graceful_shutdown_backends.sh"),
|
|
BIN_DIR)
|