jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-25 02:03:09 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-12081: Produce multiple Java docker images

Browse Source 
This changes the docker image build code so that both Java 8 and Java 11
images can be built in the same build. Specifically, it introduces new
Make targets for Java 11 docker images in addition to the regular Java 8
targets. The "docker_images" and "docker_debug_images" targets continue
to behave the same way and produce Java 8 images of the same name. The
"docker_java11_images" and "docker_debug_java11_images" produce the
daemon docker images for Java 11.

Preserves IMPALA_DOCKER_USE_JAVA11 for selecting Java 11 images when
starting a cluster with container images.

Change-Id: Ic2b124267c607242bc2fd6c8cd6486293a938f50
Reviewed-on: http://gerrit.cloudera.org:8080/19722
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
Michael Smith
2023-04-18 16:53:29 -07:00
committed by Impala Public Jenkins
parent 8785270451
commit c8a21c51ef
6 changed files with 44 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
docker/CMakeLists.txt
View File

@@ -70,27 +70,18 @@ MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}"
if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
# Add a target to build a base docker image for 'build_type'. 'build_context_args' are
# passed to the setup_build_context.py script.
function(add_base_image build_type build_context_args)
function(add_base_image build_type build_context_args install_os_packages_args)
# Build context depends on daemons and frontend jars.
# Sending the whole impala workspace including test binaries, testdata, etc
# to the docker daemon can be very expensive, so we create a build context
# with symlinks
add_custom_target(impala_base_build_context_${build_type}
COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args}
--output-dir ${IMPALA_BASE_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS daemons java ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
COMMENT "Creating impala base build context build_type=${build_type}."
VERBATIM
)
# The base image Dockerfile takes an argument INSTALL_OS_PACKAGES_ARGS to specify
# arguments for install_os_packages.sh. This can can be set to "--install-debug-tools"
# to add extra utilities. Enable this for debug builds.
set(INSTALL_OS_PACKAGES_ARGS "")
if (${build_type} STREQUAL "debug")
set(INSTALL_OS_PACKAGES_ARGS "--install-debug-tools")
endif()
if ($ENV{IMPALA_DOCKER_USE_JAVA11} STREQUAL "true")
set(INSTALL_OS_PACKAGES_ARGS "${INSTALL_OS_PACKAGES_ARGS} --use-java11")
endif()
# Target for the base Impala image.
add_custom_target(impala_base_image_${build_type}
# Run docker build inside the build context directory so that all dependencies are
@@ -99,7 +90,7 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
COMMAND tar cvh . -C ${CMAKE_SOURCE_DIR}/docker/impala_base/ . |
${DOCKER_BUILD} -t impala_base_${build_type}
--build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE}
--build-arg INSTALL_OS_PACKAGES_ARGS=${INSTALL_OS_PACKAGES_ARGS} -
--build-arg INSTALL_OS_PACKAGES_ARGS=${install_os_packages_args} -
WORKING_DIRECTORY ${IMPALA_BASE_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS impala_base_build_context_${build_type} ${CMAKE_SOURCE_DIR}/docker/impala_base/Dockerfile
DEPENDS ${CMAKE_SOURCE_DIR}/docker/daemon_entrypoint.sh
@@ -109,13 +100,18 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
VERBATIM
)
endfunction()
add_base_image(release "")
add_base_image(debug "--debug-build")
add_base_image(release "" "")
add_base_image(release_java11 "" "--java 11")
# Debug images include debug tools
add_base_image(debug "--debug-build" "--install-debug-tools")
add_base_image(debug_java11 "--debug-build" "--install-debug-tools --java 11")
# Target to build all docker images. Dependencies are added for each docker image
# instantiated below.
add_custom_target(docker_images)
add_custom_target(docker_java11_images)
add_custom_target(docker_debug_images)
add_custom_target(docker_debug_java11_images)
add_custom_target(quickstart_docker_images)
set(exported_image_names "")
@@ -146,13 +142,20 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
function(add_daemon_docker_images daemon_name)
set(release_image ${daemon_name})
set(release_target ${daemon_name}_image)
set(release_java11_image ${daemon_name}_java11)
set(release_java11_target ${daemon_name}_java11_image)
set(debug_image ${daemon_name}_debug)
set(debug_target ${daemon_name}_debug_image)
set(debug_java11_image ${daemon_name}_debug_java11)
set(debug_java11_target ${daemon_name}_debug_java11_image)
add_daemon_docker_image(${release_target} ${daemon_name} ${release_image} release)
add_daemon_docker_image(${release_java11_target} ${daemon_name} ${release_java11_image} release_java11)
add_daemon_docker_image(${debug_target} ${daemon_name} ${debug_image} debug)
add_daemon_docker_image(${debug_java11_target} ${daemon_name} ${debug_java11_image} debug_java11)
ADD_DEPENDENCIES(docker_images ${release_target})
ADD_DEPENDENCIES(docker_java11_images ${release_java11_target})
ADD_DEPENDENCIES(docker_debug_images ${debug_target})
set(exported_image_names "${exported_image_names} ${release_image}" PARENT_SCOPE)
ADD_DEPENDENCIES(docker_debug_java11_images ${debug_java11_target})
endfunction()
# Stamp out image targets for all of the Impala daemons.
@@ -211,7 +214,8 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
# to the docker daemon can be very expensive, so we create a build context
# with symlinks
add_custom_target(impala_utility_build_context_${build_type}
COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args} --utility-context
COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args}
--utility-context --output-dir ${IMPALA_UTILITY_BUILD_CONTEXT_DIR}/${build_type}
DEPENDS impala-profile-tool ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
COMMENT "Creating impala utility build context build_type=${build_type}."
VERBATIM

28
docker/install_os_packages.sh
View File

@@ -24,12 +24,12 @@
set -euo pipefail
INSTALL_DEBUG_TOOLS=false
USE_JAVA11=false
JAVA_VERSION=8
function print_usage {
echo "install_os_packages.sh - Helper script to install OS dependencies"
echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc"
echo "[--use-java11] : Use Java 11 rather than the default Java 8."
echo "[--java <version>] : Use specified Java version rather than the default Java 8."
}
while [ -n "$*" ]
@@ -38,8 +38,9 @@ do
--install-debug-tools)
INSTALL_DEBUG_TOOLS=true
;;
--use-java11)
USE_JAVA11=true
--java)
JAVA_VERSION="${2-}"
shift;
;;
--help|*)
print_usage
@@ -50,7 +51,7 @@ do
done
echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}"
echo "USE_JAVA11=${USE_JAVA11}"
echo "JAVA_VERSION=${JAVA_VERSION}"
# This can get more detailed if there are specific steps
# for specific versions, but at the moment the distribution
@@ -67,7 +68,7 @@ else
# Ubuntu 16.04 does not have Java 11 in its package repository,
# so exit with an error.
if [[ $DISTRIB_RELEASE == 16.04 ]] && $USE_JAVA11 ; then
if [[ $DISTRIB_RELEASE == 16.04 ]] && [[ $JAVA_VERSION == 11 ]] ; then
echo "ERROR: Java 11 is not supported on Ubuntu 16.04"
exit 1
fi
@@ -85,11 +86,6 @@ fi
if [[ $DISTRIBUTION == Ubuntu ]]; then
export DEBIAN_FRONTEND=noninteractive
apt-get update
if $USE_JAVA11 ; then
apt-get install -y openjdk-11-jre-headless
else
apt-get install -y openjdk-8-jre-headless
fi
apt-get install -y \
hostname \
krb5-user \
@@ -97,6 +93,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
libsasl2-2 \
libsasl2-modules \
libsasl2-modules-gssapi-mit \
openjdk-${JAVA_VERSION}-jre-headless \
tzdata
if $INSTALL_DEBUG_TOOLS ; then
echo "Installing extra debug tools"
@@ -111,17 +108,14 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
vim
fi
elif [[ $DISTRIBUTION == Redhat ]]; then
if $USE_JAVA11 ; then
yum install -y --disableplugin=subscription-manager \
java-11-openjdk-headless
else
yum install -y --disableplugin=subscription-manager \
java-1.8.0-openjdk-headless
if [[ $JAVA_VERSION == 8 ]]; then
JAVA_VERSION=1.8.0
fi
yum install -y --disableplugin=subscription-manager \
cyrus-sasl-gssapi \
cyrus-sasl-plain \
hostname \
java-${JAVA_VERSION}-openjdk-headless \
krb5-workstation \
openldap-devel \
procps-ng \

11
docker/setup_build_context.py
View File

@@ -33,17 +33,12 @@ parser.add_argument("--debug-build", help="Setup build context for debug build",
parser.add_argument("--utility-context",
help="Setup utility build context instead of daemon",
action="store_true")
parser.add_argument("--output-dir", help="Directory to use for output")
args = parser.parse_args()
IMPALA_HOME = os.environ["IMPALA_HOME"]
if args.debug_build:
BUILD_TYPE = "debug"
else:
BUILD_TYPE = "release"
if args.utility_context:
OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context_utility", BUILD_TYPE)
else:
OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context", BUILD_TYPE)
BUILD_TYPE = "debug" if args.debug_build else "release"
OUTPUT_DIR = args.output_dir
IMPALA_TOOLCHAIN_PACKAGES_HOME = os.environ["IMPALA_TOOLCHAIN_PACKAGES_HOME"]
IMPALA_GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]