IMPALA-8622,IMPALA-8696: fix docker dependencies, add image list

Adds a plain-text space-separated image list in
docker/docker-images.txt. This is generated based on the images built by
CMake, so is kept in sync with images added to or removed from the
CMake file.

Duplicated logic per image is removed - instead there is a helper
function that is called for each daemon image to be built.

Rips out the timestamp mechanism that was intended to avoid unnecessary
container rebuilds, but has turned out to be brittle. Instead the
containers are rebuilt each time the rule is invoked.

This moves some subdirectories so that the image tag matches the
subdirectory, to simplify the build scripts.

Change-Id: I4d8e215e9b07c6491faa4751969a30f0ed373fe3
Reviewed-on: http://gerrit.cloudera.org:8080/13899
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Lars Volker <lv@cloudera.com>

.gitignore

@@ -41,8 +41,9 @@ rules.ninja
 # Build timestamp files
 .*timestamp
 
-# Docker build files
+# Docker generated files
 docker/build_context
+docker/docker-images.txt
 
 Testing/
 llvm-ir/

docker/CMakeLists.txt

@@ -18,121 +18,52 @@
 set(IMPALA_BASE_BUILD_CONTEXT_DIR
   ${CMAKE_SOURCE_DIR}/docker/build_context
 )
-set(IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP
-  ${IMPALA_BASE_BUILD_CONTEXT_DIR}/.timestamp
-)
 
-# Build context depends on daemons, frontend jars and test configurations.
+# Build context depends on daemons and frontend jars
 # Sending the whole impala workspace including test binaries, testdata, etc
 # to the docker daemon can be very expensive, so we create a build context
 # with symlinks
-add_custom_command(
-  OUTPUT ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP}
+add_custom_target(impala_base_build_context
   COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
-  COMMAND touch ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP}
   DEPENDS daemons fe ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
   COMMENT "Creating impala base build context."
   VERBATIM
 )
-add_custom_target(impala_base_build_context
-  DEPENDS ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP}
-)
 
-# Command and target for the base Impala image.
-set(IMPALA_BASE_IMAGE_TIMESTAMP
-  ${CMAKE_SOURCE_DIR}/docker/.impala_base_image_timestamp
-)
-add_custom_command(
-  OUTPUT ${IMPALA_BASE_IMAGE_TIMESTAMP}
+# Target for the base Impala image.
+add_custom_target(impala_base_image
   # Use tar with -h flag to assemble a tarball including all the symlinked files and
   # directories in the build context.
   COMMAND cd ${IMPALA_BASE_BUILD_CONTEXT_DIR} && tar cvh . | docker build -t impala_base -
-  COMMAND touch ${IMPALA_BASE_IMAGE_TIMESTAMP}
   DEPENDS impala_base_build_context ${CMAKE_SOURCE_DIR}/docker/impala_base/Dockerfile
   DEPENDS ${CMAKE_SOURCE_DIR}/docker/daemon_entrypoint.sh
   COMMENT "Building Impala base docker image."
   VERBATIM
 )
-add_custom_target(impala_base_image
-  DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP}
-)
+
+# Target to build all docker images. Dependencies are added for each docker image
+# instantiated below.
+add_custom_target(docker_images)
+
+set(daemon_image_names "")
+
+function(add_daemon_docker_image image_name)
+  add_custom_target(${image_name}_image
+    COMMAND cd ${CMAKE_SOURCE_DIR}/docker/${image_name} && docker build -t ${image_name} .
+    DEPENDS impala_base_image ${CMAKE_SOURCE_DIR}/docker/${image_name}/Dockerfile
+    COMMENT "Building ${image_name} docker image."
+    VERBATIM
+  )
+  ADD_DEPENDENCIES(docker_images ${image_name}_image)
+  set(daemon_image_names "${daemon_image_names} ${image_name}" PARENT_SCOPE)
+endfunction()
 
 # Command and target for the various Impalad images.
-set(COORD_EXEC_IMAGE_TIMESTAMP
-  ${CMAKE_SOURCE_DIR}/docker/.coord_exec_image_timestamp
-)
-add_custom_command(
-  OUTPUT ${COORD_EXEC_IMAGE_TIMESTAMP}
-  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/coord_exec && docker build -t impalad_coord_exec .
-  COMMAND touch ${COORD_EXEC_IMAGE_TIMESTAMP}
-  DEPENDS impala_base_image ${CMAKE_SOURCE_DIR}/docker/coord_exec/Dockerfile
-  COMMENT "Building Impalad Coord Exec docker image."
-  VERBATIM
-)
-add_custom_target(coord_exec_image
-  DEPENDS ${COORD_EXEC_IMAGE_TIMESTAMP}
-)
-set(COORDINATOR_IMAGE_TIMESTAMP
-  ${CMAKE_SOURCE_DIR}/docker/.coordinator_image_timestamp
-)
-add_custom_command(
-  OUTPUT ${COORDINATOR_IMAGE_TIMESTAMP}
-  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/coordinator && docker build -t impalad_coordinator .
-  COMMAND touch ${COORDINATOR_IMAGE_TIMESTAMP}
-  DEPENDS impala_base_image ${CMAKE_SOURCE_DIR}/docker/coordinator/Dockerfile
-  COMMENT "Building Impalad Coordinator docker image."
-  VERBATIM
-)
-add_custom_target(coordinator_image
-  DEPENDS ${COORDINATOR_IMAGE_TIMESTAMP}
-)
-set(EXECUTOR_IMAGE_TIMESTAMP
-  ${CMAKE_SOURCE_DIR}/docker/.executor_image_timestamp
-)
-add_custom_command(
-  OUTPUT ${EXECUTOR_IMAGE_TIMESTAMP}
-  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/executor && docker build -t impalad_executor .
-  COMMAND touch ${EXECUTOR_IMAGE_TIMESTAMP}
-  DEPENDS impala_base_image ${CMAKE_SOURCE_DIR}/docker/executor/Dockerfile
-  COMMENT "Building Impalad Executor docker image."
-  VERBATIM
-)
-add_custom_target(executor_image
-  DEPENDS ${EXECUTOR_IMAGE_TIMESTAMP}
-)
+add_daemon_docker_image(impalad_coord_exec)
+add_daemon_docker_image(impalad_coordinator)
+add_daemon_docker_image(impalad_executor)
+add_daemon_docker_image(catalogd)
+add_daemon_docker_image(statestored)
 
-# Command and target for the Statestored image.
-set(STATESTORED_IMAGE_TIMESTAMP
-  ${CMAKE_SOURCE_DIR}/docker/.statestored_image_timestamp
-)
-add_custom_command(
-  OUTPUT ${STATESTORED_IMAGE_TIMESTAMP}
-  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/statestored && docker build -t statestored .
-  COMMAND touch ${STATESTORED_IMAGE_TIMESTAMP}
-  DEPENDS impala_base_image ${CMAKE_SOURCE_DIR}/docker/statestored/Dockerfile
-  COMMENT "Building Statestored docker image."
-  VERBATIM
-)
-add_custom_target(statestored_image
-  DEPENDS ${STATESTORED_IMAGE_TIMESTAMP}
-)
-
-# Command and target for the Catalogd image.
-set(CATALOGD_IMAGE_TIMESTAMP
-  ${CMAKE_SOURCE_DIR}/docker/.catalogd_image_timestamp
-)
-add_custom_command(
-  OUTPUT ${CATALOGD_IMAGE_TIMESTAMP}
-  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/catalogd && docker build -t catalogd .
-  COMMAND touch ${CATALOGD_IMAGE_TIMESTAMP}
-  DEPENDS impala_base_image ${CMAKE_SOURCE_DIR}/docker/catalogd/Dockerfile
-  COMMENT "Building Catalogd docker image."
-  VERBATIM
-)
-add_custom_target(catalogd_image
-  DEPENDS ${CATALOGD_IMAGE_TIMESTAMP}
-)
-add_custom_target(docker_images
-  DEPENDS coord_exec_image coordinator_image executor_image
-  DEPENDS statestored_image catalogd_image
-)
+# Generate a test file with all the daemon images.
+file(WRITE ${CMAKE_SOURCE_DIR}/docker/docker-images.txt "${daemon_image_names}")

docker/push-images.sh

@@ -19,6 +19,8 @@
 
 set -euo pipefail
 
+DIR=$(dirname "$0")
+
 usage() {
   echo "push_to_registry.sh [options]"
   echo
@@ -64,10 +66,10 @@ if [[ -z "$PREFIX" ]]; then
 fi
 
 # The image tags that are updated by the impala build process.
-# TODO(IMPALA-8622): get this list from a generated file.
-IMAGES=(statestored catalogd impalad_coordinator impalad_executor impalad_coord_exec)
+IMAGES=$(cat ${DIR}/docker-images.txt)
+echo "Images to push: ${IMAGES}"
 
-for IMAGE in "${IMAGES[@]}"; do
+for IMAGE in ${IMAGES}; do
   if [[ -z "$REGISTRY" ]]; then
     # Docker Hub does not require a prefix.
     DEST=