IMPALA-7948: part 1: initial docker container build

This builds an impala_base container that has all of the build artifacts
required to run the impala processes, then builds impalad, catalogd and
statestore containers based on that with the right ports exposed.
The images are based on the Ubuntu 16.04 image to align with the
most common development environment.

The container build process is integrated with CMake and is designed
to integrate with the rest of the build so that the container build
depends on the artifacts that will go into the container. You can
build the images with the following command, which will create
images called "impala_base", "impalad", "catalogd" and
"statestored":

  ninja -j $IMPALA_BUILD_THREADS docker_images

The images need some refinement to be truly useful.  The following
will be done in future patches:
* IMPALA-7947 - integrate with start-impala-cluster.py to
  automatically create docker network with containers running on it
* Mechanism to pass in command-line flags
* Mechanisms to update the various config files to point to the
  docker host rather than "localhost", which doesn't point to
  the right thing inside the container.
* Mechanisms to set mem_limit, JVM heap sizes, etc, automatically.

Testing:
Manually started up the containers connected to a user-defined bridge
network, tweaked the configurations to point to the HMS/HDFS/etc
running on my host. I then used "docker ps" to figure out the
port mappings for beeswax and debug webserver.

Confirmed that I could run a query and access debug pages:

  $ impala-shell.sh -i localhost:32860 -q "select coordinator()"
  Starting Impala Shell without Kerberos authentication
  Opened TCP connection to localhost:32860
  Connected to localhost:32860
  Server version: impalad version 3.1.0-SNAPSHOT DEBUG (build
  d7870fe03645490f95bd5ffd4a2177f90eb2f3c0)
  Query: select coordinator()
  Query submitted at: 2018-12-11 15:51:04 (Coordinator:
  http://8063e77ce999:25000)
  Query progress can be monitored at:
  http://8063e77ce999:25000/query_plan?query_id=1b4d03f0f0f1fcfb:b0b37e5000000000
  +---------------+
  | coordinator() |
  +---------------+
  | 8063e77ce999  |
  +---------------+
  Fetched 1 row(s) in 0.11s

Change-Id: Ifea707aa3cc23e4facda8ac374160c6de23ffc4e
Reviewed-on: http://gerrit.cloudera.org:8080/12074
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>

.gitignore

@@ -37,6 +37,12 @@ CTestTestfile.cmake
 build.ninja
 rules.ninja
 
+# Build timestamp files
+.*timestamp
+
+# Docker build files
+docker/build_context
+
 Testing/
 llvm-ir/
 shell/gen-py/

CMakeLists.txt

@@ -393,6 +393,7 @@ add_subdirectory(common/fbs)
 add_subdirectory(common/yarn-extras)
 add_subdirectory(common/protobuf)
 add_subdirectory(be)
+add_subdirectory(docker)
 add_subdirectory(fe)
 add_subdirectory(impala-parent)
 add_subdirectory(ext-data-source)

be/src/service/CMakeLists.txt

@@ -60,14 +60,23 @@ add_executable(impalad
 
 # All Impala daemons run from the same binary. The code that is run is determined by the
 # name (i.e. argv[0]) of the command that executes the binary, so we create symlinks for
-# statestored and catalogd.
-ADD_CUSTOM_TARGET(statestored ALL
-  ${CMAKE_COMMAND} -E create_symlink ${EXECUTABLE_OUTPUT_PATH}/impalad
-  ${BUILD_OUTPUT_ROOT_DIRECTORY}/statestore/statestored DEPENDS impalad)
+# statestored and catalogd. The symlinks are relative so they can be installed along with
+# the impalad binary.
+set (STATESTORED_SYMLINK ${EXECUTABLE_OUTPUT_PATH}/statestored)
+set (CATALOGD_SYMLINK ${EXECUTABLE_OUTPUT_PATH}/catalogd)
 
-ADD_CUSTOM_TARGET(catalogd ALL
-  ${CMAKE_COMMAND} -E create_symlink ${EXECUTABLE_OUTPUT_PATH}/impalad
-  ${BUILD_OUTPUT_ROOT_DIRECTORY}/catalog/catalogd DEPENDS impalad)
+ADD_CUSTOM_COMMAND(
+  OUTPUT ${STATESTORED_SYMLINK}
+  COMMAND ${CMAKE_COMMAND} -E create_symlink impalad ${STATESTORED_SYMLINK}
+  DEPENDS impalad)
+ADD_CUSTOM_COMMAND(
+  OUTPUT ${CATALOGD_SYMLINK}
+  COMMAND ${CMAKE_COMMAND} -E create_symlink impalad ${CATALOGD_SYMLINK}
+  DEPENDS impalad)
+ADD_CUSTOM_TARGET(statestored ALL DEPENDS ${STATESTORED_SYMLINK})
+ADD_CUSTOM_TARGET(catalogd ALL DEPENDS ${CATALOGD_SYMLINK})
+
+ADD_CUSTOM_TARGET(daemons DEPENDS impalad statestored catalogd fesupport)
 
 # This permits libraries loaded by dlopen to link to the symbols in the program.
 set_target_properties(impalad PROPERTIES LINK_FLAGS -rdynamic)

bin/start-catalogd.sh

@@ -72,4 +72,4 @@ fi
 
 . ${IMPALA_HOME}/bin/set-classpath.sh
 export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
-exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/catalog/catalogd ${CATALOGD_ARGS}
+exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/service/catalogd ${CATALOGD_ARGS}

bin/start-statestored.sh

@@ -61,4 +61,4 @@ if ${CLUSTER_DIR}/admin is_kerberized; then
 fi
 
 export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
-exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/statestore/statestored ${STATESTORED_ARGS}
+exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/service/statestored ${STATESTORED_ARGS}

docker/CMakeLists.txt

@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(IMPALA_BASE_BUILD_CONTEXT_DIR
+  ${CMAKE_SOURCE_DIR}/docker/build_context
+)
+set(IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP
+  ${IMPALA_BASE_BUILD_CONTEXT_DIR}/.timestamp
+)
+
+# Build context depends on daemons, frontend jars and test configurations.
+# Sending the whole impala workspace including test binaries, testdata, etc
+# to the docker daemon can be very expensive, so we create a build context
+# with symlinks
+add_custom_command(
+  OUTPUT ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP}
+  COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
+  COMMAND touch ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP}
+  DEPENDS daemons fe test-configuration ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
+  COMMENT "Creating impala base build context."
+  VERBATIM
+)
+
+# Command and target for the base Impala image.
+set(IMPALA_BASE_IMAGE_TIMESTAMP
+  ${CMAKE_SOURCE_DIR}/docker/.impala_base_image_timestamp
+)
+add_custom_command(
+  OUTPUT ${IMPALA_BASE_IMAGE_TIMESTAMP}
+  # Use tar with -h flag to assemble a tarball including all the symlinked files and
+  # directories in the build context.
+  COMMAND cd ${IMPALA_BASE_BUILD_CONTEXT_DIR} && tar cvh . | docker build -t impala_base -
+  COMMAND touch ${IMPALA_BASE_IMAGE_TIMESTAMP}
+  DEPENDS ${IMPALA_BASE_BUILD_CONTEXT_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/impala_base/Dockerfile
+  DEPENDS ${CMAKE_SOURCE_DIR}/docker/daemon_entrypoint.sh
+  COMMENT "Building Impala base docker image."
+  VERBATIM
+)
+add_custom_target(impala_base_image
+  DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP}
+)
+
+# Command and target for the Impalad image.
+set(IMPALAD_IMAGE_TIMESTAMP
+  ${CMAKE_SOURCE_DIR}/docker/.impalad_image_timestamp
+)
+add_custom_command(
+  OUTPUT ${IMPALAD_IMAGE_TIMESTAMP}
+  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/impalad && docker build -t impalad .
+  COMMAND touch ${IMPALAD_IMAGE_TIMESTAMP}
+  DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/impalad/Dockerfile
+  COMMENT "Building Impalad docker image."
+  VERBATIM
+)
+add_custom_target(impalad_image
+  DEPENDS ${IMPALAD_IMAGE_TIMESTAMP}
+)
+
+# Command and target for the Statestored image.
+set(STATESTORED_IMAGE_TIMESTAMP
+  ${CMAKE_SOURCE_DIR}/docker/.statestored_image_timestamp
+)
+add_custom_command(
+  OUTPUT ${STATESTORED_IMAGE_TIMESTAMP}
+  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/statestored && docker build -t statestored .
+  COMMAND touch ${STATESTORED_IMAGE_TIMESTAMP}
+  DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/statestored/Dockerfile
+  COMMENT "Building Statestored docker image."
+  VERBATIM
+)
+add_custom_target(statestored_image
+  DEPENDS ${STATESTORED_IMAGE_TIMESTAMP}
+)
+
+# Command and target for the Catalogd image.
+set(CATALOGD_IMAGE_TIMESTAMP
+  ${CMAKE_SOURCE_DIR}/docker/.catalogd_image_timestamp
+)
+add_custom_command(
+  OUTPUT ${CATALOGD_IMAGE_TIMESTAMP}
+  COMMAND cd ${CMAKE_SOURCE_DIR}/docker/catalogd && docker build -t catalogd .
+  COMMAND touch ${CATALOGD_IMAGE_TIMESTAMP}
+  DEPENDS ${IMPALA_BASE_IMAGE_TIMESTAMP} ${CMAKE_SOURCE_DIR}/docker/catalogd/Dockerfile
+  COMMENT "Building Catalogd docker image."
+  VERBATIM
+)
+add_custom_target(catalogd_image
+  DEPENDS ${CATALOGD_IMAGE_TIMESTAMP}
+)
+add_custom_target(docker_images
+  DEPENDS impalad_image statestored_image catalogd_image
+)

docker/README.md

@@ -3,3 +3,8 @@
 `test-with-docker.py` runs the Impala build and tests inside of Docker
 containers, parallelizing the test execution across test suites. See that file
 for more details.
+
+This also contains **experimental** infrastructure to build `impala_base`,
+`catalogd`, `impalad` and `statestored` container images from the output of
+an Impala build. The containers can be built via the CMake target docker_images.
+See CMakeLists.txt for the build targets.

docker/catalogd/Dockerfile

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM impala_base
+
+# Externally-facing ports
+# Debug webserver
+EXPOSE 25020
+
+CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/catalogd",\
+     "-abort_on_config_error=false", "-state_store_host=statestored"]

docker/daemon_entrypoint.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Wrapper script that runs the command-line provided as its arguments after
+# setting up the environment required for the daemon processes to run.
+
+# Set -x for easy of debugging.
+# TODO: remove to reduce verbosity once things are more stable.
+set -x
+
+export IMPALA_HOME=/opt/impala
+
+# Add directories containing dynamic libraries required by the daemons that
+# are not on the system library paths.
+export LD_LIBRARY_PATH=/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/
+LD_LIBRARY_PATH+=:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/
+LD_LIBRARY_PATH+=:/opt/kudu/release/lib
+
+# Configs should be first on classpath
+export CLASSPATH=/opt/impala/conf
+# Append all of the jars in /opt/impala/lib to the classpath.
+for jar in /opt/impala/lib/*.jar
+do
+  CLASSPATH+=:$jar
+done
+echo "CLASSPATH: $CLASSPATH"
+echo "LD_LIBRARY_PATH: $LD_LIBRARY_PAT"
+
+"$@"
+EXIT_CODE=$?
+
+# Print out any INFO logs to help with debugging container startup failures.
+# TODO: remove once we have proper logging
+cat /tmp/*.INFO
+exit $EXIT_CODE

docker/impala_base/Dockerfile

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:16.04
+
+RUN apt-get update && \
+  apt-get install -y openjdk-8-jre-headless \
+  libsasl2-2 libsasl2-modules libsasl2-modules-gssapi-mit \
+  tzdata
+
+# Copy build artifacts required for the daemon processes.
+# Need to have multiple copy commands to preserve directory structure.
+COPY bin /opt/impala/bin
+COPY conf /opt/impala/conf
+COPY lib /opt/impala/lib
+COPY www /opt/impala/www
+copy kudu /opt/kudu
+
+WORKDIR /opt/impala/

docker/impalad/Dockerfile

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM impala_base
+
+# Externally-facing ports
+# Beeswax
+EXPOSE 21000
+# HS2
+EXPOSE 21050
+# Debug webserver
+EXPOSE 25000
+
+CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
+     "-abort_on_config_error=false", "-state_store_host=statestored",\
+     "-catalog_service_host=catalogd"]

docker/setup_build_context.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env impala-python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Assembles the artifacts required to build docker containers into a single directory.
+# Most artifacts are symlinked so need to be dereferenced (e.g. with tar -h) before
+# being used as a build context.
+
+import glob
+import os
+import shutil
+
+IMPALA_HOME = os.environ["IMPALA_HOME"]
+OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context")
+DOCKERFILE = os.path.join(IMPALA_HOME, "docker/impala_base/Dockerfile")
+
+IMPALA_TOOLCHAIN = os.environ["IMPALA_TOOLCHAIN"]
+IMPALA_GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]
+GCC_HOME = os.path.join(IMPALA_TOOLCHAIN, "gcc-{0}".format(IMPALA_GCC_VERSION))
+
+# Ensure the output directory exists and is empty.
+if os.path.exists(OUTPUT_DIR):
+    shutil.rmtree(OUTPUT_DIR)
+os.makedirs(OUTPUT_DIR)
+
+os.symlink(os.path.relpath(DOCKERFILE, OUTPUT_DIR),
+    os.path.join(OUTPUT_DIR, "Dockerfile"))
+
+BIN_DIR = os.path.join(OUTPUT_DIR, "bin")
+LIB_DIR = os.path.join(OUTPUT_DIR, "lib")
+os.mkdir(BIN_DIR)
+os.mkdir(LIB_DIR)
+
+
+def symlink_file_into_dir(src_file, dst_dir):
+    """Helper to symlink 'src_file' into 'dst_dir'."""
+    os.symlink(src_file, os.path.join(dst_dir, os.path.basename(src_file)))
+
+
+# Impala binaries and native dependencies.
+for bin in ["impalad", "statestored", "catalogd", "libfesupport.so"]:
+    symlink_file_into_dir(os.path.join(IMPALA_HOME, "be/build/latest/service", bin),
+         BIN_DIR)
+for lib in ["libstdc++", "libgcc"]:
+    for so in glob.glob(os.path.join(GCC_HOME, "lib64/{0}*.so*".format(lib))):
+        symlink_file_into_dir(so, LIB_DIR)
+os.symlink(os.environ["IMPALA_KUDU_HOME"], os.path.join(OUTPUT_DIR, "kudu"))
+
+# Impala jars and dependencies.
+for glob_pattern in [os.path.join(IMPALA_HOME, "fe/target/dependency/*.jar"),
+    os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")]:
+  for jar in glob.glob(glob_pattern):
+      symlink_file_into_dir(jar, LIB_DIR)
+
+# Templates for debug web pages.
+os.symlink(os.path.join(IMPALA_HOME, "www"), os.path.join(OUTPUT_DIR, "www"))
+# Scripts
+symlink_file_into_dir(os.path.join(IMPALA_HOME, "docker/daemon_entrypoint.sh"), BIN_DIR)
+# Minicluster configs
+os.symlink(os.path.join(IMPALA_HOME, "fe/src/test/resources"),
+    os.path.join(OUTPUT_DIR, "conf"))

docker/statestored/Dockerfile

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM impala_base
+
+# Externally-facing ports
+# Debug webserver
+EXPOSE 25010
+
+CMD ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/statestored"]

fe/CMakeLists.txt

@@ -17,5 +17,9 @@
 
 add_custom_target(fe ALL DEPENDS
   thrift-deps fb-deps yarn-extras function-registry ext-data-source impala-parent
-  COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
+  COMMAND ${CMAKE_SOURCE_DIR}/bin/mvn-quiet.sh -B install -DskipTests
+)
+
+add_custom_target(test-configuration
+  COMMAND ${CMAKE_SOURCE_DIR}/bin/create-test-configuration.sh
 )