IMPALA-10262: RPM/DEB Packaging Support

This patch bases on a previous patch contributed by Shant Hovsepian: https://gerrit.cloudera.org/c/16612/ It adds a new option, -package, to buildall.sh for building a package for the current OS type (e.g. CentOS/Ubuntu). You can also use "make/ninja package" to build the package. Scripts for launching the services and the required configuration files are also added. Tests: - Built on Ubuntu 18.04/20.04 and CentOS 7 using ./buildall.sh -noclean -skiptests -release -package - Deployed the RPM package on a CDP cluster. Verifed the scripts. - Deployed the DEB package on a docker container. Verified the scripts. Change-Id: I64419fd400fe8d233dac016b6306157fe9461d82 Reviewed-on: http://gerrit.cloudera.org:8080/18939 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
2025-12-19 18:12:08 -05:00 · 2022-09-05 13:13:11 +08:00
parent a281d8eb8e
commit 8d0ab2b684
23 changed files with 579 additions and 7 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,6 +24,8 @@ set(NO_TESTS 1)
 # initialized.
 project(Impala)

+option(BUILD_PACKAGES "Build deployment packages")
+
 include(cmake_modules/kudu_cmake_fns.txt)

 if (NOT DEFINED BUILD_SHARED_LIBS)
@@ -485,6 +487,15 @@ else()
  set_property(GLOBAL PROPERTY RULE_LAUNCH_CUSTOM ${JUNITXML_WRAPPER})
 endif()

+cmake_host_system_information(RESULT OS_DISTRIB_ID QUERY DISTRIB_ID)
+cmake_host_system_information(RESULT OS_DISTRIB_VERSION_ID QUERY DISTRIB_VERSION_ID)
+message(STATUS "DISTRIB_ID: ${OS_DISTRIB_ID}")
+message(STATUS "DISTRIB_VERSION_ID: ${OS_DISTRIB_VERSION_ID}")
+
+## installation path
+set(CMAKE_INSTALL_PREFIX "/opt")
+set(IMPALA_INSTALLDIR "impala" CACHE INTERNAL "")
+
 # compile these subdirs using their own CMakeLists.txt
 add_subdirectory(common/function-registry)
 add_subdirectory(common/thrift)
@@ -495,6 +506,42 @@ add_subdirectory(docker)
 add_subdirectory(java)
 add_subdirectory(shell)

+install(DIRECTORY "www/" DESTINATION ${IMPALA_INSTALLDIR}/www)
+install(FILES fe/target/impala-frontend-$ENV{IMPALA_VERSION}.jar
+  DESTINATION ${IMPALA_INSTALLDIR}/jar)
+
+set(IMPALA_GCC_HOME $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/gcc-$ENV{IMPALA_GCC_VERSION})
+FILE(GLOB gcc_lib ${IMPALA_GCC_HOME}/lib64/libgcc_s.so.1*)
+install(FILES ${gcc_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+FILE(GLOB cpp_lib ${IMPALA_GCC_HOME}/lib64/libstdc++.so.6*)
+install(FILES ${cpp_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+set(KUDU_HOME $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$ENV{IMPALA_KUDU_VERSION}/release)
+# The parent folder is lib64 on centos/redhat, while on ubuntu it's lib.
+FILE(GLOB kudu_lib ${KUDU_HOME}/lib*/libkudu_client.so*)
+install(FILES ${kudu_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+FILE(GLOB hadoop_lib $ENV{HADOOP_LIB_DIR}/native/libhadoop.so*)
+install(FILES ${hadoop_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib)
+
+install(DIRECTORY fe/target/dependency/ DESTINATION ${IMPALA_INSTALLDIR}/jar
+  FILES_MATCHING PATTERN "*.jar")
+
+string(TOLOWER ${OS_DISTRIB_ID} OS_DISTRIB_ID)
+if (${OS_DISTRIB_ID} STREQUAL "rhel" AND ${OS_DISTRIB_VERSION_ID} VERSION_GREATER_EQUAL 8)
+  # TODO: remove this after resolving IMPALA-12255
+  message(STATUS "Not packaging impala-shell on redhat8+ due to ambiguous python shebang")
+else()
+  install(DIRECTORY shell/build/impala-shell-$ENV{IMPALA_VERSION}/
+    DESTINATION ${IMPALA_INSTALLDIR}/shell
+    USE_SOURCE_PERMISSIONS PATTERN "*.pyc" EXCLUDE)
+endif()
+
+install(DIRECTORY package/bin/ DESTINATION ${IMPALA_INSTALLDIR}/bin
+  USE_SOURCE_PERMISSIONS FILES_MATCHING PATTERN "*.sh")
+install(DIRECTORY package/conf/ DESTINATION ${IMPALA_INSTALLDIR}/conf)
+
 # Build target for all generated files which most backend code depends on
 add_custom_target(gen-deps ALL DEPENDS thrift-deps proto-deps fb-deps
  kudu-util-proto-deps kudu-rpc-proto-deps kudu-security-proto-deps gen_ir_descriptions)
@@ -543,3 +590,66 @@ if (DUMP_INCLUDE_PATHS)
 endif(DUMP_INCLUDE_PATHS)

 SET(CMAKE_EXE_LINKER_FLAGS  "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libstdc++")
+
+# packaging
+if (BUILD_PACKAGES)
+
+  if (${OS_DISTRIB_ID} MATCHES "(rhel|centos)")
+    set(CPACK_GENERATOR "RPM")
+  elseif (${OS_DISTRIB_ID} MATCHES "ubuntu")
+    set(CPACK_GENERATOR "DEB")
+  else()
+    message(WARNING "OS is ${OS_DISTRIB_ID}. No packages will be generated.")
+  endif()
+
+  if (CPACK_GENERATOR)
+    message(STATUS "Packaging enabled: ${CPACK_GENERATOR}")
+    set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME})
+    set(CPACK_PACKAGE_VENDOR "Apache")
+    set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "An open source massively parallel processing SQL query engine.")
+    set(CPACK_PACKAGE_CONTACT "dev@impala.apache.org")
+    set(CPACK_PACKAGE_HOMEPAGE_URL "https://impala.apache.org")
+    set(CPACK_PACKAGE_VERSION "$ENV{IMPALA_VERSION}")
+    set(CPACK_RPM_PACKAGE_LICENSE "ASL-2.0")
+    # Set a meaningful package name, e.g. Impala-4.3.0-SNAPSHOT_hive-3.1.3000.7.2.18.0-41-x86_64.el7
+    set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}")
+    set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}_hive-$ENV{IMPALA_HIVE_VERSION}")
+    set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}-${CMAKE_SYSTEM_PROCESSOR}")
+    if (${CPACK_GENERATOR} MATCHES "RPM")
+      set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.el${OS_DISTRIB_VERSION_ID}")
+    elseif (${CPACK_GENERATOR} MATCHES "DEB")
+      set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.ubuntu-${OS_DISTRIB_VERSION_ID}")
+    endif()
+    message(STATUS "Package name: ${CPACK_PACKAGE_FILE_NAME}")
+
+    if ($ENV{STRIP_DEPLOYMENT_IMPALAD})
+      set(CPACK_STRIP_FILES ${IMPALA_INSTALLDIR}/bin/impalad)
+      message("Binaries in the package will be stripped")
+    endif()
+
+    set(CPACK_PACKAGING_INSTALL_PREFIX "/opt")
+    set(CPACK_PACKAGE_INSTALL_DIRECTORY "impala")
+
+    execute_process(
+      COMMAND bash -c "${CMAKE_SOURCE_DIR}/docker/install_os_packages.sh --dry-run | tail -n1"
+      OUTPUT_VARIABLE PKG_LIST
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    if ("${PKG_LIST}" STREQUAL "")
+      message(FATAL_ERROR "Package list is empty: '${PKG_LIST}'")
+    else()
+      message(STATUS "Get required package list: '${PKG_LIST}'")
+    endif()
+
+    if (${CPACK_GENERATOR} MATCHES "RPM")
+      set(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
+      set(CPACK_RPM_PACKAGE_RELOCATABLE TRUE)
+      set(CPACK_RPM_PACKAGE_REQUIRES ${PKG_LIST})
+    else()
+      set(CPACK_DEBIAN_PACKAGE_DEPENDS ${PKG_LIST})
+    endif()
+  endif()
+
+include(CPack)
+
+endif()
--- a/be/src/service/CMakeLists.txt
+++ b/be/src/service/CMakeLists.txt
@@ -126,6 +126,10 @@ target_link_libraries(unifiedbetests
  ${JAVA_JSIG_LIBRARY} ${UNIFIED_TEST_LINK_LIBS})
 ADD_DEPENDENCIES(unified-be-test-executable unifiedbetests)

+install(FILES ${STATESTORED_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
+install(FILES ${CATALOGD_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
+install(TARGETS impalad DESTINATION ${IMPALA_INSTALLDIR}/bin)
+
 # Exception to unified be tests: Custom main() due to leak
 ADD_BE_TEST(session-expiry-test session-expiry-test.cc) # TODO: this leaks thrift server
 ADD_UNIFIED_BE_LSAN_TEST(hs2-util-test "StitchNullsTest.*:PrintTColumnValueTest.*")
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -260,7 +260,7 @@ ubuntu sudo update-java-alternatives -s \
 redhat sudo yum install -y file gawk gcc gcc-c++ git krb5-devel krb5-server \
        krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \
        cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \
-        postgresql postgresql-server \
+        postgresql postgresql-server rpm-build \
        wget vim-common nscd cmake zlib-devel \
        procps psmisc lsof openssh-server python3-devel python3-setuptools \
        net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -1008,6 +1008,9 @@ CLASSPATH="$IMPALA_FE_DIR/src/test/resources:$CLASSPATH"
 # A marker in the environment to prove that we really did source this file
 export IMPALA_CONFIG_SOURCED=1

+# Whether to strip the impalad binary when generating deployment package
+export STRIP_DEPLOYMENT_IMPALAD=true
+
 echo "IMPALA_VERSION          = $IMPALA_VERSION"
 echo "IMPALA_HOME             = $IMPALA_HOME"
 echo "HADOOP_HOME             = $HADOOP_HOME"
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -39,6 +39,7 @@ CONFIGS=(
  # Test gcc builds with and without -so:
  "-skiptests -noclean"
  "-skiptests -noclean -release"
+  "-skiptests -noclean -release -package"
  "-skiptests -noclean -release -so -ninja"
  # clang sanitizer builds:
  "-skiptests -noclean -asan"
@@ -46,6 +47,7 @@ CONFIGS=(
  "-skiptests -noclean -ubsan -so -ninja"
  # USE_APACHE_HIVE=true build:
  "-skiptests -noclean -use_apache_hive"
+  "-skiptests -noclean -use_apache_hive -package"
 )

 FAILED=""
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -254,3 +254,6 @@ docs/images/howto_show_histogram.png
 docs/images/howto_static_server_pools_config.png
 docs/images/impala_arch.jpeg
 docs/images/support_send_diagnostic_data.png
+
+# Files of deployment configuration
+package/conf/*
--- a/buildall.sh
+++ b/buildall.sh
@@ -69,6 +69,7 @@ BUILD_ASAN=0
 BUILD_FE_ONLY=0
 BUILD_TESTS=1
 GEN_CMAKE_ONLY=0
+GEN_PACKAGE=0
 BUILD_RELEASE_AND_DEBUG=0
 BUILD_TIDY=0
 BUILD_UBSAN=0
@@ -204,6 +205,9 @@ do
    -cmake_only)
      GEN_CMAKE_ONLY=1
      ;;
+    -package)
+      GEN_PACKAGE=1
+      ;;
    -help|*)
      echo "buildall.sh - Builds Impala and runs all tests."
      echo "[-noclean] : Omits cleaning all packages before building. Will not kill"\
@@ -258,6 +262,7 @@ do
      echo "[-fe_only] : Build just the frontend"
      echo "[-ninja] : Use ninja instead of make"
      echo "[-cmake_only] : Generate makefiles only, instead of doing a full build"
+      echo "[-package] : Generate a package for deployment."
      echo "-----------------------------------------------------------------------------
 Examples of common tasks:

@@ -537,6 +542,9 @@ generate_cmake_files() {
    echo "CACHELINESIZE_AARCH64:$CACHELINESIZE_AARCH64"
    CMAKE_ARGS+=(-DCACHELINESIZE_AARCH64=$CACHELINESIZE_AARCH64)
  fi
+  if [[ "$GEN_PACKAGE" -eq 1 ]]; then
+    CMAKE_ARGS+=(-DBUILD_PACKAGES=ON)
+  fi

  cmake . ${CMAKE_ARGS[@]}
 }
@@ -660,6 +668,10 @@ else
  build_all_components $CMAKE_BUILD_TYPE 1
 fi

+if [[ "$GEN_PACKAGE" -eq 1 ]]; then
+  ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} package
+fi
+
 if [[ $NEED_MINICLUSTER -eq 1 ]]; then
  reconfigure_test_cluster
 fi
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -25,11 +25,36 @@ set -euo pipefail

 INSTALL_DEBUG_TOOLS=false
 JAVA_VERSION=8
+DRY_RUN=false
+PKG_LIST=""
+NON_PKG_NAMES=(apt-get yum install update)

 function print_usage {
    echo "install_os_packages.sh - Helper script to install OS dependencies"
    echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc"
    echo "[--java <version>] : Use specified Java version rather than the default Java 8."
+    echo "[--dry-run] : Print the list of packages to install."
+}
+
+# Wraps the passed in command to either execute it (DRY_RUN=false) or just use it
+# to update PKG_LIST.
+function wrap {
+  if $DRY_RUN; then
+    for arg in $@; do
+      if [[ "${NON_PKG_NAMES[@]}"  =~ "$arg" ]]; then
+        continue
+      elif [[ "$arg" == "-"* ]]; then
+        # Ignores command options
+        continue
+      elif [[ "$PKG_LIST" != "" ]]; then
+        PKG_LIST="$PKG_LIST,$arg"
+      else
+        PKG_LIST="$arg"
+      fi
+    done
+  else
+    "$@"
+  fi
 }

 while [ -n "$*" ]
@@ -42,6 +67,9 @@ do
      JAVA_VERSION="${2-}"
      shift;
      ;;
+    --dry-run)
+      DRY_RUN=true
+      ;;
    --help|*)
      print_usage
      exit 1
@@ -52,6 +80,7 @@ done

 echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}"
 echo "JAVA_VERSION=${JAVA_VERSION}"
+echo "DRY_RUN=${DRY_RUN}"

 # This can get more detailed if there are specific steps
 # for specific versions, but at the moment the distribution
@@ -85,8 +114,8 @@ fi
 # Optionally install extra debug tools.
 if [[ $DISTRIBUTION == Ubuntu ]]; then
  export DEBIAN_FRONTEND=noninteractive
-  apt-get update
-  apt-get install -y \
+  wrap apt-get update
+  wrap apt-get install -y \
      hostname \
      krb5-user \
      language-pack-en \
@@ -97,7 +126,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
      tzdata
  if $INSTALL_DEBUG_TOOLS ; then
    echo "Installing extra debug tools"
-    apt-get install -y \
+    wrap apt-get install -y \
        curl \
        dnsutils \
        iproute2 \
@@ -111,7 +140,7 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
  if [[ $JAVA_VERSION == 8 ]]; then
    JAVA_VERSION=1.8.0
  fi
-  yum install -y --disableplugin=subscription-manager \
+  wrap yum install -y --disableplugin=subscription-manager \
      cyrus-sasl-gssapi \
      cyrus-sasl-plain \
      hostname \
@@ -125,14 +154,14 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
  # Install the appropriate language packs. Redhat/Centos 7 come
  # with en_US.utf8, so there is no need to install anything.
  if ! grep 'release 7\.' /etc/redhat-release; then
-      yum install -y --disableplugin=subscription-manager \
+      wrap yum install -y --disableplugin=subscription-manager \
          glibc-langpack-en \
          langpacks-en
  fi

  if $INSTALL_DEBUG_TOOLS ; then
    echo "Installing extra debug tools"
-    yum install -y --disableplugin=subscription-manager \
+    wrap yum install -y --disableplugin=subscription-manager \
        bind-utils \
        curl \
        iproute \
@@ -145,6 +174,12 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
  fi
 fi

+if $DRY_RUN; then
+  echo "The following packages would be installed:"
+  echo "$PKG_LIST"
+  exit 0
+fi
+
 # Verify en_US.utf8 is present
 if ! locale -a | grep en_US.utf8 ; then
  echo "ERROR: en_US.utf8 locale is not present."
--- a/package/bin/impala-env.sh
+++ b/package/bin/impala-env.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [[ -z "$JAVA_HOME" ]]; then
+  echo "JAVA_HOME not set!"
+  exit 1
+fi
+
+echo "Using JAVA_HOME: $JAVA_HOME"
+LIB_JVM_DIR=$(dirname $(find $JAVA_HOME -type f -name libjvm.so))
+LIB_JSIG_DIR=$(dirname $(find $JAVA_HOME -type f -name libjsig.so))
+
+export LC_ALL=en_US.utf8
+export LD_LIBRARY_PATH="/opt/impala/lib/:$LIB_JVM_DIR:$LIB_JSIG_DIR"
+export CLASSPATH="/opt/impala/conf:/opt/impala/jar/*"
+
+#TODO: Add graceful shutdown for impalads
+function stop_process {
+  name=$1
+  pid_file="/tmp/${name}.pid"
+  if [[ -f $pid_file ]]; then
+    PID=$(cat $pid_file)
+    if ps $PID | grep $name; then
+      echo "Killing $name with PID=$PID"
+      kill $PID
+      rm $pid_file
+      echo "Killed $name"
+    else
+      rm $pid_file
+      echo "Already stopped: $name is not running with PID=$PID. Removed stale $pid_file"
+    fi
+  else
+    echo "PID file $pid_file not found!"
+  fi
+}
+
+function wait_for_ready {
+  name=$1
+  port=$2
+  pid=$3
+
+  NUM_WAIT_ITERATIONS=20
+  i=0
+  while [[ $i -lt $NUM_WAIT_ITERATIONS ]]; do
+    if ! ps $pid | grep $name > /dev/null 2>&1; then
+      echo "$name with PID $pid doesn't exist"
+      break
+    fi
+    STATUS=$(curl -s http://localhost:$port/healthz)
+    if [[ $? != 0 ]]; then
+      echo "Waiting for $name. Port $port not ready."
+    elif [[ "$STATUS" != "OK" ]]; then
+      echo "Waiting for $name to be ready"
+    else
+      echo "$name is ready"
+      break
+    fi
+    sleep 2
+    i=$((i+1))
+  done
+  if [[ "$STATUS" == "OK" ]]; then
+    echo "Launched $name with PID $pid"
+  elif [[ $i -eq $NUM_WAIT_ITERATIONS ]]; then
+    echo "Timed out waiting for $name to be ready. Check logs for more details."
+  else
+    echo "Failed to launch $name"
+    exit 1
+  fi
+}
--- a/package/bin/start-catalogd.sh
+++ b/package/bin/start-catalogd.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch catalogd. Required JAVA_HOME being set.
+# Edit conf/catalogd_flags to set the correct hostname and state_store_host.
+# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster.
+# Example usage:
+#   export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+#   bin/start-catalogd.sh
+# To launch catalogd using another username (e.g. "impala"):
+#   sudo -E -u impala bin/start-catalogd.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+$IMPALA_HOME/bin/catalogd --flagfile=$IMPALA_HOME/conf/catalogd_flags &
+PID=$!
+echo $PID > /tmp/catalogd.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready catalogd 25020 $PID
--- a/package/bin/start-impalad.sh
+++ b/package/bin/start-impalad.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch impalad. Required JAVA_HOME being set.
+# Edit conf/impalad_flags to set the correct hostnames.
+# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster.
+# Example usage:
+#   export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+#   bin/start-impalad.sh
+# To launch impalad using another username (e.g. "impala"):
+#   sudo -E -u impala bin/start-impalad.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+if [[ -n "$HADOOP_HOME" ]]; then
+  echo "Using HADOOP_HOME: $HADOOP_HOME"
+  export HADOOP_LIB_DIR="${HADOOP_HOME}/lib"
+  export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${HADOOP_LIB_DIR}/native/"
+  echo "Using hadoop native libs in ${HADOOP_LIB_DIR}/native/"
+else
+  export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${IMPALA_HOME}/lib"
+  echo "HADOOP_HOME not set. Using hadoop native libs in ${IMPALA_HOME}/lib"
+fi
+
+$IMPALA_HOME/bin/impalad --flagfile=$IMPALA_HOME/conf/impalad_flags &
+PID=$!
+echo $PID > /tmp/impalad.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready impalad 25000 $PID
--- a/package/bin/start-statestored.sh
+++ b/package/bin/start-statestored.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch statestore. Required JAVA_HOME being set.
+# Edit conf/statestore_flags to set a correct hostname.
+# Example usage:
+#   export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+#   bin/start-statestored.sh
+# To launch statestore using another username (e.g. "impala"):
+#   sudo -E -u impala bin/start-statestored.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+$IMPALA_HOME/bin/statestored --flagfile=$IMPALA_HOME/conf/statestore_flags &
+PID=$!
+echo $PID > /tmp/statestored.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready statestored 25010 $PID
--- a/package/bin/stop-catalogd.sh
+++ b/package/bin/stop-catalogd.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process catalogd
--- a/package/bin/stop-impalad.sh
+++ b/package/bin/stop-impalad.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process impalad
--- a/package/bin/stop-statestored.sh
+++ b/package/bin/stop-statestored.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process statestored
--- a/package/conf/catalogd_flags
+++ b/package/conf/catalogd_flags
@@ -0,0 +1,13 @@
+-hostname=localhost
+-state_store_host=localhost
+#-kudu_master_hosts=localhost
+
+-log_dir=/var/log/catalogd
+-log_filename=catalogd
+-state_store_port=24000
+-minidump_path=/var/log/impala-minidumps
+-webserver_doc_root=/opt/impala
+-catalog_topic_mode=minimal
+-hms_event_polling_interval_s=0
+-v=1
+-max_log_size=200
--- a/package/conf/core-site.xml
+++ b/package/conf/core-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://localhost:8020</value>
+  </property>
+  <property>
+    <name>ipc.client.connection.maxidletime</name>
+    <value>30000</value>
+  </property>
+  <property>
+    <name>ipc.client.connect.max.retries</name>
+    <value>50</value>
+  </property>
+  <property>
+    <name>fs.trash.interval</name>
+    <value>1</value>
+  </property>
+</configuration>
--- a/package/conf/fair-scheduler.xml
+++ b/package/conf/fair-scheduler.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<allocations>
+    <queue name="root">
+        <queue name="default"/>
+    </queue>
+</allocations>
--- a/package/conf/hdfs-site.xml
+++ b/package/conf/hdfs-site.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+  <property>
+    <name>dfs.namenode.servicerpc-address</name>
+    <value>localhost:8022</value>
+  </property>
+  <property>
+    <name>dfs.namenode.http-address</name>
+    <value>localhost:20101</value>
+  </property>
+  <property>
+    <name>dfs.replication</name>
+    <value>3</value>
+  </property>
+  <property>
+    <name>dfs.blocksize</name>
+    <value>134217728</value>
+  </property>
+  <property>
+    <name>dfs.domain.socket.path</name>
+    <value>/var/run/hdfs-sockets/dn</value>
+  </property>
+  <property>
+    <name>dfs.client.read.shortcircuit</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.client.read.shortcircuit.streams.cache.size</name>
+    <value>4096</value>
+  </property>
+  <property>
+    <name>dfs.client.read.shortcircuit.skip.checksum</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>dfs.client.file-block-storage-locations.timeout.millis</name>
+    <value>10000</value>
+  </property>
+</configuration>
--- a/package/conf/hive-site.xml
+++ b/package/conf/hive-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+  <property>
+    <name>hive.metastore.uris</name>
+    <value>thrift://localhost:9083</value>
+  </property>
+  <property>
+    <name>hive.support.concurrency</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hive.metastore.client.socket.timeout</name>
+    <value>3600</value>
+  </property>
+  <property>
+    <name>hive.metastore.connect.retries</name>
+    <value>5</value>
+  </property>
+</configuration>
--- a/package/conf/impalad_flags
+++ b/package/conf/impalad_flags
@@ -0,0 +1,16 @@
+-hostname=localhost
+-state_store_host=localhost
+-catalog_service_host=localhost
+#-kudu_master_hosts=localhost
+
+-mem_limit=80%
+-use_local_catalog=true
+-log_dir=/var/log/impalad
+-log_filename=impalad
+-minidump_path=/var/log/impala-minidumps
+-local_library_dir=/var/lib/impala/udfs
+-fair_scheduler_allocation_path=/opt/impala/conf/fair-scheduler.xml
+-llama_site_path=/opt/impala/conf/llama-site.xml
+-webserver_doc_root=/opt/impala
+-v=1
+-max_log_size=200
--- a/package/conf/llama-site.xml
+++ b/package/conf/llama-site.xml
@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration/>
--- a/package/conf/statestore_flags
+++ b/package/conf/statestore_flags
@@ -0,0 +1,7 @@
+-hostname=localhost
+-log_dir=/var/log/statestore
+-log_filename=statestored
+-minidump_path=/var/log/impala-minidumps
+-webserver_doc_root=/opt/impala
+-v=1
+-max_log_size=200