impala/testdata/cluster/admin

#!/bin/bash

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This will create/control/destroy a local hdfs/yarn/kms or ozone + kudu cluster.
#
# All roles run on 127.0.0.1, just like the standard mini cluster included with hadoop.
# The difference is with this cluster, each role runs in its own process and has its own
# configs. For each node, the layout of the configs, logs, start/stop scripts, etc, is
# kept as close as possible to a real cluster. For example, the first node will live in
# the dir "cdh-<version>/node-1" and its logs would be at "cdh-<version>/node-1/var/log".
# TODO: Run each node on its own IP address, e.g. 127.0.0.1, 127.0.0.2, and so on.

set -euo pipefail
. $IMPALA_HOME/bin/report_build_error.sh
setup_report_build_error

while getopts vy OPT; do
  case $OPT in
    v) set -x;;
    ?) echo "Usage: $0 [-v (verbose) -k (kerberize)] ACTION (see source...)"; exit 1;;
  esac
done
shift $(($OPTIND-1))

DIR=$(dirname $0)
NODE_COUNT=3
if [[ "$ERASURE_CODING" = true ]]; then
  NODE_COUNT=5
fi
NODE_PREFIX=node-
COMMON_NODE_TEMPLATE="$DIR/node_templates/common"
NODE_TEMPLATE="$DIR/node_templates/cdh$CDH_MAJOR_VERSION"
TEMPLATE_SUFFIX=".tmpl"
PY_TEMPLATE_SUFFIX=".xml.py"

# Each process should be marked with this so a "pkill -f" can be done to nuke everything.
export KILL_CLUSTER_MARKER=IBelongToTheMiniCluster

if [[ "$TARGET_FILESYSTEM" == "hdfs" ]]; then
  # The check above indicates that the regular mini-cluster is in use.
  SUPPORTED_SERVICES=(hdfs kms yarn)
elif [[ "$TARGET_FILESYSTEM" == "ozone" ]]; then
  SUPPORTED_SERVICES=(kms ozone)
else
  # Either a remote distributed file system or a local non-distributed file system is
  # in use. Currently the only service that is expected to work is Kudu, though in theory
  # the other services could work after the proper configuration changes.
  SUPPORTED_SERVICES=()
fi

# All DataNodes and NodeManagers need a unique but fixed address. The IP is fixed at
# 127.0.0.1, so the only difference is the port. The address must be fixed because it is
# used as an identifier. If a node were restarted with a different address it would be
# considered a new node. The values below are arbitrary and may conflict with existing
# services. Fixed ports were preferred to dynamically chosen free ports for consistency.
DATANODE_FREE_PORT_START=31000
DATANODE_FREE_HTTP_PORT_START=31010
DATANODE_FREE_IPC_PORT_START=31020
DATANODE_FREE_HTTPS_PORT_START=31030
DATANODE_FREE_CLIENT_PORT_START=31040
NODEMANAGER_FREE_PORT_START=31100
NODEMANAGER_FREE_LOCALIZER_PORT_START=31120
NODEMANAGER_FREE_WEBUI_PORT_START=31140
KUDU_TS_RPC_FREE_PORT_START=31200
KUDU_TS_WEBUI_FREE_PORT_START=31300

# Used to populate config templates later. Any changes made here have no effect on an
# existing cluster.
export HDFS_WEBUI_PORT=5070   # changed from 50070 so it is not ephemeral
export YARN_WEBUI_PORT=8088   # same as default
export KMS_WEBUI_PORT=9600    # changed to make room for non-ephemeral HBase ports
                              # (HADOOP-12811)
export KUDU_WEBUI_PORT=8051   # same as default
export OZONE_WEBUI_PORT=9874  # same as default for OM WebUI

# Empty dirs that should be included in the templates. Since git ignores empty dirs it is
# easier to maintain them here.
EMPTY_NODE_DIRS=$(echo data/dfs/{dn,nn} var/{run,lib/hadoop-hdfs,log} \
    var/{log,run}/kudu/{master,ts} var/lib/kudu/{master,ts}/{wal,data})

EASY_ACCESS_LOG_DIR="$IMPALA_CLUSTER_LOGS_DIR"

FIND_EXECUTABLE_FILTER="-executable"
if ! find /dev/null "${FIND_EXECUTABLE_FILTER}" 2> /dev/null; then
  # OSX and RHEL5, among others, don't have -executable
  FIND_EXECUTABLE_FILTER="-perm +0111"
fi

#
# Various initialization routines for a Kerberized cluster setup.  We
# do this if either
#    1) IMPALA_KERBERIZE is set
#    2) The config files exist and use kerberos
# If either are true, then we set IMPALA_KERBERIZE ourselves, ensure the
# minikdc is started, source the appropriate env vars from it, and get
# ourselves a fresh TGT (Ticket-Granting-Ticket).
#
function kerberize_setup {
  # No kerberos?  We're done.
  if [[ "${IMPALA_KERBERIZE}" != "true" ]]; then
    return
  fi

  # We must have the 'kinit' binary installed so that the minikdc can
  # grant the user a ticket-granting-ticket.
  if ! type -f kinit > /dev/null 2>&1; then
    echo "Unable to locate 'kinit' on this machine, it must be installed"
    echo "in order for you to actually communicate with the kerberized"
    echo "components of the cluster."
    echo "Try this (on ubuntu):"
    echo " --> $ sudo apt-get install krb5-user"
    echo
    exit 1
  fi

  # While we're checking things, make sure the SASL GSSAPI libraries
  # are installed.  These aren't required until we go to start up
  # the impala daemons, but we might as well fail early...
  IMPALA_SASL_PATH="/usr/lib/sasl2 /usr/lib64/sasl2 /usr/local/lib/sasl2"
  IMPALA_SASL_PATH="${IMPALA_SASL_PATH} /usr/lib/x86_64-linux-gnu/sasl2"
  SASL_GSSAPI_FOUND=0
  for i in ${IMPALA_SASL_PATH}; do
    if ls $i/libgssapi* > /dev/null 2>&1; then
      SASL_GSSAPI_FOUND=1
      break
    fi
  done
  if [ ${SASL_GSSAPI_FOUND} -eq 0 ]; then
    echo "Unable to locate the SASL GSSAPI libraries in the following path:"
    echo "${IMPALA_SASL_PATH}"
    echo "This is required for Kerberos communication using SASL, which"
    echo "the Impala daemons depend on.  To install (on Ubuntu), try:"
    echo " --> $ sudo apt-get install libsasl2-modules-gssapi-mit"
    echo
    exit 1
  fi

  # Source the appropriate minikdc environment variables
  . ${MINIKDC_ENV}

  # Give ourselves a fresh TGT.  Debate here whether that should be
  # ${USER} or 'impala'.  Each has issues; see comments in buildall.sh.
  # Stick with ${USER} for now.
  kinit -k -t "${KRB5_KTNAME}" "${MINIKDC_PRINC_USER}"
  if [ $? -ne 0 ]; then
    echo "kinit failure; aborting"
    exit 1
  fi
}

# Return success if the cluster is kerberized
function is_kerberized {
  HCONFSC="`get_hadoop_client_conf_dir`/core-site.xml"
  if [ -f "${HCONFSC}" ]; then
    if grep -qi "kerberos" "${HCONFSC}"; then
      # If the config exists and has kerberos things in it, treat as kerberized
      return 0
    fi
  fi

  return 1
}

function cluster_exists {
  # Just use the first node as an indicator...
  if [[ ! -e "$IMPALA_CLUSTER_NODES_DIR/${NODE_PREFIX}1" ]]; then
    return 1
  fi
}

function kudu_cluster_exists {
  # Just use the first master as an indicator.
  if [[ ! -e "$IMPALA_CLUSTER_NODES_DIR/${NODE_PREFIX}1/var/lib/kudu/master" ]]; then
    return 1
  fi
}

function create_cluster {
  mkdir -p "$IMPALA_CLUSTER_NODES_DIR"

  # Used to populate config templates later
  GROUP=$(id -gn)
  export GROUP

  # Blow away existing config files (so we don't pick up kerberos settings)
  rm -f `get_hadoop_client_conf_dir`/*

  if [[ "${IMPALA_KERBERIZE}" = "true" ]]; then
    kerberize_setup
    echo "Creating Kerberized cluster."
  fi

  echo "Hostname for internal communication: ${INTERNAL_LISTEN_HOST}" \
       "and for external communication: ${EXTERNAL_LISTEN_HOST}"
  # For consistency, the first node will host all the master roles.
  for ((NODE_IDX=$NODE_COUNT; NODE_IDX >= 1; NODE_IDX--)); do
    NODE=${NODE_PREFIX}$NODE_IDX
    NODE_DIR=$(get_node_dir $NODE)

    echo "Creating $NODE at $NODE_DIR"

    mkdir -p "$NODE_DIR"
    # Copy recursively and dereference symlinks. This allows us to use symlinks
    # in the node_template directory without worrying about dangling symlinks.
    cp -RL "$COMMON_NODE_TEMPLATE"/* "$NODE_DIR"
    if [[ -e "$NODE_TEMPLATE" ]]; then
      cp -RL "$NODE_TEMPLATE"/* "$NODE_DIR"
    fi
    if [[ $NODE_IDX -gt 1 ]]; then
      # Remove master role scripts from slave nodes
      rm -f "$NODE_DIR/etc/init.d/"{hdfs-namenode,yarn-resourcemanager} \
            "$NODE_DIR/etc/init.d/"{kms,kudu-master,ozone-scm,ozone-manager}
      # Only run one YARN nodemanager (more memory-efficient to scale up a
      # single NM than run several)
      rm -f "$NODE_DIR/etc/init.d/yarn-nodemanager"
    fi
    for EMPTY_NODE_DIR in $EMPTY_NODE_DIRS; do
      mkdir -p "$NODE_DIR/$EMPTY_NODE_DIR"
    done

    # Add some easy access links closer to IMPALA_HOME
    EASY_ACCESS_LOG_LINK="$EASY_ACCESS_LOG_DIR/cdh$CDH_MAJOR_VERSION-$NODE"
    if [[ -e "$EASY_ACCESS_LOG_LINK" ]]; then
      rm "${EASY_ACCESS_LOG_LINK}"
    fi
    mkdir -p "$EASY_ACCESS_LOG_DIR"
    ln -s "$NODE_DIR/var/log" "$EASY_ACCESS_LOG_DIR"
    mv "$EASY_ACCESS_LOG_DIR/log" "$EASY_ACCESS_LOG_LINK"

    # Template population
    DATANODE_PORT=$((DATANODE_FREE_PORT_START++))
    DATANODE_HTTP_PORT=$((DATANODE_FREE_HTTP_PORT_START++))
    DATANODE_IPC_PORT=$((DATANODE_FREE_IPC_PORT_START++))
    DATANODE_HTTPS_PORT=$((DATANODE_FREE_HTTPS_PORT_START++))
    DATANODE_CLIENT_PORT=$((DATANODE_FREE_CLIENT_PORT_START++))
    ULIMIT_LOCKED_MEM="$(ulimit -l)"
    if [[ "${ULIMIT_LOCKED_MEM}" == "unlimited" ]]; then
      # Use a default of 64MB for HDFS caching. Should match memlock in bootstrap_system.
      ULIMIT_LOCKED_MEM=65536
    fi
    # Allocate slightly less than memlock to each datanode.
    DATANODE_LOCKED_MEM=$((ULIMIT_LOCKED_MEM*1000))
    NODEMANAGER_PORT=$((NODEMANAGER_FREE_PORT_START++))
    NODEMANAGER_LOCALIZER_PORT=$((NODEMANAGER_FREE_LOCALIZER_PORT_START++))
    NODEMANAGER_WEBUI_PORT=$((NODEMANAGER_FREE_WEBUI_PORT_START++))
    KUDU_TS_RPC_PORT=$((KUDU_TS_RPC_FREE_PORT_START++))
    KUDU_TS_WEBUI_PORT=$((KUDU_TS_WEBUI_FREE_PORT_START++))
    echo "$NODE will use ports DATANODE_PORT=$DATANODE_PORT," \
        "DATANODE_HTTP_PORT=$DATANODE_HTTP_PORT," \
        "DATANODE_IPC_PORT=$DATANODE_IPC_PORT," \
        "DATANODE_HTTPS_PORT=$DATANODE_HTTPS_PORT," \
        "DATANODE_CLIENT_PORT=$DATANODE_CLIENT_PORT," \
        "NODEMANAGER_PORT=$NODEMANAGER_PORT," \
        "NODEMANAGER_LOCALIZER_PORT=$NODEMANAGER_LOCALIZER_PORT," \
        "NODEMANAGER_WEBUI_PORT=$NODEMANAGER_WEBUI_PORT," \
        "KUDU_TS_RPC_PORT=$KUDU_TS_RPC_PORT," \
        "and KUDU_TS_WEBUI_PORT=$KUDU_TS_WEBUI_PORT;" \
        "DATANODE_LOCKED_MEM=$DATANODE_LOCKED_MEM"

    export NODE NODE_DIR
    export DATANODE_PORT DATANODE_HTTP_PORT DATANODE_IPC_PORT DATANODE_HTTPS_PORT
    export DATANODE_CLIENT_PORT DATANODE_LOCKED_MEM
    export NODEMANAGER_PORT NODEMANAGER_LOCALIZER_PORT NODEMANAGER_WEBUI_PORT
    export KUDU_TS_RPC_PORT KUDU_TS_WEBUI_PORT
    for TEMPLATE_PATH in $(find "$NODE_DIR" -name "*$TEMPLATE_SUFFIX"); do
      ACTUAL_PATH="${TEMPLATE_PATH%$TEMPLATE_SUFFIX}"

      # Search for strings like ${FOO}, if FOO is defined in the environment then replace
      # "${FOO}" with the environment value.
      perl -wpl -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' \
          "$TEMPLATE_PATH" > "$ACTUAL_PATH.1"

      # Chop out everything between the BEGIN/END Kerberos comments if
      # not kerberized
      if [[ "${IMPALA_KERBERIZE}" != "true" ]]; then
        sed '/<!-- BEGIN Kerberos/,/END Kerberos settings -->/d' \
            "$ACTUAL_PATH.1" > "$ACTUAL_PATH"
      else
        cp "$ACTUAL_PATH.1" "$ACTUAL_PATH"
      fi

      # Assumes that environment variables will be ALL CAPS...
      if grep '\${[A-Z_]*}' "$ACTUAL_PATH"; then
        echo "Found undefined variables in $ACTUAL_PATH, aborting"
        exit 1
      fi

      if [[ -x "$TEMPLATE_PATH" ]]; then
        chmod u+x "$ACTUAL_PATH"
      fi
      rm "$TEMPLATE_PATH" "$ACTUAL_PATH.1"
    done
    # Substitute python-templated XML files.
    # TODO(todd): move over all the XML templates to be Python-based.
    for TEMPLATE_PATH in $(find "$NODE_DIR" -name "*$PY_TEMPLATE_SUFFIX"); do
      ACTUAL_PATH="${TEMPLATE_PATH%$PY_TEMPLATE_SUFFIX}".xml
      $IMPALA_HOME/bin/generate_xml_config.py $TEMPLATE_PATH $ACTUAL_PATH
      rm $TEMPLATE_PATH
      # Remove pycache if created
      rm -rf $(dirname $TEMPLATE_PATH)/__pycache__
    done
  done
}

function start_cluster {
  if ! cluster_exists; then
    echo "The cluster must be created first"
    return 1
  fi

  if [[ "${IMPALA_KERBERIZE}" = "true" ]] && ! is_kerberized; then
    echo "Kerberized start requested, but the config files aren't set up"
    echo "for kerberos. You must regenerate configurations with "
    echo  "  ./bin/create-test-configuration.sh"
    exit 1
  fi

  kerberize_setup
  if is_kerberized; then
    echo "Starting Kerberized cluster."
  fi

  if [ ${#SUPPORTED_SERVICES[@]} -gt 0 ]; then
    for SERVICE in ${SUPPORTED_SERVICES[@]-}; do
      start $SERVICE
    done

    # If there any services to start, check that the cluster is still alive...
    sleep 10
    check_cluster_status
  fi

  if [[ "${TARGET_FILESYSTEM}" = "ozone" ]]; then
    local replication=''
    echo "Creating Ozone volume/bucket"
    if [ -n "${OZONE_ERASURECODE_POLICY:-}" ]; then
      replication="--type EC --replication ${OZONE_ERASURECODE_POLICY}"
      echo "with erasure coding ${OZONE_ERASURECODE_POLICY}"
    fi

    local bucketkey=''
    if $USE_OZONE_ENCRYPTION; then
      echo "Ozone encryption enabled for ${OZONE_VOLUME}/test-warehouse"

      # Encryption is done at the bucket level, so ensure the keys are available first.
      ${IMPALA_HOME}/testdata/bin/setup-dfs-keys.sh testkey
      bucketkey='--bucketkey testkey'
    fi

    ozone sh volume create ${OZONE_VOLUME} || true
    ozone sh bucket create ${bucketkey} ${replication} \
        ${OZONE_VOLUME}/test-warehouse || true
  fi

  return $?
}

function start_kudu_cluster {
  if ! kudu_cluster_exists; then
    echo "The Kudu cluster must be created first"
    return 1
  fi
  start "kudu"
  sleep 10
  check_kudu_cluster_status
}

function start {
  if [[ $# -ne 1 ]]; then
    echo start must be called with a single argument -- the service to start. 1>&2
    exit 1
  fi
  local SERVICE=$1

  WEBUI_PORT_VAR="$(echo "$SERVICE" | awk '{print toupper($0)}')_WEBUI_PORT"
  echo "Starting ${SERVICE} (Web UI - http://localhost:${!WEBUI_PORT_VAR})"
  exec_init_script "$SERVICE*" start
  if [ "${SERVICE}" = "hdfs" ]; then
      chmod_hdfs_root
  fi
}

function chmod_hdfs_root {
    # When created, the cluster only contains the root directory with
    # 0755 permissions and owned by hdfs.  In a kerberized
    # environment, this means that no one but hdfs can create
    # anything!  Chmod / to 01777 so that other users can create
    # things.  Insecure, but easy.  While we're here, make other sane
    # and permissive initial directories
    if is_kerberized; then
        PREVIOUS_PRINCIPAL=`klist | grep ^Default | awk '{print $3}'`
        # Become hdfs:
        kinit -k -t ${KRB5_KTNAME} ${MINIKDC_PRINC_HDFS}
        # Do the chmod, etc
        hadoop fs -chmod 1777 /
        if ! hadoop fs -ls /tmp > /dev/null 2>&1; then
            for i in /tmp /home /user; do
                hadoop fs -mkdir $i
                hadoop fs -chmod 1777 $i
            done
        fi
        # Become ourselves again.
        kinit -k -t ${KRB5_KTNAME} ${PREVIOUS_PRINCIPAL}
    fi
}

function exec_init_script {
  local SCRIPT_NAME="$1"
  shift
  local CMD="$1"

  local PIDS=()
  for SCRIPT in $(find "$IMPALA_CLUSTER_NODES_DIR" -path "*/$NODE_PREFIX*/etc/init.d/$SCRIPT_NAME" \
      $FIND_EXECUTABLE_FILTER -type f); do
    if "$SCRIPT" status &>/dev/null; then
      RUNNING=true
    else
      RUNNING=false
    fi
    case "$CMD" in
      start) if ! $RUNNING; then "$SCRIPT" start; fi;;
      stop) if $RUNNING; then "$SCRIPT" stop; fi;;
      *) "$SCRIPT" "$@";;
    esac &
    PIDS+=($!)
  done

  # Always wait for all the pids, otherwise the output gets messed up. If this function
  # were to return while the background processes are still running, they may print
  # output that makes it appear as though this process is still running.
  local RET=0
  for PID in ${PIDS[@]}; do
    if ! wait $PID; then
      RET=1
    fi
  done
  return $RET
}

function check_cluster_status {
  if ! cluster_exists; then
    echo "The cluster does not exist"
    return 1
  fi

  for SERVICE in ${SUPPORTED_SERVICES[@]-}; do
    check_status $SERVICE
  done
}

function check_kudu_cluster_status {
  if ! kudu_cluster_exists; then
    echo "The Kudu cluster does not exist"
    return 1
  fi

  check_status "kudu"
}

function check_status {
  if [[ $# -ne 1 ]]; then
    echo stop must be called with a single argument -- the service to check. 1>&2
    exit 1
  fi
  local SERVICE=$1

  ROLE_COUNT=0
  NOT_RUNNING=()
  for NODE_DIR in "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*; do
    for SCRIPT in $(find "$NODE_DIR" -path "*/etc/init.d/$SERVICE*" $FIND_EXECUTABLE_FILTER \
        -type f); do
      ROLE_COUNT=$((ROLE_COUNT + 1))
      if ! "$SCRIPT" status &>/dev/null; then
        NOT_RUNNING+=("\n$(basename $SCRIPT) is not running on $(basename $NODE_DIR)")
      fi
    done
  done

  case ${#NOT_RUNNING[@]} in
    0) echo "The ${SERVICE} cluster is running"; return;;
    $ROLE_COUNT) echo "The ${SERVICE} cluster is not running"; return 1;;
    *) echo -e "${NOT_RUNNING[@]}"; return 1;;
  esac
}

function stop_cluster {
  if cluster_exists; then
    # Stop services in reverse order to give them a chance to shutdown cleanly
    for ((SERVICE_IDX=${#SUPPORTED_SERVICES[@]} - 1; SERVICE_IDX >= 0; SERVICE_IDX--)); do
      stop "${SUPPORTED_SERVICES[$SERVICE_IDX]}"
    done
  fi
  # Kill everything anyways just in case a git clean -xdf was done
  pkill -u $USER -f $KILL_CLUSTER_MARKER || true
}

function stop_kudu_cluster {
  if kudu_cluster_exists; then
    stop "kudu"
  fi
}

function stop {
  if [[ $# -ne 1 ]]; then
    echo stop must be called with a single argument -- the service to stop. 1>&2
    exit 1
  fi
  local SERVICE=$1

  echo "Stopping ${SERVICE}"
  exec_init_script "$SERVICE*" stop
}

function restart {
  if [[ $# -ne 1 ]]; then
    echo restart must be called with a single argument -- the service to restart. 1>&2
    exit 1
  fi
  local SERVICE=$1
  stop $SERVICE
  start $SERVICE
}

function delete_data {
  # Delete namenode, datanode and KMS data while preserving directory structure.
  rm -rf "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/dfs/{nn,dn}/*
  rm -f "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/kms.keystore
  rm -rf "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/ozone
}

function delete_kudu_data {
  rm -rf "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/var/lib/kudu/{master,ts}/*
}

function delete_cluster {
  # Delete all cluster data and directory structure.
  pkill -u $USER -f $KILL_CLUSTER_MARKER || true
  rm -rf "$IMPALA_CLUSTER_NODES_DIR"
}

function get_node_dir {
  if $IS_OSX; then
    greadlink -f "$IMPALA_CLUSTER_NODES_DIR/$1"
  else
    readlink -f "$IMPALA_CLUSTER_NODES_DIR/$1"
  fi
}

function get_hadoop_client_conf_dir {
  echo "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"1/etc/hadoop/conf
}

COMMAND=$1
shift
case $COMMAND in
  check_cluster_status | check_kudu_cluster_status | cluster_exists | \
        kudu_cluster_exists | is_kerberized)
    # Use an "if" to avoid triggering the ERR trap.
    if ! $COMMAND "$@"; then
      exit 1
    fi;;
  *) $COMMAND "$@";;
esac