IMPALA-12852: Make Kudu service start and stop independent

This patch decouples run-kudu.sh and kill-kudu.sh from run-mini-dfs.sh
and kill-mini-dfs.sh. These scripts can be useful for setting up test
environments that require no or only Kudu service.

Testing:
  - Ran the modified and new scripts and checked they worked as expected.

Change-Id: I9624aaa61353bb4520e879570e5688d5e3493201
Reviewed-on: http://gerrit.cloudera.org:8080/21090
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

testdata/bin/kill-kudu.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+$IMPALA_HOME/testdata/cluster/admin stop_kudu_cluster
+sleep 2

testdata/bin/run-all.sh

@@ -23,11 +23,11 @@ set -euo pipefail
 . $IMPALA_HOME/bin/report_build_error.sh
 setup_report_build_error
 
-# If -format is passed, format the mini-dfs cluster.
+# If -format is passed, format the mini-dfs cluster and kudu cluster.
 
 if [[ $# -eq 1 && "$1" == "-format" ]]; then
   echo "Formatting cluster"
-  HDFS_FORMAT_CLUSTER="-format"
+  FORMAT_CLUSTER="-format"
 elif [[ $# -ne 0 ]]; then
   echo "Usage: run-all.sh [-format]"
   echo "[-format] : Format the mini-dfs cluster before starting"
@@ -52,8 +52,10 @@ fi
 popd
 
 echo "Starting cluster services..."
-$IMPALA_HOME/testdata/bin/run-mini-dfs.sh ${HDFS_FORMAT_CLUSTER-} 2>&1 | \
+$IMPALA_HOME/testdata/bin/run-mini-dfs.sh ${FORMAT_CLUSTER-} 2>&1 | \
     tee ${IMPALA_CLUSTER_LOGS_DIR}/run-mini-dfs.log
+$IMPALA_HOME/testdata/bin/run-kudu.sh ${FORMAT_CLUSTER-} 2>&1 | \
+    tee ${IMPALA_CLUSTER_LOGS_DIR}/run-kudu.log
 
 # Starts up a mini-cluster which includes:
 # - HDFS with 3 DNs

testdata/bin/run-kudu.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
+
+if [[ $# -eq 1 && "$1" == -format ]]; then
+  SHOULD_FORMAT=true
+elif [[ $# -ne 0 ]]; then
+  echo "Usage: $0 [-format]"
+  echo "[-format] : Format the kudu cluster before starting"
+  exit 1
+else
+  SHOULD_FORMAT=false
+fi
+
+# Kill and clean data for a clean start.
+$IMPALA_HOME/testdata/bin/kill-kudu.sh
+
+if $SHOULD_FORMAT; then
+  $IMPALA_HOME/testdata/cluster/admin delete_kudu_data
+fi
+
+set +e
+$IMPALA_HOME/testdata/cluster/admin start_kudu_cluster

testdata/cluster/admin

@@ -64,7 +64,6 @@ else
   # the other services could work after the proper configuration changes.
   SUPPORTED_SERVICES=()
 fi
-SUPPORTED_SERVICES+=(kudu)
 
 # All DataNodes and NodeManagers need a unique but fixed address. The IP is fixed at
 # 127.0.0.1, so the only difference is the port. The address must be fixed because it is
@@ -186,6 +185,13 @@ function cluster_exists {
   fi
 }
 
+function kudu_cluster_exists {
+  # Just use the first master as an indicator.
+  if [[ ! -e "$IMPALA_CLUSTER_NODES_DIR/${NODE_PREFIX}1/var/lib/kudu/master" ]]; then
+    return 1
+  fi
+}
+
 function create_cluster {
   mkdir -p "$IMPALA_CLUSTER_NODES_DIR"
 
@@ -366,6 +372,16 @@ function start_cluster {
   return $?
 }
 
+function start_kudu_cluster {
+  if ! kudu_cluster_exists; then
+    echo "The Kudu cluster must be created first"
+    return 1
+  fi
+  start "kudu"
+  sleep 10
+  check_kudu_cluster_status
+}
+
 function start {
   if [[ $# -ne 1 ]]; then
     echo start must be called with a single argument -- the service to start. 1>&2
@@ -444,23 +460,42 @@ function check_cluster_status {
     return 1
   fi
 
+  for SERVICE in ${SUPPORTED_SERVICES[@]-}; do
+    check_status $SERVICE
+  done
+}
+
+function check_kudu_cluster_status {
+  if ! kudu_cluster_exists; then
+    echo "The Kudu cluster does not exist"
+    return 1
+  fi
+
+  check_status "kudu"
+}
+
+function check_status {
+  if [[ $# -ne 1 ]]; then
+    echo stop must be called with a single argument -- the service to check. 1>&2
+    exit 1
+  fi
+  local SERVICE=$1
+
   ROLE_COUNT=0
   NOT_RUNNING=()
   for NODE_DIR in "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*; do
-    for SERVICE in ${SUPPORTED_SERVICES[@]-}; do
-      for SCRIPT in $(find "$NODE_DIR" -path "*/etc/init.d/$SERVICE*" $FIND_EXECUTABLE_FILTER \
-          -type f); do
-        ROLE_COUNT=$((ROLE_COUNT + 1))
-        if ! "$SCRIPT" status &>/dev/null; then
-          NOT_RUNNING+=("\n$(basename $SCRIPT) is not running on $(basename $NODE_DIR)")
-        fi
-      done
+    for SCRIPT in $(find "$NODE_DIR" -path "*/etc/init.d/$SERVICE*" $FIND_EXECUTABLE_FILTER \
+        -type f); do
+      ROLE_COUNT=$((ROLE_COUNT + 1))
+      if ! "$SCRIPT" status &>/dev/null; then
+        NOT_RUNNING+=("\n$(basename $SCRIPT) is not running on $(basename $NODE_DIR)")
+      fi
     done
   done
 
   case ${#NOT_RUNNING[@]} in
-    0) echo "The cluster is running"; return;;
-    $ROLE_COUNT) echo "The cluster is not running"; return 1;;
+    0) echo "The ${SERVICE} cluster is running"; return;;
+    $ROLE_COUNT) echo "The ${SERVICE} cluster is not running"; return 1;;
     *) echo -e "${NOT_RUNNING[@]}"; return 1;;
   esac
 }
@@ -476,6 +511,12 @@ function stop_cluster {
   pkill -u $USER -f $KILL_CLUSTER_MARKER || true
 }
 
+function stop_kudu_cluster {
+  if kudu_cluster_exists; then
+    stop "kudu"
+  fi
+}
+
 function stop {
   if [[ $# -ne 1 ]]; then
     echo stop must be called with a single argument -- the service to stop. 1>&2
@@ -502,7 +543,6 @@ function delete_data {
   rm -rf "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/dfs/{nn,dn}/*
   rm -f "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/kms.keystore
   rm -rf "$IMPALA_CLUSTER_NODES_DIR/$NODE_PREFIX"*/data/ozone
-  delete_kudu_data
 }
 
 function delete_kudu_data {
@@ -510,6 +550,7 @@ function delete_kudu_data {
 }
 
 function delete_cluster {
+  # Delete all cluster data and directory structure.
   pkill -u $USER -f $KILL_CLUSTER_MARKER || true
   rm -rf "$IMPALA_CLUSTER_NODES_DIR"
 }
@@ -529,7 +570,8 @@ function get_hadoop_client_conf_dir {
 COMMAND=$1
 shift
 case $COMMAND in
-  check_cluster_status | cluster_exists | is_kerberized)
+  check_cluster_status | check_kudu_cluster_status | cluster_exists | \
+        kudu_cluster_exists | is_kerberized)
     # Use an "if" to avoid triggering the ERR trap.
     if ! $COMMAND "$@"; then
       exit 1