IMPALA-2847: only recreate Sentry Policy DB when formatting cluster

We should only need to recreate the Sentry Policy DB when formatting a
cluster. Previously buildall.sh always tried to create the database
regardless of whether it was needed. E.g. if a machine was just building
Impala without running tests, there is no need to create any of the test
databases. This fixes a regression when running buildall.sh on a machine
without postgres set up.

Change-Id: I35bb1cb275bb4da3f91f496010a7f6ee4daa2792
Reviewed-on: http://gerrit.cloudera.org:8080/1782
Reviewed-by: Casey Ching <casey@cloudera.com>
Tested-by: Internal Jenkins

bin/create-test-configuration.sh

@@ -21,6 +21,7 @@ set -euo pipefail
 trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
 
 CREATE_METASTORE=0
+CREATE_SENTRY_POLICY_DB=0
 : ${IMPALA_KERBERIZE=}
 
 # parse command line options
@@ -30,12 +31,16 @@ do
     -create_metastore)
       CREATE_METASTORE=1
       ;;
+    -create_sentry_policy_db)
+      CREATE_SENTRY_POLICY_DB=1
+      ;;
     -k|-kerberize|-kerberos|-kerb)
       # This could also come in through the environment...
       export IMPALA_KERBERIZE=1
       ;;
     -help|*)
       echo "[-create_metastore] : If true, creates a new metastore."
+      echo "[-create_sentry_policy_db] : If true, creates a new sentry policy db."
       echo "[-kerberize] : Enable kerberos on the cluster"
       exit 1
       ;;
@@ -97,9 +102,11 @@ if [ $CREATE_METASTORE -eq 1 ]; then
       | psql -U hiveuser -d hive_$METASTORE_DB
 fi
 
-echo "Creating Sentry Policy Server DB"
-dropdb -U hiveuser sentry_policy 2> /dev/null || true
-createdb -U hiveuser sentry_policy
+if [ $CREATE_SENTRY_POLICY_DB -eq 1 ]; then
+  echo "Creating Sentry Policy Server DB"
+  dropdb -U hiveuser sentry_policy 2> /dev/null || true
+  createdb -U hiveuser sentry_policy
+fi
 
 # Perform search-replace on $1, output to $2.
 # Search $1 ($GCIN) for strings that look like "${FOO}".  If FOO is defined in

buildall.sh

@@ -41,6 +41,7 @@ TESTDATA_ACTION=0
 TESTS_ACTION=1
 FORMAT_CLUSTER=0
 FORMAT_METASTORE=0
+FORMAT_SENTRY_POLICY_DB=0
 IMPALA_KERBERIZE=0
 SNAPSHOT_FILE=
 METASTORE_SNAPSHOT_FILE=
@@ -77,6 +78,7 @@ do
     -format)
       FORMAT_CLUSTER=1
       FORMAT_METASTORE=1
+      FORMAT_SENTRY_POLICY_DB=1
       ;;
     -format_cluster)
       FORMAT_CLUSTER=1
@@ -84,6 +86,9 @@ do
     -format_metastore)
       FORMAT_METASTORE=1
       ;;
+    -format_sentry_policy_db)
+      FORMAT_SENTRY_POLICY_DB=1
+      ;;
     -codecoverage_debug)
       TARGET_BUILD_TYPE=CODE_COVERAGE_DEBUG
       ;;
@@ -139,9 +144,11 @@ do
       echo "buildall.sh - Builds Impala and runs all tests."
       echo "[-noclean] : Omits cleaning all packages before building. Will not kill"\
            "running Hadoop services unless any -format* is True"
-      echo "[-format] : Format the minicluster and metastore db [Default: False]"
+      echo "[-format] : Format the minicluster, metastore db, and sentry policy db"\
+           " [Default: False]"
       echo "[-format_cluster] : Format the minicluster [Default: False]"
       echo "[-format_metastore] : Format the metastore db [Default: False]"
+      echo "[-format_sentry_policy_db] : Format the Sentry policy db [Default: False]"
       echo "[-codecoverage_release] : Release code coverage build"
       echo "[-codecoverage_debug] : Debug code coverage build"
       echo "[-asan] : Build with address sanitizer"
@@ -224,7 +231,7 @@ fi
 ${IMPALA_HOME}/bin/start-impala-cluster.py --kill --force
 
 if [[ $CLEAN_ACTION -eq 1 || $FORMAT_METASTORE -eq 1 || $FORMAT_CLUSTER -eq 1 ||
-      -n $METASTORE_SNAPSHOT_FILE ]]
+       $FORMAT_SENTRY_POLICY_DB -eq 1 || -n $METASTORE_SNAPSHOT_FILE ]]
 then
   # Kill any processes that may be accessing postgres metastore. To be safe, this is done
   # before we make any changes to the config files.
@@ -238,13 +245,18 @@ if [ $CLEAN_ACTION -eq 1 ]; then
     $IMPALA_HOME/bin/clean.sh
 fi
 
-# Generate the Hadoop configs needed by Impala
-if [[ $FORMAT_METASTORE -eq 1 && -z $METASTORE_SNAPSHOT_FILE ]]; then
-  ${IMPALA_HOME}/bin/create-test-configuration.sh -create_metastore
-else
-  ${IMPALA_HOME}/bin/create-test-configuration.sh
+CREATE_TEST_CONFIG_ARGS=""
+if [[ $FORMAT_SENTRY_POLICY_DB -eq 1 ]]; then
+  CREATE_TEST_CONFIG_ARGS+=" -create_sentry_policy_db"
 fi
 
+if [[ $FORMAT_METASTORE -eq 1 && -z $METASTORE_SNAPSHOT_FILE ]]; then
+  CREATE_TEST_CONFIG_ARGS+=" -create_metastore"
+fi
+
+# Generate the Hadoop configs needed by Impala
+${IMPALA_HOME}/bin/create-test-configuration.sh ${CREATE_TEST_CONFIG_ARGS}
+
 # If a metastore snapshot exists, load it.
 if [ $METASTORE_SNAPSHOT_FILE ]; then
   echo "Loading metastore snapshot"