IMPALA-9448: Use Ozone TDE in minicluster

Enables Ozone's Transparent Data Encryption when creating a bucket in
the minicluster. Requires `-format` to turn on encryption in an existing
minicluster.

Tested by running e2e and custom cluster tests with Ozone.

Change-Id: Icd82967cdab7b222e5876936f39e5198ba9773ea
Reviewed-on: http://gerrit.cloudera.org:8080/18931
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

bin/impala-config.sh

@@ -657,6 +657,7 @@ elif [ "${TARGET_FILESYSTEM}" = "hdfs" ]; then
     export HDFS_ERASURECODE_PATH="/test-warehouse"
   fi
 elif [ "${TARGET_FILESYSTEM}" = "ozone" ]; then
+  export USE_OZONE_ENCRYPTION=${USE_OZONE_ENCRYPTION-true}
   export OZONE_VOLUME="impala"
   export OZONE_BUCKET="base"
   export DEFAULT_FS="o3fs://${OZONE_BUCKET}.${OZONE_VOLUME}.${INTERNAL_LISTEN_HOST}:9862"

testdata/bin/run-all.sh

@@ -81,7 +81,6 @@ else
   #   - We don't yet have a good way to start YARN using a different defaultFS. Moreoever
   #     we currently don't run hive queries against Isilon for testing.
   #   - LLAMA is avoided because we cannot start YARN.
-  #   - KMS is used for encryption testing, which is not available on remote storage.
   #   - Hive needs YARN, and we don't run Hive queries.
   # Impala can also run on a local file system without additional services.
   # TODO: Figure out how to start YARN, LLAMA and Hive with a different defaultFs.

testdata/bin/setup-dfs-keys.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -euo pipefail
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
+
+# Create encryption keys for HDFS encryption tests. Keys are stored by the KMS.
+EXISTING_KEYS=$(hadoop key list)
+for KEY in $*; do
+  if ! grep $KEY <<< $EXISTING_KEYS &>/dev/null; then
+    hadoop key create $KEY -f
+  fi
+done

testdata/bin/setup-hdfs-env.sh

@@ -46,15 +46,8 @@ fi
 # against a remote cluster, rather than the local mini-cluster (i.e., when REMOTE_LOAD
 # is true. See: IMPALA-4344)
 
-if [[ $TARGET_FILESYSTEM == hdfs && -z "$REMOTE_LOAD" ]]; then  # Otherwise assume KMS isn't setup.
-  # Create encryption keys for HDFS encryption tests. Keys are stored by the KMS.
-  EXISTING_KEYS=$(hadoop key list)
-  for KEY in testkey{1,2}; do
-    if grep $KEY <<< $EXISTING_KEYS &>/dev/null; then
-      hadoop key delete $KEY -f
-    fi
-    hadoop key create $KEY
-  done
+if [[ -z "$REMOTE_LOAD" ]]; then  # Otherwise assume KMS isn't setup.
+  ${IMPALA_HOME}/testdata/bin/setup-dfs-keys.sh testkey{1,2}
 fi
 
 if [[ -n "${REMOTE_LOAD:-}" ]]; then

testdata/cluster/admin

@@ -57,7 +57,7 @@ if [[ "$TARGET_FILESYSTEM" == "hdfs" ]]; then
   # The check above indicates that the regular mini-cluster is in use.
   SUPPORTED_SERVICES=(hdfs kms yarn)
 elif [[ "$TARGET_FILESYSTEM" == "ozone" ]]; then
-  SUPPORTED_SERVICES=(ozone)
+  SUPPORTED_SERVICES=(kms ozone)
 else
   # Either a remote distributed file system or a local non-distributed file system is
   # in use. Currently the only service that is expected to work is Kudu, though in theory
@@ -327,8 +327,17 @@ function start_cluster {
   fi
 
   if [[ "${TARGET_FILESYSTEM}" = "ozone" ]]; then
-    ozone sh volume create /${OZONE_VOLUME} || true
-    ozone sh bucket create /${OZONE_VOLUME}/${OZONE_BUCKET} || true
+    local bucketkey=''
+    if $USE_OZONE_ENCRYPTION; then
+      echo "Ozone encryption enabled for ${OZONE_VOLUME}/${OZONE_BUCKET}"
+
+      # Encryption is done at the bucket level, so ensure the keys are available first.
+      ${IMPALA_HOME}/testdata/bin/setup-dfs-keys.sh testkey
+      bucketkey='--bucketkey testkey'
+    fi
+
+    ozone sh volume create ${OZONE_VOLUME} || true
+    ozone sh bucket create ${bucketkey} ${OZONE_VOLUME}/${OZONE_BUCKET} || true
   fi
 
   return $?