mirror of
https://github.com/apache/impala.git
synced 2026-01-02 03:00:32 -05:00
75a97d3d7e
This is the first iteration of a kerberized development environment. All the daemons start and use kerberos, with the sole exception of the hive metastore. This is sufficient to test impala authentication. When buildall.sh is run using '-kerberize', it will stop before loading data or attempting to run tests. Loading data into the cluster is known to not work at this time, the root causes being that Beeline -> HiveServer2 -> MapReduce throws errors, and Beeline -> HiveServer2 -> HBase has problems. These are left for later work. However, the impala daemons will happily authenticate using kerberos both from clients (like the impala shell) and amongst each other. This means that if you can get data into the mini-cluster, you could query it. Usage: * Supply a '-kerberize' option to buildall.sh, or * Supply a '-kerberize' option to create-test-configuration.sh, then 'run-all.sh -format', re-source impala-config.sh, and then start impala daemons as usual. You must reformat the cluster because kerberizing it will change all the ownership of all files in HDFS. Notable changes: * Added clean start/stop script for the llama-minikdc * Creation of Kerberized HDFS - namenode and datanodes * Kerberized HBase (and Zookeeper) * Kerberized Hive (minus the MetaStore) * Kerberized Impala * Loading of data very nearly working Still to go: * Kerberize the MetaStore * Get data loading working * Run all tests * The unknown unknowns * Extensive testing Change-Id: Iee3f56f6cc28303821fc6a3bf3ca7f5933632160 Reviewed-on: http://gerrit.sjc.cloudera.com:8080/4019 Reviewed-by: Michael Yoder <myoder@cloudera.com> Tested-by: jenkins
50 lines
1.9 KiB
Bash
Executable File
50 lines
1.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Copyright 2012 Cloudera Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# The purpose of this script is to avoid CDH-17414 which causes data files loaded
|
|
# with Hive to incorrectly have a replication factor of 1. When using beeline
|
|
# this problem only appears to occur immediately after creating the first HBase table
|
|
# since starting HiveServer2, i.e., subsequent loads seem to function correctly.
|
|
# This script creates an external HBase table in Hive to 'warm up' HiveServer2.
|
|
# Subsequent loads should assign a correct replication factor.
|
|
|
|
set -e
|
|
|
|
cat > /tmp/create_hive_table.q << EOF
|
|
DROP TABLE if exists hive_replication_bug_warmup_table;
|
|
create table hive_replication_bug_warmup_table(x int, y string)
|
|
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
|
|
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf1:c1")
|
|
TBLPROPERTIES ("hbase.table.name" = "hive_replication_bug_warmup_table");
|
|
EOF
|
|
|
|
cat > /tmp/drop_hive_tables.q << EOF
|
|
DROP TABLE if exists hive_replication_bug_warmup_table;
|
|
EOF
|
|
|
|
JDBC_URL="jdbc:hive2://localhost:11050/default;"
|
|
if ${CLUSTER_DIR}/admin is_kerberized; then
|
|
# Making a kerberized cluster... set some more environment variables.
|
|
. ${MINIKDC_ENV}
|
|
JDBC_URL="${JDBC_URL}principal=${MINIKDC_PRINC_HIVE}"
|
|
fi
|
|
|
|
beeline -n $USER -u "${JDBC_URL}" -f /tmp/create_hive_table.q
|
|
beeline -n $USER -u "${JDBC_URL}" -f /tmp/drop_hive_tables.q
|
|
|
|
# Clean up temp files.
|
|
rm /tmp/create_hive_table.q
|
|
rm /tmp/drop_hive_tables.q
|