IMPALA-10262: RPM/DEB Packaging Support

This patch bases on a previous patch contributed by Shant Hovsepian:
https://gerrit.cloudera.org/c/16612/

It adds a new option, -package, to buildall.sh for building a package
for the current OS type (e.g. CentOS/Ubuntu). You can also use
"make/ninja package" to build the package. Scripts for launching the
services and the required configuration files are also added.

Tests:
 - Built on Ubuntu 18.04/20.04 and CentOS 7 using
   ./buildall.sh -noclean -skiptests -release -package
 - Deployed the RPM package on a CDP cluster. Verifed the scripts.
 - Deployed the DEB package on a docker container. Verified the scripts.

Change-Id: I64419fd400fe8d233dac016b6306157fe9461d82
Reviewed-on: http://gerrit.cloudera.org:8080/18939
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

package/bin/impala-env.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [[ -z "$JAVA_HOME" ]]; then
+  echo "JAVA_HOME not set!"
+  exit 1
+fi
+
+echo "Using JAVA_HOME: $JAVA_HOME"
+LIB_JVM_DIR=$(dirname $(find $JAVA_HOME -type f -name libjvm.so))
+LIB_JSIG_DIR=$(dirname $(find $JAVA_HOME -type f -name libjsig.so))
+
+export LC_ALL=en_US.utf8
+export LD_LIBRARY_PATH="/opt/impala/lib/:$LIB_JVM_DIR:$LIB_JSIG_DIR"
+export CLASSPATH="/opt/impala/conf:/opt/impala/jar/*"
+
+#TODO: Add graceful shutdown for impalads
+function stop_process {
+  name=$1
+  pid_file="/tmp/${name}.pid"
+  if [[ -f $pid_file ]]; then
+    PID=$(cat $pid_file)
+    if ps $PID | grep $name; then
+      echo "Killing $name with PID=$PID"
+      kill $PID
+      rm $pid_file
+      echo "Killed $name"
+    else
+      rm $pid_file
+      echo "Already stopped: $name is not running with PID=$PID. Removed stale $pid_file"
+    fi
+  else
+    echo "PID file $pid_file not found!"
+  fi
+}
+
+function wait_for_ready {
+  name=$1
+  port=$2
+  pid=$3
+
+  NUM_WAIT_ITERATIONS=20
+  i=0
+  while [[ $i -lt $NUM_WAIT_ITERATIONS ]]; do
+    if ! ps $pid | grep $name > /dev/null 2>&1; then
+      echo "$name with PID $pid doesn't exist"
+      break
+    fi
+    STATUS=$(curl -s http://localhost:$port/healthz)
+    if [[ $? != 0 ]]; then
+      echo "Waiting for $name. Port $port not ready."
+    elif [[ "$STATUS" != "OK" ]]; then
+      echo "Waiting for $name to be ready"
+    else
+      echo "$name is ready"
+      break
+    fi
+    sleep 2
+    i=$((i+1))
+  done
+  if [[ "$STATUS" == "OK" ]]; then
+    echo "Launched $name with PID $pid"
+  elif [[ $i -eq $NUM_WAIT_ITERATIONS ]]; then
+    echo "Timed out waiting for $name to be ready. Check logs for more details."
+  else
+    echo "Failed to launch $name"
+    exit 1
+  fi
+}

package/bin/start-catalogd.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch catalogd. Required JAVA_HOME being set.
+# Edit conf/catalogd_flags to set the correct hostname and state_store_host.
+# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster.
+# Example usage:
+#   export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+#   bin/start-catalogd.sh
+# To launch catalogd using another username (e.g. "impala"):
+#   sudo -E -u impala bin/start-catalogd.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+$IMPALA_HOME/bin/catalogd --flagfile=$IMPALA_HOME/conf/catalogd_flags &
+PID=$!
+echo $PID > /tmp/catalogd.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready catalogd 25020 $PID

package/bin/start-impalad.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch impalad. Required JAVA_HOME being set.
+# Edit conf/impalad_flags to set the correct hostnames.
+# Edit core-site.xml, hdfs-site.xml, hive-site.xml, etc. in conf based on the cluster.
+# Example usage:
+#   export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+#   bin/start-impalad.sh
+# To launch impalad using another username (e.g. "impala"):
+#   sudo -E -u impala bin/start-impalad.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+if [[ -n "$HADOOP_HOME" ]]; then
+  echo "Using HADOOP_HOME: $HADOOP_HOME"
+  export HADOOP_LIB_DIR="${HADOOP_HOME}/lib"
+  export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${HADOOP_LIB_DIR}/native/"
+  echo "Using hadoop native libs in ${HADOOP_LIB_DIR}/native/"
+else
+  export LIBHDFS_OPTS="${LIBHDFS_OPTS:-} -Djava.library.path=${IMPALA_HOME}/lib"
+  echo "HADOOP_HOME not set. Using hadoop native libs in ${IMPALA_HOME}/lib"
+fi
+
+$IMPALA_HOME/bin/impalad --flagfile=$IMPALA_HOME/conf/impalad_flags &
+PID=$!
+echo $PID > /tmp/impalad.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready impalad 25000 $PID

package/bin/start-statestored.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Script to launch statestore. Required JAVA_HOME being set.
+# Edit conf/statestore_flags to set a correct hostname.
+# Example usage:
+#   export JAVA_HOME=/usr/java/jdk1.8.0_232-cloudera
+#   bin/start-statestored.sh
+# To launch statestore using another username (e.g. "impala"):
+#   sudo -E -u impala bin/start-statestored.sh
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+$IMPALA_HOME/bin/statestored --flagfile=$IMPALA_HOME/conf/statestore_flags &
+PID=$!
+echo $PID > /tmp/statestored.pid
+
+# Sleep 1s so the glog output won't be messed up with waiting messages
+sleep 1
+
+wait_for_ready statestored 25010 $PID

package/bin/stop-catalogd.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process catalogd

package/bin/stop-impalad.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process impalad

package/bin/stop-statestored.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Using IMPALA_HOME: ${IMPALA_HOME:=/opt/impala}"
+source $IMPALA_HOME/bin/impala-env.sh
+
+stop_process statestored

package/conf/catalogd_flags

@@ -0,0 +1,13 @@
+-hostname=localhost
+-state_store_host=localhost
+#-kudu_master_hosts=localhost
+
+-log_dir=/var/log/catalogd
+-log_filename=catalogd
+-state_store_port=24000
+-minidump_path=/var/log/impala-minidumps
+-webserver_doc_root=/opt/impala
+-catalog_topic_mode=minimal
+-hms_event_polling_interval_s=0
+-v=1
+-max_log_size=200

package/conf/core-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://localhost:8020</value>
+  </property>
+  <property>
+    <name>ipc.client.connection.maxidletime</name>
+    <value>30000</value>
+  </property>
+  <property>
+    <name>ipc.client.connect.max.retries</name>
+    <value>50</value>
+  </property>
+  <property>
+    <name>fs.trash.interval</name>
+    <value>1</value>
+  </property>
+</configuration>

package/conf/fair-scheduler.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<allocations>
+    <queue name="root">
+        <queue name="default"/>
+    </queue>
+</allocations>

package/conf/hdfs-site.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+  <property>
+    <name>dfs.namenode.servicerpc-address</name>
+    <value>localhost:8022</value>
+  </property>
+  <property>
+    <name>dfs.namenode.http-address</name>
+    <value>localhost:20101</value>
+  </property>
+  <property>
+    <name>dfs.replication</name>
+    <value>3</value>
+  </property>
+  <property>
+    <name>dfs.blocksize</name>
+    <value>134217728</value>
+  </property>
+  <property>
+    <name>dfs.domain.socket.path</name>
+    <value>/var/run/hdfs-sockets/dn</value>
+  </property>
+  <property>
+    <name>dfs.client.read.shortcircuit</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.client.read.shortcircuit.streams.cache.size</name>
+    <value>4096</value>
+  </property>
+  <property>
+    <name>dfs.client.read.shortcircuit.skip.checksum</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>dfs.client.file-block-storage-locations.timeout.millis</name>
+    <value>10000</value>
+  </property>
+</configuration>

package/conf/hive-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration>
+  <property>
+    <name>hive.metastore.uris</name>
+    <value>thrift://localhost:9083</value>
+  </property>
+  <property>
+    <name>hive.support.concurrency</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hive.metastore.client.socket.timeout</name>
+    <value>3600</value>
+  </property>
+  <property>
+    <name>hive.metastore.connect.retries</name>
+    <value>5</value>
+  </property>
+</configuration>

package/conf/impalad_flags

@@ -0,0 +1,16 @@
+-hostname=localhost
+-state_store_host=localhost
+-catalog_service_host=localhost
+#-kudu_master_hosts=localhost
+
+-mem_limit=80%
+-use_local_catalog=true
+-log_dir=/var/log/impalad
+-log_filename=impalad
+-minidump_path=/var/log/impala-minidumps
+-local_library_dir=/var/lib/impala/udfs
+-fair_scheduler_allocation_path=/opt/impala/conf/fair-scheduler.xml
+-llama_site_path=/opt/impala/conf/llama-site.xml
+-webserver_doc_root=/opt/impala
+-v=1
+-max_log_size=200

package/conf/llama-site.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration/>

package/conf/statestore_flags

@@ -0,0 +1,7 @@
+-hostname=localhost
+-log_dir=/var/log/statestore
+-log_filename=statestored
+-minidump_path=/var/log/impala-minidumps
+-webserver_doc_root=/opt/impala
+-v=1
+-max_log_size=200