IMPALA-7995: part 2: Jenkins script to automate e2e tests

Testing:
Ran on https://jenkins.impala.io/job/ubuntu-16.04-dockerised-tests/

Change-Id: I67a3562904c959b51f4bde52107193c4002cb1ce
Reviewed-on: http://gerrit.cloudera.org:8080/12937
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

bin/jenkins/dockerized-impala-bootstrap-and-test.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script sets up docker on Ubuntu 16.04 and then run's
+# Impala's tests with a dockerised minicluster.
+
+set -eu -o pipefail
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
+cd "$ROOT_DIR"
+
+source ./bin/bootstrap_system.sh
+
+# Following install instructions from
+# https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-docker-ce-1
+sudo apt-get install -y apt-transport-https ca-certificates curl \
+     gnupg-agent software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+# Bail if the fingerprint isn't what we expected.
+sudo apt-key fingerprint 0EBFCD88 | \
+  grep '9DC8 5822 9FC7 DD38 854A  E2D8 8D81 803C 0EBF CD88'
+sudo add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+sudo service docker restart
+sudo groupadd -f docker
+sudo usermod -aG docker $USER
+
+# Execute the tests using su to re-login so that group change made above
+# setup_docker takes effect.
+sudo su $USER -c "./bin/jenkins/dockerized-impala-run-tests.sh"

bin/jenkins/dockerized-impala-run-tests.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# This script runs Impala's tests with a dockerised minicluster.
+# It has been tested on Ubuntu 16.04.
+set -x
+set -eu -o pipefail
+# Helper to source impala-config.sh, which may have unbound variables
+source_impala_config() {
+  set +u
+  . ./bin/impala-config.sh
+  set -u
+}
+source_impala_config
+
+# Check that docker is running and that our user can interact with it.
+docker run hello-world
+
+# Set up the test network.
+DOCKER_NETWORK="test-impala-cluster"
+./docker/configure_test_network.sh $DOCKER_NETWORK
+
+# Pick up the new variables.
+source_impala_config
+
+# Dump diagnostics for networks and check connectivity.
+ifconfig
+ping -c 1 $INTERNAL_LISTEN_HOST
+
+# Check that ssh to localhost via Docker gateway works.
+if ! ssh -n $INTERNAL_LISTEN_HOST "echo 'SSH success!'"; then
+  echo "Failed to ssh, will try to add docker network gateway to known hosts"
+  ssh-keyscan $INTERNAL_LISTEN_HOST >> ~/.ssh/known_hosts
+  ssh -n $INTERNAL_LISTEN_HOST "echo 'SSH success!'"
+fi
+
+# Build Impala and load data with the non-dockerised path.
+# TODO: this is a bit awkward. It would be better to have a single invocation
+# to run everything with docker.
+# Skip building backend tests, which aren't used.
+time -p ./buildall.sh -format -testdata -notests < /dev/null
+# Kill non-dockerised cluster.
+start-impala-cluster.py --kill
+
+# Build the docker images required to start the cluster.
+# parquet-reader is needed for e2e tests but not built for non-test build
+# Build serially to work around IMPALA-8392 TODO: do parallel build when fixed
+make docker_images parquet-reader
+
+source_impala_config
+
+export TEST_START_CLUSTER_ARGS="--docker_network=${DOCKER_NETWORK}"
+export MAX_PYTEST_FAILURES=0
+export NUM_CONCURRENT_TESTS=$(nproc)
+# Frontend tests fail because of localhost hardcoded everywhere
+export FE_TEST=false
+# No need to run backend tests - they are identical with non-docker build.
+export BE_TEST=false
+# TODO: custom cluster tests may provide some useful coverage but require work
+# to make them start up dockerised clusters and will probably make more assumptions
+# about the cluster being tested.
+export CLUSTER_TEST=false
+./bin/run-all-tests.sh

docker/configure_test_network.sh

@@ -51,3 +51,5 @@ echo "# Configuration to use docker network ${NETWORK_NAME}" \
 echo "export INTERNAL_LISTEN_HOST=${GATEWAY}" >> "$IMPALA_HOME"/bin/impala-config-local.sh
 echo "export DEFAULT_FS=hdfs://\${INTERNAL_LISTEN_HOST}:20500" \
       >> "$IMPALA_HOME"/bin/impala-config-local.sh
+echo "export KUDU_MASTER_HOSTS=\${INTERNAL_LISTEN_HOST}" \
+      >> "$IMPALA_HOME"/bin/impala-config-local.sh