mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
c3dc7f9667
Configure separate compile and link pools for ninja. Configures link parallelism based on expected memory use, which can be reduced by setting IMPALA_MINIMAL_DEBUG_INFO=true or IMPALA_SPLIT_DEBUG_INFO=true. Adds IMPALA_MAKE_CMD to simplify using the ninja build tool for all make operations in scripts. Install ninja on Ubuntu. Adds a '-make' option to buildall.sh to force using 'make'. Adds MOLD_JOBS=1 to avoid overloading the system when trying 'mold' and linking test binaries. However 'mold' is not selected as the default due to test failures around SASL/GSSAPI (see IMPALA-14527). Switches bin/jenkins/all-tests.sh to use ninja and removes the guard in bootstrap_development.sh limiting IMPALA_BUILD_THREADS as it's no longer needed with ninja. SKIP_BE_TEST_PATTERN in run-backend-tests is unused (only used with TARGET_FILESYSTEM=local) so I don't attempt to make it work with ninja. Tested with local 'IMPALA_SPLIT_DEBUG_INFO=true buildall.sh -skiptests' with default (make) and IMPALA_MAKE_CMD=ninja. Change-Id: I0952dc19ace5c9c42bed0d2ffb61499656c0a2db Reviewed-on: http://gerrit.cloudera.org:8080/23572 Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com> Reviewed-by: Pranav Lodha <pranav.lodha@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
101 lines
2.8 KiB
Bash
101 lines
2.8 KiB
Bash
#!/bin/bash
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Run all Impala tests.
|
|
|
|
set -euo pipefail
|
|
. $IMPALA_HOME/bin/report_build_error.sh
|
|
setup_report_build_error
|
|
|
|
# Start time of run.
|
|
START_TIME=$(date +"%Y-%m-%d %H:%M:%S")
|
|
|
|
cd "${IMPALA_HOME}"
|
|
|
|
export IMPALA_MAVEN_OPTIONS="-U"
|
|
# Allow unlimited pytest failures
|
|
export MAX_PYTEST_FAILURES=0
|
|
|
|
# Use ninja for better link concurrency.
|
|
export IMPALA_MAKE_CMD=ninja
|
|
|
|
# When UBSAN_FAIL is "death", the logs are monitored for UBSAN errors. Any errors will
|
|
# then cause this script to exit.
|
|
#
|
|
# When UBSAN_FAIL is "error", monitoring is delayed until tests have finished running.
|
|
#
|
|
# Any other value ignores UBSAN errors.
|
|
: ${UBSAN_FAIL:=error}
|
|
export UBSAN_FAIL
|
|
|
|
if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \
|
|
&& [ "${UBSAN_FAIL}" = "death" ]
|
|
then
|
|
export PID_TO_KILL="$(echo $$)"
|
|
mkdir -p "${IMPALA_HOME}/logs"
|
|
|
|
function killer {
|
|
while ! grep -rI ": runtime error: " "${IMPALA_HOME}/logs"
|
|
do
|
|
sleep 1
|
|
if ! test -e "/proc/$PID_TO_KILL"
|
|
then
|
|
return
|
|
fi
|
|
done
|
|
>&2 echo "Killing process $PID_TO_KILL because it invoked undefined behavior"
|
|
kill -9 $PID_TO_KILL
|
|
}
|
|
|
|
killer &
|
|
export KILLER_PID="$(echo $!)"
|
|
disown
|
|
trap "kill -i $KILLER_PID" EXIT
|
|
fi
|
|
|
|
RET_CODE=0
|
|
if ! bin/bootstrap_development.sh; then
|
|
RET_CODE=1
|
|
fi
|
|
|
|
source bin/impala-config.sh > /dev/null 2>&1
|
|
|
|
# Sanity check: bootstrap_development.sh should not have modified any of
|
|
# the Impala files. This is important for the continued functioning of
|
|
# bin/single_node_perf_run.py.
|
|
NUM_MODIFIED_FILES=$(git status --porcelain --untracked-files=no | wc -l)
|
|
if [[ "${NUM_MODIFIED_FILES}" -ne 0 ]]; then
|
|
echo "ERROR: Impala source files were modified during bin/bootstrap_development.sh"
|
|
echo "Dumping diff:"
|
|
git status --porcelain --untracked-files=no
|
|
git --no-pager diff
|
|
RET_CODE=1
|
|
fi
|
|
|
|
# Skip tests if build/dataload failed
|
|
if [[ $RET_CODE -eq 0 ]]; then
|
|
if ! bin/run-all-tests.sh; then
|
|
RET_CODE=1
|
|
fi
|
|
fi
|
|
|
|
# Always shutdown minicluster at the end and run finalize.sh
|
|
testdata/bin/kill-all.sh
|
|
bin/jenkins/finalize.sh "${START_TIME}"
|
|
exit $RET_CODE
|