IMPALA-11961/IMPALA-12207: Add Redhat 9 / Ubuntu 22 support

This adds support for Redhat 9 / Ubuntu 22. It updates
to a newer toolchain that has those builds, and it adds
supporting code in bootstrap_system.sh.

Redhat 9 and Ubuntu 22 use python = python3, which requires
various changes to build scripts and tests. Ubuntu 22 uses
Python 3.10, which deprecates certain ssl.PROTOCOL_TLS, so
this adapts test_client_ssl.py to that change until it
can be fully addressed in IMPALA-12219.

Various OpenSSL methods have been deprecated. As a workaround
until these can be addressed properly, this specifies
-Wno-deprecated-declarations. This can be removed once the
code is adapted to the non-deprecated APIs in IMPALA-12226.

Impala crashes with tcmalloc errors unless we update to a newer
gperftools, so this moves to gperftools 2.10. gperftools changed
the default for tcmalloc.aggressive_memory_decommit to off, so
this adapts our code to set it for backend tests. The gperftools
upgrade does not show any performance regression:

+----------+-----------------------+---------+------------+------------+----------------+
| Workload | File Format           | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) |
+----------+-----------------------+---------+------------+------------+----------------+
| TPCH(42) | parquet / none / none | 3.08    | -0.64%     | 2.20       | -0.37%         |
+----------+-----------------------+---------+------------+------------+----------------+

With newer Python versions, the impala-virtualenv command
fails to create a Python 3 virtualenv. This switches to
using Python 3's builtin venv command for Python >=3.6.

Kudu needed a newer version and LLVM required a couple patches.

Testing:
 - Ran a core job on Ubuntu 22 and Redhat 9. The tests run
   to completion without crashing. There are test failures
   that will be addressed in follow-up JIRAs.
 - Ran dockerised tests on Ubuntu 22.
 - Ran dockerised tests on Ubuntu 20 and Rocky 8.5.

Change-Id: If1fcdb2f8c635ecd6dc7a8a1db81f5f389c78b86
Reviewed-on: http://gerrit.cloudera.org:8080/20073
Reviewed-by: Michael Smith <michael.smith@cloudera.com>
Tested-by: Joe McDonnell <joemcdonnell@cloudera.com>

be/CMakeLists.txt

@@ -35,6 +35,9 @@ PROJECT(ASSEMBLER)
 #  -fsigned-char: on aarch64 platform, type of char default is unsigned char, here
 #        set it to signed-char to be compatible with x86-64
 #  -Wno-deprecated: gutil contains deprecated headers
+#  -Wno-deprecated-declarations: OpenSSL3 deprecated various APIs currently used by
+#       Impala, so this disables those warnings until they can be addressed.
+#       See IMPALA-12226.
 #  -Wno-vla: we use C99-style variable-length arrays
 #  -pthread: enable multithreaded malloc
 #  -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG: enable nanosecond precision for boost
@@ -46,7 +49,7 @@ SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-p
 SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-strict-aliasing -fno-omit-frame-pointer")
 SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fsigned-char")
 SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=c++17")
-SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla")
+SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-deprecated-declarations -Wno-vla")
 SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG")
 SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED")
 SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_BIND_GLOBAL_PLACEHOLDERS")

be/src/runtime/exec-env.cc

@@ -568,8 +568,7 @@ void ExecEnv::SetImpalaServer(ImpalaServer* server) {
   }
 }
 
-void ExecEnv::InitBufferPool(int64_t min_buffer_size, int64_t capacity,
+void ExecEnv::InitTcMallocAggressiveDecommit() {
-    int64_t clean_pages_limit) {
 #if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
   // Aggressive decommit is required so that unused pages in the TCMalloc page heap are
   // not backed by physical pages and do not contribute towards memory consumption.
@@ -577,6 +576,11 @@ void ExecEnv::InitBufferPool(int64_t min_buffer_size, int64_t capacity,
   MallocExtension::instance()->SetNumericProperty(
       "tcmalloc.aggressive_memory_decommit", 1);
 #endif
+}
+
+void ExecEnv::InitBufferPool(int64_t min_buffer_size, int64_t capacity,
+    int64_t clean_pages_limit) {
+  InitTcMallocAggressiveDecommit();
   buffer_pool_.reset(
       new BufferPool(metrics_.get(), min_buffer_size, capacity, clean_pages_limit));
   buffer_reservation_.reset(new ReservationTracker());

be/src/runtime/exec-env.h

@@ -327,6 +327,13 @@ class ExecEnv {
   /// Initialize ExecEnv based on Hadoop config from frontend.
   Status InitHadoopConfig();
 
+  /// Set tcmalloc's aggressive_memory_decommit=1. This needs to be called before
+  /// initializing the buffer pool, because the buffer pool asserts that this
+  /// property is set and newer versions of tcmalloc do not set it by default.
+  /// InitBufferPool() calls this automatically, so this is only used directly by
+  /// TestEnv.
+  void InitTcMallocAggressiveDecommit();
+
   /// Initialise 'buffer_pool_' and 'buffer_reservation_' with given capacity.
   void InitBufferPool(int64_t min_page_len, int64_t capacity, int64_t clean_pages_limit);
 

be/src/runtime/test-env.cc

@@ -73,6 +73,11 @@ Status TestEnv::Init() {
   if (enable_buffer_pool_) {
     exec_env_->InitBufferPool(buffer_pool_min_buffer_len_, buffer_pool_capacity_,
         static_cast<int64_t>(0.1 * buffer_pool_capacity_));
+  } else {
+    // The buffer pool requires tcmalloc's aggressive_memory_decommit to be set.
+    // Since this codepath will never call InitBufferPool(), this needs to manually
+    // set it.
+    exec_env_->InitTcMallocAggressiveDecommit();
   }
   if (process_mem_tracker_use_metrics_) {
     exec_env_->InitMemTracker(process_mem_limit_);

bin/bootstrap_build.sh

@@ -34,8 +34,8 @@ set -euxo pipefail
 export DEBIAN_FRONTEND=noninteractive
 sudo -E apt-get update
 sudo -E apt-get --yes install g++ gcc git libsasl2-dev libssl-dev make python-dev \
-     python-setuptools python3-dev python3-setuptools libffi-dev libkrb5-dev \
+     python-setuptools python3-dev python3-setuptools python3-venv libffi-dev \
-     krb5-admin-server krb5-kdc krb5-user libxml2-dev libxslt-dev
+     libkrb5-dev krb5-admin-server krb5-kdc krb5-user libxml2-dev libxslt-dev
 
 source /etc/lsb-release
 

bin/bootstrap_system.sh

@@ -70,17 +70,22 @@ set -x
 
 # Determine whether we're running on redhat or ubuntu
 REDHAT=
-REDHAT6=
 REDHAT7=
 REDHAT8=
+REDHAT9=
 UBUNTU=
 UBUNTU16=
 UBUNTU18=
 UBUNTU20=
+UBUNTU22=
 IN_DOCKER=
 if [[ -f /etc/redhat-release ]]; then
   REDHAT=true
   echo "Identified redhat system."
+  if grep 'release 9\.' /etc/redhat-release; then
+    REDHAT9=true
+    echo "Identified redhat9 system."
+  fi
   if grep 'release 8\.' /etc/redhat-release; then
     REDHAT8=true
     echo "Identified redhat8 system."
@@ -89,10 +94,6 @@ if [[ -f /etc/redhat-release ]]; then
     REDHAT7=true
     echo "Identified redhat7 system."
   fi
-  if grep 'release 6\.' /etc/redhat-release; then
-    REDHAT6=true
-    echo "Identified redhat6 system."
-  fi
   # TODO: restrict redhat versions
 else
   source /etc/lsb-release
@@ -114,8 +115,12 @@ else
     then
       UBUNTU20=true
       echo "Identified Ubuntu 20.04 system."
+    elif [[ $DISTRIB_RELEASE = 22.04 ]]
+    then
+      UBUNTU22=true
+      echo "Identified Ubuntu 22.04 system."
     else
-      echo "This script supports Ubuntu versions 16.04, 18.04 or 20.04" >&2
+      echo "This script supports Ubuntu versions 16.04, 18.04, 20.04, or 22.04" >&2
       exit 1
     fi
   else
@@ -155,6 +160,12 @@ function ubuntu20 {
   fi
 }
 
+function ubuntu22 {
+  if [[ "$UBUNTU22" == true ]]; then
+    "$@"
+  fi
+}
+
 # Helper function to execute following command only on RedHat
 function redhat {
   if [[ "$REDHAT" == true ]]; then
@@ -162,12 +173,6 @@ function redhat {
   fi
 }
 
-# Helper function to execute following command only on RedHat6
-function redhat6 {
-  if [[ "$REDHAT6" == true ]]; then
-    "$@"
-  fi
-}
 # Helper function to execute following command only on RedHat7
 function redhat7 {
   if [[ "$REDHAT7" == true ]]; then
@@ -180,6 +185,12 @@ function redhat8 {
     "$@"
   fi
 }
+# Helper function to execute following command only on RedHat8
+function redhat9 {
+  if [[ "$REDHAT9" == true ]]; then
+    "$@"
+  fi
+}
 # Helper function to execute following command only in docker
 function indocker {
   if [[ "$IN_DOCKER" == true ]]; then
@@ -219,16 +230,22 @@ fi
 source "$IMPALA_HOME/bin/impala-config-java.sh"
 
 ubuntu apt-get update
-ubuntu apt-get --yes install ccache curl gawk g++ gcc apt-utils git libffi-dev \
+ubuntu apt-get --yes install ccache curl file gawk g++ gcc apt-utils git libffi-dev \
         libkrb5-dev krb5-admin-server krb5-kdc krb5-user libsasl2-dev \
         libsasl2-modules libsasl2-modules-gssapi-mit libssl-dev make ninja-build \
-        python-dev python-setuptools python3-dev python3-setuptools postgresql \
+        python3-dev python3-setuptools python3-venv postgresql \
         ssh wget vim-common psmisc lsof net-tools language-pack-en libxml2-dev \
         libxslt-dev openjdk-${UBUNTU_JAVA_VERSION}-jdk \
         openjdk-${UBUNTU_JAVA_VERSION}-source openjdk-${UBUNTU_JAVA_VERSION}-dbg
 
+# Regular python packages don't exist on Ubuntu 22. Everything is Python 3.
+ubuntu16 apt-get --yes install python python-dev python-setuptools
+ubuntu18 apt-get --yes install python python-dev python-setuptools
+ubuntu20 apt-get --yes install python python-dev python-setuptools
+
 # Required by Kudu in the minicluster
 ubuntu20 apt-get --yes install libtinfo5
+ubuntu22 apt-get --yes install libtinfo5
 ARCH_NAME=$(uname -p)
 if [[ $ARCH_NAME == 'aarch64' ]]; then
   ubuntu apt-get --yes install unzip pkg-config flex maven python3-pip build-essential \
@@ -240,18 +257,24 @@ fi
 ubuntu sudo update-java-alternatives -s \
     java-1.${UBUNTU_JAVA_VERSION}.0-openjdk-${UBUNTU_PACKAGE_ARCH}
 
-redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \
+redhat sudo yum install -y file gawk gcc gcc-c++ git krb5-devel krb5-server \
         krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \
         cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \
         postgresql postgresql-server \
-        wget vim-common nscd cmake fuse-devel zlib-devel \
+        wget vim-common nscd cmake zlib-devel \
-        psmisc lsof openssh-server python3-devel python3-setuptools \
+        procps psmisc lsof openssh-server python3-devel python3-setuptools \
         net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \
         java-${REDHAT_JAVA_VERSION}-openjdk-src java-${REDHAT_JAVA_VERSION}-openjdk-devel
 
+# fuse-devel doesn't exist for Redhat 9
+redhat7 sudo yum install -y fuse-devel curl
+redhat8 sudo yum install -y fuse-devel curl
+# Redhat9 can have curl-minimal preinstalled, which can conflict with curl.
+# Adding --allowerasing allows curl to replace curl-minimal.
+redhat9 sudo yum install -y --allowerasing curl
+
 # RedHat / CentOS 8 exposes only specific versions of Python.
 # Set up unversioned default Python 2.x for older CentOS versions
-redhat6 sudo yum install -y python-devel python-setuptools python-argparse
 redhat7 sudo yum install -y python-devel python-setuptools python-argparse
 
 # Install Python 2.x explicitly for CentOS 8
@@ -278,6 +301,38 @@ function setup_python2() {
 redhat8 setup_python2
 redhat8 pip install --user argparse
 
+# Point Python to Python 3 for Redhat 9 and Ubuntu 22
+function setup_python3() {
+  # If python is already set, then use it. Otherwise, try to point python to python3.
+  if ! command -v python > /dev/null; then
+    if command -v python3 ; then
+      # Newer OSes (e.g. Redhat 9 and equivalents) make it harder to get Python 2, and we
+      # need to start using Python 3 by default.
+      # For these new OSes (Ubuntu 22, Redhat 9), there is no alternative entry for
+      # python, so we need to create one from scratch.
+      if command -v alternatives > /dev/null; then
+        if sudo alternatives --list | grep python > /dev/null ; then
+          sudo alternatives --set python /usr/bin/python3
+        else
+          # The alternative doesn't exist, create it
+          sudo alternatives --install /usr/bin/python python /usr/bin/python3 20
+        fi
+      elif command -v update-alternatives > /dev/null; then
+        # This is what Ubuntu 20/22+ does. There is no official python alternative,
+        # so we need to create one.
+        sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 20
+      else
+        echo "ERROR: trying to set python to point to python3"
+        echo "ERROR: alternatives/update-alternatives don't exist, so giving up..."
+        exit 1
+      fi
+    fi
+  fi
+}
+
+redhat9 setup_python3
+ubuntu22 setup_python3
+
 # CentOS repos don't contain ccache, so install from EPEL
 redhat sudo yum install -y epel-release
 redhat sudo yum install -y ccache
@@ -299,19 +354,16 @@ if [ ! -d /usr/local/apache-maven-3.5.4 ]; then
   MAVEN_DIRECTORY="/usr/local/apache-maven-3.5.4"
   redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}
   redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot}
+  redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}
+  redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot}
 fi
 
 if ! { service --status-all | grep -E '^ \[ \+ \]  ssh$'; }
 then
   ubuntu sudo service ssh start
-  # TODO: CentOS/RH 7 uses systemd, and this doesn't work.
+  redhat notindocker sudo service sshd start
-  redhat6 sudo service sshd start
+  redhat indocker sudo /usr/bin/ssh-keygen -A
-  redhat7 notindocker sudo service sshd start
+  redhat indocker sudo /usr/sbin/sshd
-  redhat8 notindocker sudo service sshd start
-  redhat7 indocker sudo /usr/bin/ssh-keygen -A
-  redhat7 indocker sudo /usr/sbin/sshd
-  redhat8 indocker sudo /usr/bin/ssh-keygen -A
-  redhat8 indocker sudo /usr/sbin/sshd
   # The CentOS 8.1 image includes /var/run/nologin by mistake; this file prevents
   # SSH logins. See https://github.com/CentOS/sig-cloud-instance-images/issues/60
   redhat8 indocker sudo rm -f /var/run/nologin
@@ -323,14 +375,9 @@ fi
 
 echo ">>> Configuring system"
 
-redhat6 sudo service postgresql initdb
+redhat notindocker sudo service postgresql initdb
-redhat6 sudo service postgresql stop
+redhat notindocker sudo service postgresql stop
-redhat7 notindocker sudo service postgresql initdb
+redhat indocker sudo -u postgres PGDATA=/var/lib/pgsql/data pg_ctl init
-redhat7 notindocker sudo service postgresql stop
-redhat7 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data pg_ctl init
-redhat8 notindocker sudo service postgresql initdb
-redhat8 notindocker sudo service postgresql stop
-redhat8 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data pg_ctl init
 ubuntu sudo service postgresql stop
 
 # These configurations expose connectiong to PostgreSQL via md5-hashed
@@ -344,14 +391,10 @@ redhat sudo sed -ri 's/local +all +all +(ident|peer)/local all all trust/g' \
 redhat sudo sed -i -e 's,\(host.*\)ident,\1md5,' /var/lib/pgsql/data/pg_hba.conf
 
 ubuntu sudo service postgresql start
-redhat6 sudo service postgresql start
+redhat notindocker sudo service postgresql start
-redhat7 notindocker sudo service postgresql start
-redhat8 notindocker sudo service postgresql start
 # Important to redirect pg_ctl to a logfile, lest it keep the stdout
 # file descriptor open, preventing the shell from exiting.
-redhat7 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data bash -c \
+redhat indocker sudo -u postgres PGDATA=/var/lib/pgsql/data bash -c \
-  "pg_ctl start -w --timeout=120 >> /var/lib/pgsql/pg.log 2>&1"
-redhat8 indocker sudo -u postgres PGDATA=/var/lib/pgsql/data bash -c \
   "pg_ctl start -w --timeout=120 >> /var/lib/pgsql/pg.log 2>&1"
 
 # Set up postgres for HMS
@@ -419,11 +462,10 @@ echo -e "\n* - nofile 1048576" | sudo tee -a /etc/security/limits.conf
 
 # Default on CentOS limits a user to 1024 or 4096 processes (threads) , which isn't
 # enough for minicluster with all of its friends.
-redhat6 sudo sed -i 's,\*\s*soft\s*nproc\s*[0-9]*$,* soft nproc unlimited,' \
-  /etc/security/limits.d/*-nproc.conf
 redhat7 sudo sed -i 's,\*\s*soft\s*nproc\s*[0-9]*$,* soft nproc unlimited,' \
   /etc/security/limits.d/*-nproc.conf
 redhat8 echo -e "* soft nproc unlimited" | sudo tee -a /etc/security/limits.conf
+redhat9 echo -e "* soft nproc unlimited" | sudo tee -a /etc/security/limits.conf
 
 echo ">>> Checking out Impala"
 

bin/bootstrap_toolchain.py

@@ -80,11 +80,15 @@ OS_MAPPING = [
   OsMapping("centos8", "ec2-package-centos-8"),
   OsMapping("rocky8", "ec2-package-centos-8"),
   OsMapping("almalinux8", "ec2-package-centos-8"),
+  OsMapping("rhel9", "ec2-package-rocky-9"),
+  OsMapping("rocky9", "ec2-package-rocky-9"),
+  OsMapping("almalinux9", "ec2-package-rocky-9"),
   OsMapping("sles12", "ec2-package-sles-12"),
   OsMapping("sles15", "ec2-package-sles-15"),
   OsMapping('ubuntu16', "ec2-package-ubuntu-16-04"),
   OsMapping('ubuntu18', "ec2-package-ubuntu-18-04"),
-  OsMapping('ubuntu20', "ec2-package-ubuntu-20-04")
+  OsMapping('ubuntu20', "ec2-package-ubuntu-20-04"),
+  OsMapping('ubuntu22', "ec2-package-ubuntu-22-04")
 ]
 
 

bin/cmake_aux/create_py3_virtualenv.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Create a python3 virtualenv. When system python is 3.6 or higher,
+# we can just use the built-in venv module to create the virtualenv.
+# If system python is older or the built-in venv module fails, then
+# this falls back to impala-virtualenv, which uses python2 to
+# initialize a virtualenv using python3.
+#
+# This takes a single argument, which is the destination directory
+# for the virtualenv:
+# create_py3_virtualenv.sh venv_dir
+
+set -euo pipefail
+
+# We should only be calling this when system python3 is available
+[[ -n ${IMPALA_SYSTEM_PYTHON3} ]]
+
+VENV_DIR=$1
+
+IS_PY36_OR_HIGHER=$(${IMPALA_SYSTEM_PYTHON3} -c \
+  "import sys; print('true' if sys.version_info.minor >= 6 else 'false')")
+
+# If using Python >= 3.6, try to use the builtin venv package.
+if $IS_PY36_OR_HIGHER ; then
+  if ${IMPALA_SYSTEM_PYTHON3} -m venv ${VENV_DIR} ; then
+    # Success
+    exit 0
+  fi
+fi
+
+if $IS_PY36_OR_HIGHER ; then
+  echo "WARNING: Tried to create virtualenv with Python3's venv module and failed."
+  echo "Falling back to old impala-virtualenv path..."
+fi
+# Fall back to using the old impala-virtualenv method
+impala-virtualenv --python ${IMPALA_SYSTEM_PYTHON3} ${VENV_DIR}
+

bin/impala-config.sh

@@ -81,7 +81,7 @@ export USE_APACHE_HIVE=${USE_APACHE_HIVE-false}
 # moving to a different build of the toolchain, e.g. when a version is bumped or a
 # compile option is changed. The build id can be found in the output of the toolchain
 # build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=296-f7e1d0d78b
+export IMPALA_TOOLCHAIN_BUILD_ID=320-33e177225e
 # Versions of toolchain dependencies.
 # -----------------------------------
 export IMPALA_AVRO_VERSION=1.7.4-p5
@@ -114,7 +114,7 @@ export IMPALA_GFLAGS_VERSION=2.2.0-p2
 unset IMPALA_GFLAGS_URL
 export IMPALA_GLOG_VERSION=0.3.5-p3
 unset IMPALA_GLOG_URL
-export IMPALA_GPERFTOOLS_VERSION=2.5-p4
+export IMPALA_GPERFTOOLS_VERSION=2.10
 unset IMPALA_GPERFTOOLS_URL
 export IMPALA_GTEST_VERSION=1.6.0
 unset IMPALA_GTEST_URL
@@ -124,9 +124,9 @@ export IMPALA_LIBEV_VERSION=4.20-p1
 unset IMPALA_LIBEV_URL
 export IMPALA_LIBUNWIND_VERSION=1.5.0-p1
 unset IMPALA_LIBUNWIND_URL
-export IMPALA_LLVM_VERSION=5.0.1-p5
+export IMPALA_LLVM_VERSION=5.0.1-p7
 unset IMPALA_LLVM_URL
-export IMPALA_LLVM_ASAN_VERSION=5.0.1-p5
+export IMPALA_LLVM_ASAN_VERSION=5.0.1-p7
 unset IMPALA_LLVM_ASAN_URL
 
 # To limit maximum memory available for the mini-cluster and CDH cluster, add the
@@ -140,7 +140,7 @@ export IMPALA_LLVM_UBSAN_BASE_VERSION=5.0.1
 
 # Debug builds should use the release+asserts build to get additional coverage.
 # Don't use the LLVM debug build because the binaries are too large to distribute.
-export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p5
+export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p7
 unset IMPALA_LLVM_DEBUG_URL
 export IMPALA_LZ4_VERSION=1.9.3
 unset IMPALA_LZ4_URL
@@ -265,6 +265,7 @@ export IMPALA_OBS_VERSION=3.1.1-hw-42
 # free distributions, but Redhat UBI images are known to work.
 export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"}
 export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"}
+export IMPALA_REDHAT9_DOCKER_BASE=${IMPALA_REDHAT9_DOCKER_BASE:-"rockylinux:9.2"}
 
 # When set to true, this instructs start-impala-cluster.py to start with Java 11 images
 # created via 'make docker_java11_images' (or 'docker_debug_java11_images'). The Java
@@ -947,7 +948,7 @@ fi
 # overall build type) and does not apply when using a local Kudu build.
 export USE_KUDU_DEBUG_BUILD=${USE_KUDU_DEBUG_BUILD-false}
 
-export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"345fd44ca3"}
+export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"14c326461c"}
 export IMPALA_KUDU_HOME=${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$IMPALA_KUDU_VERSION
 export IMPALA_KUDU_JAVA_HOME=\
 ${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-${IMPALA_KUDU_VERSION}/java

docker/CMakeLists.txt

@@ -39,14 +39,16 @@ set(DISTRO_BASE_IMAGE "UNSUPPORTED")
 if(${OS_DISTRIB_ID} STREQUAL "ubuntu")
   if(${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
      ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04" OR
-     ${OS_DISTRIB_VERSION_ID} STREQUAL "20.04")
+     ${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR
+     ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04")
     set(DISTRO_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
     set(QUICKSTART_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
   endif()
   if (${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
       ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04")
     set(PIP "python-pip")
-  elseif (${OS_DISTRIB_VERSION_ID} STREQUAL "20.04")
+  elseif (${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR
+          ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04")
     set(PIP "python3-pip")
   endif()
 elseif(${OS_DISTRIB_ID} STREQUAL "rhel" OR
@@ -59,6 +61,8 @@ elseif(${OS_DISTRIB_ID} STREQUAL "rhel" OR
     set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT7_DOCKER_BASE}")
   elseif(${OS_DISTRIB_VERSION_ID} MATCHES "8.*")
     set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT8_DOCKER_BASE}")
+  elseif(${OS_DISTRIB_VERSION_ID} MATCHES "9.*")
+    set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT9_DOCKER_BASE}")
   endif()
 endif()
 MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}")

docker/entrypoint.sh

@@ -108,7 +108,10 @@ function build() {
   fi
   if command -v apt-get > /dev/null; then
     apt-get update
-    apt-get install -y sudo git lsb-release python
+    # The 'python' package doesn't exist on Ubuntu 22, so this installs python3.
+    # TODO: It might not be necessary to install python here, as the container
+    # will invoke bootstrap_system.sh.
+    apt-get install -y sudo git python3
   elif grep 'release 8\.' /etc/redhat-release; then
     # WARNING: Install the following packages one by one!
     # Installing them in a common transaction breaks something inside yum/dnf,

fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java

@@ -75,7 +75,7 @@ public class LdapImpalaShellTest {
     // Runs the following command:
     // python -c "import ssl; print hasattr(ssl, 'create_default_context')"
     String[] cmd = {
-        "python", "-c", "import ssl; print hasattr(ssl, 'create_default_context')"};
+        "python", "-c", "import ssl; print(hasattr(ssl, 'create_default_context'))"};
     return Boolean.parseBoolean(RunShellCommand.Run(cmd, true, "", "").replace("\n", ""));
   }
 

lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py

@@ -58,7 +58,7 @@ def junitxml_prune_notrun(junitxml_filename):
     # not support an XML declaration on Python 2.6, so use minidom to write the XML.
     root_node_minidom = minidom.parseString(ET.tostring(root))
     junitxml_string = root_node_minidom.toxml(encoding="utf-8")
-    with open(junitxml_filename, "w") as f:
+    with open(junitxml_filename, "wb") as f:
         f.write(junitxml_string)
 
 

shell/CMakeLists.txt

@@ -53,8 +53,10 @@ add_custom_target(shell_python2_install DEPENDS "${PYTHON2_VENV}" shell_pypi_tes
   COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PYTHON2_PIP_CACHE}" "${SHELL_TEST_PKG}"
 )
 
+# In cases where system python3 is old, this can use impala-virtualenv, so it
+# needs to depend on impala_python.
 add_custom_command(OUTPUT "${PYTHON3_VENV}" DEPENDS impala_python
-  COMMAND impala-virtualenv --python "$ENV{IMPALA_SYSTEM_PYTHON3}" "${PYTHON3_VENV}"
+  COMMAND "${CMAKE_SOURCE_DIR}/bin/cmake_aux/create_py3_virtualenv.sh" "${PYTHON3_VENV}"
 )
 
 add_custom_target(shell_python3_install DEPENDS "${PYTHON3_VENV}" shell_pypi_test_package

tests/custom_cluster/test_client_ssl.py

@@ -21,6 +21,7 @@ import json
 import logging
 import os
 import pytest
+import re
 import requests
 import signal
 import ssl
@@ -244,18 +245,26 @@ class TestClientSsl(CustomClusterTestSuite):
     run_impala_shell_cmd(vector, args, expect_success=False)
 
   def _validate_positive_cases(self, vector, ca_cert=""):
+    python3_10_version_re = re.compile(r"using Python 3\.1[0-9]")
     shell_options = ["--ssl", "-q", "select 1 + 2"]
     result = run_impala_shell_cmd(vector, shell_options, wait_until_connected=False)
     for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]:
       assert msg in result.stderr
-    assert self.DEPRECATION_WARNING not in result.stderr
+    # Python >3.10 has deprecated ssl.PROTOCOL_TLS and impala-shell currently emits a
+    # DeprecationWarning for that version. As a temporary workaround, this skips the
+    # assert about deprecation for Python 3.10 or above. This can be removed when
+    # IMPALA-12219 is fixed.
+    # Note: This is the version that impala-shell uses, not the version pytests uses.
+    if not python3_10_version_re.search(result.stderr):
+      assert self.DEPRECATION_WARNING not in result.stderr
 
     if ca_cert != "":
       shell_options = shell_options + ["--ca_cert=%s" % ca_cert]
       result = run_impala_shell_cmd(vector, shell_options, wait_until_connected=False)
       for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]:
         assert msg in result.stderr
-      assert self.DEPRECATION_WARNING not in result.stderr
+      if not python3_10_version_re.search(result.stderr):
+        assert self.DEPRECATION_WARNING not in result.stderr
 
   def _verify_ssl_webserver(self):
     for port in ["25000", "25010", "25020"]: