impala/docker/install_os_packages.sh

#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This installs the minimal dependencies needed for Impala
# services to run. It currently handles Ubuntu and Redhat
# distributions. This takes an optional argument "--install-debug-tools"
# which installs extra debugging tools like curl, ping, etc.

set -euo pipefail

# Default level of extra debugging tools, controlled by the --install-debug-tools flag.
INSTALL_DEBUG_TOOLS=none

JAVA_VERSION=8
DRY_RUN=false
PKG_LIST=""
NON_PKG_NAMES=(apt-get yum apk install update add)

function print_usage {
    echo "install_os_packages.sh - Helper script to install OS dependencies"
    echo "[--install-debug-tools <none|basic|full>] : set the level of debug tools"\
         "to install"
    echo "[--java <version>] : Use specified Java version rather than the default Java 8."
    echo "[--dry-run] : Print the list of packages to install."
}

# Wraps the passed in command to either execute it (DRY_RUN=false) or just use it
# to update PKG_LIST.
function wrap {
  if $DRY_RUN; then
    for arg in $@; do
      if [[ "${NON_PKG_NAMES[@]}"  =~ "$arg" ]]; then
        continue
      elif [[ "$arg" == "-"* ]]; then
        # Ignores command options
        continue
      elif [[ "$PKG_LIST" != "" ]]; then
        PKG_LIST="$PKG_LIST,$arg"
      else
        PKG_LIST="$arg"
      fi
    done
  else
    "$@"
  fi
}

while [ -n "$*" ]
do
  case "$1" in
    --install-debug-tools)
      INSTALL_DEBUG_TOOLS="${2-}"
      shift;
      ;;
    --java)
      JAVA_VERSION="${2-}"
      shift;
      ;;
    --dry-run)
      DRY_RUN=true
      ;;
    --help|*)
      print_usage
      exit 1
      ;;
  esac
  shift
done

echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}"
echo "JAVA_VERSION=${JAVA_VERSION}"
echo "DRY_RUN=${DRY_RUN}"

case "$INSTALL_DEBUG_TOOLS" in
  none | basic | full)
    # These are valid.
    ;;
  "" | *)
    # The argument to --install-debug-tools is either missing, or is not a recognized
    # value.
    print_usage
    exit 1
    ;;
esac

# This can get more detailed if there are specific steps
# for specific versions, but at the moment the distribution
# is all we need.
DISTRIBUTION=Unknown
if [[ -f /etc/redhat-release ]]; then
  echo "Identified Redhat system."
  DISTRIBUTION=Redhat
elif [[ -f /etc/lsb-release ]]; then
  source /etc/lsb-release
  if [[ $DISTRIB_ID == Ubuntu ]]; then
    echo "Identified Ubuntu system."
    DISTRIBUTION=Ubuntu

    # Ubuntu 16.04 does not have Java 11 in its package repository,
    # so exit with an error.
    if [[ $DISTRIB_RELEASE == 16.04 ]] && [[ $JAVA_VERSION == 11 ]] ; then
      echo "ERROR: Java 11 is not supported on Ubuntu 16.04"
      exit 1
    fi
  fi
# Check /etc/os-release last: it exists on Red Hat and Ubuntu systems as well
elif [[ -f /etc/os-release ]]; then
  source /etc/os-release
  if [[ $ID == wolfi || $ID == chainguard ]]; then
    echo "Identified Wolfi-based system."
    DISTRIBUTION=Chainguard
  fi
fi

if [[ $DISTRIBUTION == Unknown ]]; then
  echo "ERROR: Did not detect supported distribution."
  echo "Only Ubuntu, Red Hat (or related), or Wolfi base images are supported."
  exit 1
fi

# Install minimal set of files.
# Optionally install extra debug tools.
if [[ $DISTRIBUTION == Ubuntu ]]; then
  export DEBIAN_FRONTEND=noninteractive
  wrap apt-get update
  wrap apt-get install -y \
      hostname \
      krb5-user \
      language-pack-en \
      libsasl2-2 \
      libsasl2-modules \
      libsasl2-modules-gssapi-mit \
      openjdk-${JAVA_VERSION}-jre-headless \
      tzdata

  # On Ubuntu there are no extra tools installed for $INSTALL_DEBUG_TOOLS == basic

  if [[ $INSTALL_DEBUG_TOOLS == full ]]; then
    echo "Installing full debug tools"
    wrap apt-get install -y \
        curl \
        dnsutils \
        iproute2 \
        iputils-ping \
        less \
        netcat-openbsd \
        sudo \
        vim
  fi
elif [[ $DISTRIBUTION == Redhat ]]; then
  if [[ $JAVA_VERSION == 8 ]]; then
    JAVA_VERSION=1.8.0
  fi
  wrap yum install -y --disableplugin=subscription-manager \
      cyrus-sasl-gssapi \
      cyrus-sasl-plain \
      hostname \
      java-${JAVA_VERSION}-openjdk-headless \
      krb5-workstation \
      openldap-devel \
      procps-ng \
      tzdata

  # UTF-8 masking functions require the presence of en_US.utf8.
  # Install the appropriate language packs. Redhat/Centos 7 come
  # with en_US.utf8, so there is no need to install anything.
  if ! grep 'release 7\.' /etc/redhat-release; then
      wrap yum install -y --disableplugin=subscription-manager \
          glibc-langpack-en \
          langpacks-en
  fi

  if [[ $INSTALL_DEBUG_TOOLS == basic || $INSTALL_DEBUG_TOOLS == full ]]; then
    echo "Installing basic debug tools"
    wrap yum install -y --disableplugin=subscription-manager \
        java-${JAVA_VERSION}-openjdk-devel \
        gdb
  fi

  if [[ $INSTALL_DEBUG_TOOLS == full ]]; then
    echo "Installing full debug tools"
    # Redhat 8 and 9 come with curl-minimal preinstalled, which conflicts with the
    # full curl package. Add --allowerasing to the yum command to allow package
    # replacement.

    wrap yum install -y --disableplugin=subscription-manager --allowerasing \
        bind-utils \
        curl \
        iproute \
        iputils \
        less \
        nmap-ncat \
        sudo \
        vim \
        which
  fi
elif [[ $DISTRIBUTION == Chainguard ]]; then
  # Package inventory:
  # glibc-locale-en and posix-libc-utils: for locale and 'locale' tool support
  # shadow: for groupadd and friends
  wrap apk add --no-cache --no-interactive \
    glibc-locale-posix \
    glibc-locale-en \
    glibc-iconv \
    posix-libc-utils \
    localedef \
    cyrus-sasl \
    krb5-libs \
    krb5 \
    openssl \
    openldap-dev \
    openjdk-${JAVA_VERSION}-jre \
    openjdk-${JAVA_VERSION}-default-jvm \
    shadow \
    tzdata

  # Set up /etc/localtime so that the container has a default local timezone.
  # Make this UTC.
  ln -sf /usr/share/zoneinfo/UTC /etc/localtime

  if [[ $INSTALL_DEBUG_TOOLS == basic || $INSTALL_DEBUG_TOOLS == full ]]; then
    echo "Installing basic debug tools"
    wrap apk add --no-cache --no-interactive \
        gdb \
        openjdk-${JAVA_VERSION}-default-jdk
  fi

  if [[ $INSTALL_DEBUG_TOOLS == full ]]; then
    echo "Installing full debug tools"
    wrap apk add --no-cache --no-interactive \
        bind-tools \
        curl \
        iproute2 \
        iputils \
        less \
        nmap \
        sudo \
        tzutils \
        vim
  fi
fi

if $DRY_RUN; then
  echo "The following packages would be installed:"
  echo "$PKG_LIST"
  exit 0
fi

# Verify en_US.utf8 is present
if ! locale -a | grep en_US.utf8 ; then
  echo "ERROR: en_US.utf8 locale is not present."
  exit 1
fi

if ! hostname ; then
  echo "ERROR: 'hostname' command failed."
  exit 1
fi

# graceful_shutdown_backends.sh requires the pgrep utility. Verify it is present.
if ! command -v pgrep ; then
  echo "ERROR: 'pgrep' is not present."
  exit 1
fi

# Java must be accessible for both the frontend and the backend. Verify it is present.
if ! command -v java; then
  echo "ERROR: Java cannot be found."
  exit 1
fi

# Impala will fail to start if the permissions on /var/tmp are not set to include
# the sticky bit (i.e. +t). Some versions of Redhat UBI images do not have
# this set by default, so specifically set the sticky bit for both /tmp and /var/tmp.
mkdir -p /var/tmp
chmod a=rwxt /var/tmp /tmp
# The busybox implementation of chmod is known to ignore certain chmod syntax variations,
# so check if the directories actually have the correct permissions
check_dir_stat() {
    target_dir=$1
    stat_expect=$2
    stat=$(stat -c '%u %g %a' $target_dir)
    if [[ "$stat" != "$stat_expect" ]]; then
        echo "Expect $target_dir permission '$stat_expect' but got '$stat' instead."
        exit 1
    fi
}
check_dir_stat "/tmp" "0 0 1777"
check_dir_stat "/var/tmp" "0 0 1777"

# To minimize the size for the Docker image, clean up any unnecessary files.
if [[ $DISTRIBUTION == Ubuntu ]]; then
  apt-get clean
  rm -rf /var/lib/apt/lists/*
elif [[ $DISTRIBUTION == Redhat ]]; then
  yum clean all
  rm -rf /var/cache/yum/*
elif [[ $DISTRIBUTION == Chainguard ]]; then
  rm -rf /var/cache/apk/*
fi