IMPALA-10608: Update kudu-python version and remove some unused packages

This updates kudu-python to version 1.14.0 (from 1.2.0).
As part of this, it disables ccache for bootstrap_virtualenv.py.
ccache wasn't working anyway, because pip install uses random
temporary directories. It also needs to copy a few files to
the build directory for the Kudu install. The advantage to
upgrading is that the new version no longer has a numpy dependency.

Additionally, this modifies a few minor packages:
 - virtualenv moves to the latest version prior to the rewrite
   that accompanied version 20 (i.e. 16.10.7).
 - setuptools moves to the last version that supports python 2.7 (44.1.1)
 - remove botos3, ipython, and ordereddict

These changes speed up installing the virtualenv
Before:
real	3m11.956s
user	2m49.620s
sys	0m14.266s
After:
real    1m38.798s
user    1m33.591s
sys     0m8.112s

Testing:
 - Hand tests, GVO run

Change-Id: Ib47770df9e46de448fe2bffef7abe2c3aa942fb9
Reviewed-on: http://gerrit.cloudera.org:8080/17231
Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

bin/compare_branches.py

@@ -68,10 +68,7 @@ import subprocess
 import sys
 
 from collections import defaultdict
-try:
+from collections import OrderedDict
-  from collections import OrderedDict
-except ImportError:
-  from ordereddict import OrderedDict
 from pprint import pformat
 
 def create_parser():

bin/impala-ipython

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-##############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-##############################################################################
-
-source $(dirname "$0")/impala-python-common.sh
-exec "$PY_ENV_DIR/bin/ipython" "$@"

infra/python/bootstrap_virtualenv.py

@@ -108,16 +108,6 @@ def exec_cmd(args, **kwargs):
   return output
 
 
-def use_ccache():
-  '''Returns true if ccache is available and should be used'''
-  if 'DISABLE_CCACHE' in os.environ: return False
-  try:
-    exec_cmd(['ccache', '-V'])
-    return True
-  except:
-    return False
-
-
 def select_cc():
   '''Return the C compiler command that should be used as a string or None if the
   compiler is not available '''
@@ -127,7 +117,6 @@ def select_cc():
   toolchain_gcc_dir = toolchain_pkg_dir("gcc")
   cc = os.path.join(toolchain_gcc_dir, "bin/gcc")
   if not os.path.exists(cc): return None
-  if use_ccache(): cc = "ccache %s" % cc
   return cc
 
 
@@ -284,6 +273,10 @@ def install_kudu_client_if_possible():
     env = dict(os.environ)
     env["KUDU_HOME"] = fake_kudu_build_dir
     kudu_client_dir = find_kudu_client_install_dir()
+    # Copy the include directory to the fake build directory
+    kudu_include_dir = os.path.join(kudu_client_dir, "include")
+    shutil.copytree(kudu_include_dir,
+                    os.path.join(fake_kudu_build_dir, "build", "latest", "src"))
     env["CPLUS_INCLUDE_PATH"] = os.path.join(kudu_client_dir, "include")
     env["LIBRARY_PATH"] = os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'),
                                                 os.path.join(kudu_client_dir, 'lib64')])

infra/python/deps/kudu-requirements.txt

@@ -19,4 +19,4 @@
 # and also depends on Cython being installed into the virtualenv, so it must be installed
 # after the toolchain is bootstrapped and all requirements in requirements.txt and
 # compiled-requirements.txt are installed into the virtualenv.
-kudu-python==1.2.0
+kudu-python==1.14.0

infra/python/deps/requirements.txt

@@ -22,14 +22,6 @@
 
 allpairs == 2.0.1
 argparse == 1.4.0
-# TODO: boto3 is now unused, it can be removed.
-boto3 == 1.2.3
-  simplejson == 3.3.0 # For python version 2.6
-  botocore == 1.3.30
-  python_dateutil == 2.5.2
-  docutils == 0.12
-  jmespath == 0.9.0
-  futures == 3.0.5
 cm-api == 10.0.0
   # Already available as part of python on Linux.
   readline == 6.2.4.1; sys_platform == 'darwin'
@@ -41,13 +33,13 @@ gcovr == 3.4
 hdfs == 2.0.2
   docopt == 0.6.2
   execnet == 1.4.0
+    apipkg == 1.4
 impyla == 0.17a1
   bitarray == 1.2.1
   sasl == 0.2.1
   # six == 1.14.0 (specified separately)
   thrift_sasl == 0.4.2
 kazoo == 2.2.1
-ordereddict == 1.1
 pexpect == 3.3
 pg8000 == 1.10.2
 prettytable == 0.7.2
@@ -72,14 +64,9 @@ sh == 1.11
 six == 1.14.0
 sqlparse == 0.3.1
 texttable == 0.8.3
-virtualenv == 13.1.0
+virtualenv == 16.7.10
 
 # Required for Kudu:
-  Cython == 0.23.4
+  Cython == 0.29.14
-  numpy == 1.10.4
+  pytz == 2020.1
-  pytz == 2018.3
 
-# For dev purposes, not used in scripting. Version 1.2.1 is the latest that supports 2.6.
-# TODO: this seems unused and could be removed
-ipython == 1.2.1
-  apipkg == 1.4

infra/python/deps/setuptools-requirements.txt

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Newer versions of setuptools don't support Python 2.6
+# Newer versions of setuptools don't support Python 2.7
-setuptools == 36.8.0
+setuptools == 44.1.1
-setuptools-scm == 1.15.4
+  wheel == 0.35.1
+setuptools-scm == 4.1.2

tests/comparison/cluster.py

@@ -29,8 +29,8 @@ import subprocess
 from abc import ABCMeta, abstractproperty
 from cm_api.api_client import ApiResource as CmApiResource
 from collections import defaultdict
+from collections import OrderedDict
 from contextlib import contextmanager
-from ordereddict import OrderedDict
 from getpass import getuser
 from itertools import izip
 from multiprocessing.pool import ThreadPool