diff --git a/be/src/codegen/gen_ir_descriptions.py b/be/src/codegen/gen_ir_descriptions.py index 62fda01fb..0944de851 100755 --- a/be/src/codegen/gen_ir_descriptions.py +++ b/be/src/codegen/gen_ir_descriptions.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# This uses system python to avoid a dependency on impala-python, +#!/usr/bin/env python3 +# This uses system python to avoid a dependency on impala-python3, # because this runs during the build. # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index b91df6fc0..3f0ce3619 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -491,7 +491,7 @@ ssh localhost whoami # listening on localhost. See also HDFS-13797. To reproduce this, the following # snippet may be useful: # -# $impala-python +# $impala-python3 # >>> import logging # >>> logging.basicConfig(level=logging.DEBUG) # >>> logging.getLogger("requests.packages.urllib3").setLevel(logging.DEBUG) diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 20e759c26..59984fe05 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py index b3781928f..920375c38 100755 --- a/bin/check-rat-report.py +++ b/bin/check-rat-report.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/collect_minidumps.py b/bin/collect_minidumps.py index 57bc50f3f..6b378e5df 100755 --- a/bin/collect_minidumps.py +++ b/bin/collect_minidumps.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/bin/compare_branches.py b/bin/compare_branches.py index 5fcb3fbbe..82b7bab72 100755 --- a/bin/compare_branches.py +++ b/bin/compare_branches.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bin/diagnostics/collect_diagnostics.py b/bin/diagnostics/collect_diagnostics.py index 2257d6ff5..48497edf1 100755 --- a/bin/diagnostics/collect_diagnostics.py +++ b/bin/diagnostics/collect_diagnostics.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/dump_breakpad_symbols.py b/bin/dump_breakpad_symbols.py index ce10daa91..844a01382 100755 --- a/bin/dump_breakpad_symbols.py +++ b/bin/dump_breakpad_symbols.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/gen-backend-test-script.py b/bin/gen-backend-test-script.py index 6e572a839..1fcd2e084 100755 --- a/bin/gen-backend-test-script.py +++ b/bin/gen-backend-test-script.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/bin/gen_build_version.py b/bin/gen_build_version.py index 3f4f62b1c..e30d9b75d 100755 --- a/bin/gen_build_version.py +++ b/bin/gen_build_version.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# This uses system python to avoid a dependency on impala-python, +#!/usr/bin/env python3 +# This uses system python to avoid a dependency on impala-python3, # because this runs during the build. # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/bin/generate_minidump_collection_testdata.py b/bin/generate_minidump_collection_testdata.py index 9149e814d..947de636b 100755 --- a/bin/generate_minidump_collection_testdata.py +++ b/bin/generate_minidump_collection_testdata.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -95,7 +95,7 @@ def generate_conf_files(): f.write(CONFIG_FILE.format(options.log_dir, options.minidump_dir)) def random_bytes(num): - return ''.join(chr(random.randint(0, 255)) for _ in range(num)) + return bytearray(random.getrandbits(8) for _ in range(num)) def write_minidump(common_data, timestamp, target_dir): '''Generate and write the minidump into the target_dir. atime and mtime of the minidump diff --git a/bin/generate_xml_config.py b/bin/generate_xml_config.py index dc6e5eca8..67a28244a 100755 --- a/bin/generate_xml_config.py +++ b/bin/generate_xml_config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/get_code_size.py b/bin/get_code_size.py index dedded74d..e956a0011 100755 --- a/bin/get_code_size.py +++ b/bin/get_code_size.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/inline_pom.py b/bin/inline_pom.py index 3b99560d1..4cc9c5703 100755 --- a/bin/inline_pom.py +++ b/bin/inline_pom.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/jenkins/critique-gerrit-review.py b/bin/jenkins/critique-gerrit-review.py index c0620ec33..e4c8967e7 100755 --- a/bin/jenkins/critique-gerrit-review.py +++ b/bin/jenkins/critique-gerrit-review.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/bin/jenkins/dockerized-impala-preserve-vars.py b/bin/jenkins/dockerized-impala-preserve-vars.py index 53bfe61dc..3ecb8cc8d 100755 --- a/bin/jenkins/dockerized-impala-preserve-vars.py +++ b/bin/jenkins/dockerized-impala-preserve-vars.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/jenkins/populate_m2_directory.py b/bin/jenkins/populate_m2_directory.py index 6b8f526ce..a91f3e411 100755 --- a/bin/jenkins/populate_m2_directory.py +++ b/bin/jenkins/populate_m2_directory.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/bin/jenkins/thrift_parser.py b/bin/jenkins/thrift_parser.py index 6d26ca205..9e79ee79c 100755 --- a/bin/jenkins/thrift_parser.py +++ b/bin/jenkins/thrift_parser.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/bin/load-data.py b/bin/load-data.py index e86a3d698..82c4bb6d4 100755 --- a/bin/load-data.py +++ b/bin/load-data.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -220,7 +220,7 @@ def run_dataset_preload(dataset): dataset_preload_script = os.path.join(DATASET_DIR, dataset, "preload") if os.path.exists(dataset_preload_script): LOG.info("Running preload script for " + dataset) - if options.scale_factor > 1: + if options.scale_factor != "" and int(options.scale_factor) > 1: dataset_preload_script += " " + str(options.scale_factor) exec_cmd(dataset_preload_script, error_msg="Error executing preload script for " + dataset, exit_on_error=True) @@ -250,8 +250,8 @@ def get_dataset_for_workload(workload): if not os.path.isfile(dimension_file_name): LOG.error('Dimension file not found: ' + dimension_file_name) sys.exit(1) - with open(dimension_file_name, 'rb') as input_file: - match = re.search('dataset:\s*([\w\-\.]+)', input_file.read()) + with open(dimension_file_name, 'r') as input_file: + match = re.search(r'dataset:\s*([\w\-\.]+)', input_file.read()) if match: return match.group(1) else: diff --git a/bin/parse-thrift-profile.py b/bin/parse-thrift-profile.py index 6062d6536..b8cd4d372 100755 --- a/bin/parse-thrift-profile.py +++ b/bin/parse-thrift-profile.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/run-workload.py b/bin/run-workload.py index b40070ef8..78d118ee2 100755 --- a/bin/run-workload.py +++ b/bin/run-workload.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/bin/set-pythonpath.sh b/bin/set-pythonpath.sh index 07be68172..845d731c5 100755 --- a/bin/set-pythonpath.sh +++ b/bin/set-pythonpath.sh @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# Sets up the python path for impala-python. This is needed because tests and other +# Sets up the python path for impala-python3. This is needed because tests and other # utility scripts depend on some modules external to infra/python/env-*. # TODO: we should try to reduce our reliance on PYTHONPATH if possible. # diff --git a/bin/single_node_perf_run.py b/bin/single_node_perf_run.py index 88f6fb34d..7623a3d52 100755 --- a/bin/single_node_perf_run.py +++ b/bin/single_node_perf_run.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -210,7 +210,7 @@ def generate_profile_file(name, hash, base_dir): Writes the runtime profiles back in a simple text file in the same directory. """ - with open(name) as fid: + with open(name, 'rb') as fid: data = json.loads(fid.read().decode("utf-8", "ignore")) with open(os.path.join(base_dir, hash + "_profile.txt"), "w+") as out: # For each query diff --git a/bin/validate-unified-backend-test-filters.py b/bin/validate-unified-backend-test-filters.py index 42db3bf36..718cefc5d 100755 --- a/bin/validate-unified-backend-test-filters.py +++ b/bin/validate-unified-backend-test-filters.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/common/function-registry/gen_builtins_catalog.py b/common/function-registry/gen_builtins_catalog.py index 485c47365..ed2b0b48c 100755 --- a/common/function-registry/gen_builtins_catalog.py +++ b/common/function-registry/gen_builtins_catalog.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/common/function-registry/gen_geospatial_udf_wrappers.py b/common/function-registry/gen_geospatial_udf_wrappers.py index 19fa0191d..a0d9d000f 100755 --- a/common/function-registry/gen_geospatial_udf_wrappers.py +++ b/common/function-registry/gen_geospatial_udf_wrappers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py index 8a6c20f31..549dad2cd 100755 --- a/common/thrift/generate_error_codes.py +++ b/common/thrift/generate_error_codes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/common/thrift/generate_metrics.py b/common/thrift/generate_metrics.py index 66125ed14..527d0244d 100755 --- a/common/thrift/generate_metrics.py +++ b/common/thrift/generate_metrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py index 66b5e7eb6..1761452cc 100755 --- a/docker/setup_build_context.py +++ b/docker/setup_build_context.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapImpylaHttpTest.java b/fe/src/test/java/org/apache/impala/customcluster/LdapImpylaHttpTest.java index ff1ea9a02..5eea786f4 100644 --- a/fe/src/test/java/org/apache/impala/customcluster/LdapImpylaHttpTest.java +++ b/fe/src/test/java/org/apache/impala/customcluster/LdapImpylaHttpTest.java @@ -147,7 +147,7 @@ public class LdapImpylaHttpTest { long failedBasicAuthBefore = (long) client_.getMetric("impala.thrift-server.hiveserver2-http-frontend." + "total-basic-auth-failure"); - String[] noAuthCmd = {"impala-python", helper_, "--query", query_}; + String[] noAuthCmd = {"impala-python3", helper_, "--query", query_}; RunShellCommand.Run( noAuthCmd, /*shouldSucceed*/ false, "", "HTTP code 401: Unauthorized"); // Check that there is no authentication attempt. @@ -196,7 +196,7 @@ public class LdapImpylaHttpTest { private String[] buildCommand(String user, String password, String httpPath, String cookieNames) { - List command = Lists.newArrayList(Arrays.asList("impala-python", helper_, + List command = Lists.newArrayList(Arrays.asList("impala-python3", helper_, "--user", user, "--password", password, "--query", query_)); if (httpPath != null) command.addAll(Arrays.asList("--http_path", httpPath)); if (cookieNames != null) { @@ -216,7 +216,7 @@ public class LdapImpylaHttpTest { String errTemplate = "User '%s' is not authorized to delegate to '%s'"; // Run with an invalid proxy user. - //String[] command = {"impala-python", helper_, "--user", testUser2_, "--password", + //String[] command = {"impala-python3", helper_, "--user", testUser2_, "--password", // testPassword2_, "--http_path=/?doAs=" + delegateUser_, "--query", query}; String[] cmd = buildCommand(testUser2_, testPassword2_, "/?doAs=" + delegateUser_, null); @@ -230,7 +230,8 @@ public class LdapImpylaHttpTest { // 'doAs' parameter that cannot be decoded. cmd = buildCommand(testUser_, testPassword_, "/?doAs=%", null); - RunShellCommand.Run(cmd, /*shouldSucceed*/ false, "", "httplib.BadStatusLine"); + RunShellCommand.Run(cmd, /*shouldSucceed*/ false, "", + "http.client.RemoteDisconnected"); // Successfully delegate. cmd = buildCommand(testUser_, testPassword_, "/?doAs=" + delegateUser_, null); diff --git a/fe/src/test/resources/hive-site.xml.py b/fe/src/test/resources/hive-site.xml.py index d2bf9a9d9..aa55d8682 100644 --- a/fe/src/test/resources/hive-site.xml.py +++ b/fe/src/test/resources/hive-site.xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/infra/python/README b/infra/python/README index 9713dadd3..4355a445f 100644 --- a/infra/python/README +++ b/infra/python/README @@ -4,7 +4,7 @@ To install new packages: the package needs a C/C++ compiler to build . You should specify the version number using the "foo == x.y.z" notation so future upgrades can be done automatically. 2) Run deps/download_requirements, it will download the package to the deps dir. -3) Run the "impala-python" command, this should detect that requirements.txt changed and +3) Run the "impala-python3" command, this should detect that requirements.txt changed and automatically rebuild the virtualenv. 4) Now in the python prompt, you should be able to import the new module. diff --git a/infra/python/deps/py2-requirements.txt b/infra/python/deps/py2-requirements.txt index 919f7d41c..be2b8dd32 100644 --- a/infra/python/deps/py2-requirements.txt +++ b/infra/python/deps/py2-requirements.txt @@ -17,9 +17,6 @@ # Python2-only requirements -cm-api == 10.0.0 - # Already available as part of python on Linux. - readline == 6.2.4.1; sys_platform == 'darwin' flake8 == 3.9.2 mccabe == 0.6.1 pycodestyle == 2.7.0 diff --git a/infra/python/deps/py3-requirements.txt b/infra/python/deps/py3-requirements.txt index 051226125..7ea875d0e 100644 --- a/infra/python/deps/py3-requirements.txt +++ b/infra/python/deps/py3-requirements.txt @@ -31,6 +31,7 @@ pylint == 2.10.2 platformdirs == 2.4.1 typing-extensions == 3.10.0.2 k5test==0.10.3 +kazoo==2.8.0 # Below are needed only for dev flake8==3.9.2 diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt index de320c3e2..ae27ff169 100644 --- a/infra/python/deps/requirements.txt +++ b/infra/python/deps/requirements.txt @@ -32,7 +32,6 @@ impyla == 0.21a3 pure-sasl == 0.6.2 # six == 1.14.0 (specified separately) thrift_sasl == 0.4.3 -kazoo == 2.2.1 kerberos == 1.3.1 pexpect == 3.3 pg8000 == 1.10.2 diff --git a/lib/python/impala_py_lib/jenkins/generate_junitxml.py b/lib/python/impala_py_lib/jenkins/generate_junitxml.py index 9382d8ac3..0cac2d5d9 100755 --- a/lib/python/impala_py_lib/jenkins/generate_junitxml.py +++ b/lib/python/impala_py_lib/jenkins/generate_junitxml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -50,7 +50,7 @@ class JunitReport(object): Log files are timestamped, so they will not overwrite previous files containing output of the same step. - For use from within a python script (must be invoked with impala-python), an + For use from within a python script (must be invoked with impala-python3), an example might look like: >>> from impala_py_lib.jenkins.generate_junitxml import JunitReport diff --git a/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py b/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py index 3abfb8823..533923ae9 100755 --- a/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py +++ b/lib/python/impala_py_lib/jenkins/junitxml_prune_notrun.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/lib/python/impala_py_lib/profiles.py b/lib/python/impala_py_lib/profiles.py index 5425e06ef..c1232a93c 100644 --- a/lib/python/impala_py_lib/profiles.py +++ b/lib/python/impala_py_lib/profiles.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/lib/python/setup.py b/lib/python/setup.py index c2b3cecc8..210bca373 100644 --- a/lib/python/setup.py +++ b/lib/python/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/CMakeLists.txt b/shell/CMakeLists.txt index 467849ff2..c0c378d97 100644 --- a/shell/CMakeLists.txt +++ b/shell/CMakeLists.txt @@ -75,7 +75,8 @@ add_custom_target(shell_impala_build_version COMMAND "${CMAKE_SOURCE_DIR}/shell/gen_impala_build_version.sh" ) -add_custom_target(shell_pypi_package DEPENDS gen-deps shell_impala_build_version impala_python +add_custom_target(shell_pypi_package + DEPENDS gen-deps shell_impala_build_version impala_python3 COMMAND "${CMAKE_SOURCE_DIR}/shell/packaging/make_python_package.sh" ) @@ -88,7 +89,8 @@ set(SHELL_TEST_PKG "${CMAKE_SOURCE_DIR}/shell/build/dist/impala_shell-${PKG_VERSION}.tar.gz") get_filename_component(SHELL_TEST_PKG_DIR "${SHELL_TEST_PKG}" DIRECTORY) # Generates SHELL_TEST_PKG -add_custom_target(shell_pypi_test_package DEPENDS gen-deps shell_impala_build_version impala_python +add_custom_target(shell_pypi_test_package + DEPENDS gen-deps shell_impala_build_version impala_python3 COMMAND env BUILD_VERSION=${PKG_VERSION} OFFICIAL=true DIST_DIR="${SHELL_TEST_PKG_DIR}" "${CMAKE_SOURCE_DIR}/shell/packaging/make_python_package.sh" ) diff --git a/shell/impala_shell/compatibility.py b/shell/impala_shell/compatibility.py index a578c9b1b..0011ddad7 100644 --- a/shell/impala_shell/compatibility.py +++ b/shell/impala_shell/compatibility.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/impala_shell/exec_summary.py b/shell/impala_shell/exec_summary.py index 7431cc704..d61b1ea40 100644 --- a/shell/impala_shell/exec_summary.py +++ b/shell/impala_shell/exec_summary.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/impala_shell/impala_client.py b/shell/impala_shell/impala_client.py old mode 100755 new mode 100644 index 58a33855c..10bc1b963 --- a/shell/impala_shell/impala_client.py +++ b/shell/impala_shell/impala_client.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/impala_shell/impala_shell.py b/shell/impala_shell/impala_shell.py old mode 100755 new mode 100644 index 6c5bab0bc..d8ed39fca --- a/shell/impala_shell/impala_shell.py +++ b/shell/impala_shell/impala_shell.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/impala_shell/impala_shell_config_defaults.py b/shell/impala_shell/impala_shell_config_defaults.py index f05d0ed48..ca61912e8 100644 --- a/shell/impala_shell/impala_shell_config_defaults.py +++ b/shell/impala_shell/impala_shell_config_defaults.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/impala_shell/option_parser.py b/shell/impala_shell/option_parser.py index fe5f5ab78..fbd39db11 100644 --- a/shell/impala_shell/option_parser.py +++ b/shell/impala_shell/option_parser.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/impala_shell/shell_output.py b/shell/impala_shell/shell_output.py index b2e638c4f..ba57d0dfa 100644 --- a/shell/impala_shell/shell_output.py +++ b/shell/impala_shell/shell_output.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/shell/packaging/make_python_package.sh b/shell/packaging/make_python_package.sh index 92700918f..ea8f70ddb 100755 --- a/shell/packaging/make_python_package.sh +++ b/shell/packaging/make_python_package.sh @@ -65,7 +65,7 @@ create_distributable_python_package() { pushd "${PACKAGE_DIR}" echo "Building package..." PACKAGE_TYPE="${PACKAGE_TYPE:-}" OFFICIAL="${OFFICIAL:-}" \ - impala-python setup.py sdist --dist-dir "${DIST_DIR}" + impala-python3 setup.py sdist --dist-dir "${DIST_DIR}" popd if [[ "${NO_CLEAN_DIST}" != "true" ]]; then diff --git a/shell/setup.py b/shell/setup.py index ec53615a9..ef56d355d 100644 --- a/shell/setup.py +++ b/shell/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one @@ -94,7 +93,7 @@ def get_version(): if os.getenv('BUILD_VERSION') is not None: package_version = os.getenv('BUILD_VERSION') else: - version_match = re.search('\d+\.\d+\.\d+', impala_build_version.get_version()) + version_match = re.search(r'\d+\.\d+\.\d+', impala_build_version.get_version()) if version_match is None: sys.exit('Unable to acquire Impala version.') package_version = version_match.group(0) @@ -102,7 +101,7 @@ def get_version(): # packages can be marked as alpha, beta, or rc RELEASE_TYPE release_type = os.getenv('RELEASE_TYPE') if release_type: - if not re.match('(a|b|rc)\d+?', release_type): + if not re.match(r'(a|b|rc)\d+?', release_type): msg = """\ RELEASE_TYPE \'{0}\' does not conform to any PEP-440 release format: diff --git a/testdata/bin/check-hbase-nodes.py b/testdata/bin/check-hbase-nodes.py index 876955de5..5b21ca5c8 100755 --- a/testdata/bin/check-hbase-nodes.py +++ b/testdata/bin/check-hbase-nodes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh index 213ee1432..42deb1650 100755 --- a/testdata/bin/create-load-data.sh +++ b/testdata/bin/create-load-data.sh @@ -290,7 +290,7 @@ function load-data { LOG_FILE=${IMPALA_DATA_LOADING_LOGS_DIR}/${LOG_BASENAME} echo "$LOAD_MSG. Logging to ${LOG_FILE}" # Use unbuffered logging by executing with -u - if ! impala-python -u ${IMPALA_HOME}/bin/load-data.py ${ARGS[@]} &> ${LOG_FILE}; then + if ! impala-python3 -u ${IMPALA_HOME}/bin/load-data.py ${ARGS[@]} &> ${LOG_FILE}; then echo Error loading data. The end of the log file is: tail -n 50 $LOG_FILE return 1 @@ -321,7 +321,7 @@ function load-aux-workloads { # Load all the auxiliary workloads (if any exist) if [ -d ${IMPALA_AUX_WORKLOAD_DIR} ] && [ -d ${IMPALA_AUX_DATASET_DIR} ]; then echo Loading auxiliary workloads. Logging to $LOG_FILE. - if ! impala-python -u ${IMPALA_HOME}/bin/load-data.py --workloads all\ + if ! impala-python3 -u ${IMPALA_HOME}/bin/load-data.py --workloads all\ --impalad=${IMPALAD}\ --hive_hs2_hostport=${HS2_HOST_PORT}\ --hdfs_namenode=${HDFS_NN}\ diff --git a/testdata/bin/create-tpc-jdbc-tables.py b/testdata/bin/create-tpc-jdbc-tables.py index d6bf226b8..668d6ec42 100755 --- a/testdata/bin/create-tpc-jdbc-tables.py +++ b/testdata/bin/create-tpc-jdbc-tables.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py index 8ceccbb2c..229c49891 100755 --- a/testdata/bin/generate-schema-statements.py +++ b/testdata/bin/generate-schema-statements.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/generate-test-vectors.py b/testdata/bin/generate-test-vectors.py index 39bd7b0e3..4eb0e32fb 100755 --- a/testdata/bin/generate-test-vectors.py +++ b/testdata/bin/generate-test-vectors.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/load-tpc-kudu.py b/testdata/bin/load-tpc-kudu.py index 6f0610ffa..b64a08bb0 100755 --- a/testdata/bin/load-tpc-kudu.py +++ b/testdata/bin/load-tpc-kudu.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/load_nested.py b/testdata/bin/load_nested.py index 929a421b7..31ba7295b 100755 --- a/testdata/bin/load_nested.py +++ b/testdata/bin/load_nested.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/random_avro_schema.py b/testdata/bin/random_avro_schema.py index ef084edbc..f1b30c454 100755 --- a/testdata/bin/random_avro_schema.py +++ b/testdata/bin/random_avro_schema.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/rewrite-iceberg-metadata.py b/testdata/bin/rewrite-iceberg-metadata.py index a435b5993..47ae6023f 100755 --- a/testdata/bin/rewrite-iceberg-metadata.py +++ b/testdata/bin/rewrite-iceberg-metadata.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/wait-for-hiveserver2.py b/testdata/bin/wait-for-hiveserver2.py index 1b7344596..ccf154ed9 100755 --- a/testdata/bin/wait-for-hiveserver2.py +++ b/testdata/bin/wait-for-hiveserver2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/bin/wait-for-metastore.py b/testdata/bin/wait-for-metastore.py index 983c40763..c988a5005 100755 --- a/testdata/bin/wait-for-metastore.py +++ b/testdata/bin/wait-for-metastore.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py index 53f31ed46..e465f726f 100644 --- a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py +++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/ozone-site.xml.py b/testdata/cluster/node_templates/common/etc/hadoop/conf/ozone-site.xml.py index de647278e..23e31d5cd 100644 --- a/testdata/cluster/node_templates/common/etc/hadoop/conf/ozone-site.xml.py +++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/ozone-site.xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py index dbcd9f9be..832c4e240 100644 --- a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py +++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/common/text_delims_table.py b/testdata/common/text_delims_table.py index 5a605e548..e807ae695 100755 --- a/testdata/common/text_delims_table.py +++ b/testdata/common/text_delims_table.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/testdata/common/widetable.py b/testdata/common/widetable.py index 6bb8cd354..682dfa966 100755 --- a/testdata/common/widetable.py +++ b/testdata/common/widetable.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/comparison/ORACLE.txt b/tests/comparison/ORACLE.txt index 1576aca54..f80f12f62 100644 --- a/tests/comparison/ORACLE.txt +++ b/tests/comparison/ORACLE.txt @@ -61,13 +61,13 @@ package dependencies: Ubuntu 14.04.2 LTS libaio1 -3. Ensure impala-python is working. It may be enough to simply run: +3. Ensure impala-python3 is working. It may be enough to simply run: - $ impala-python + $ impala-python3 Getting it working is out of scope of this document. -4. Run these commands to install cx_Oracle into the impala-python +4. Run these commands to install cx_Oracle into the impala-python3 virtual environment: $ source "${IMPALA_HOME}"/infra/python/env-gcc${IMPALA_GCC_VERSION}/bin/activate diff --git a/tests/comparison/cli_options.py b/tests/comparison/cli_options.py index 339faff49..493bf9d43 100644 --- a/tests/comparison/cli_options.py +++ b/tests/comparison/cli_options.py @@ -31,13 +31,10 @@ from warnings import filterwarnings from tests.comparison import db_connection from tests.comparison.cluster import ( - CmCluster, DEFAULT_HIVE_HOST, DEFAULT_HIVE_PASSWORD, DEFAULT_HIVE_PORT, DEFAULT_HIVE_USER, - CM_CLEAR_PORT, - CM_TLS_PORT, MiniCluster, MiniHiveCluster, ) @@ -115,7 +112,6 @@ def add_db_name_option(parser): def add_cluster_options(parser): add_minicluster_options(parser) - add_cm_options(parser) add_ssh_options(parser) parser.add_argument( '--hadoop-user-name', default=getuser(), @@ -146,33 +142,6 @@ def add_minicluster_options(parser): help='The number of impalads in the mini cluster.') -def add_cm_options(parser): - parser.add_argument( - '--cm-host', metavar='host name', - help='The host name of the CM server.') - # IMPALA-5455: --cm-port defaults to None so that --use-tls can later influence the - # default value of --cm-port: it needs to default to 7180, or 7183 if --use-tls is - # included. - parser.add_argument( - '--cm-port', default=None, type=int, metavar='port number', - help='Override the CM port. Defaults to {clear}, or {tls} with --use-tls'.format( - clear=CM_CLEAR_PORT, - tls=CM_TLS_PORT)) - parser.add_argument( - '--cm-user', default="admin", metavar='user name', - help='The name of the CM user.') - parser.add_argument( - '--cm-password', default="admin", metavar='password', - help='The password for the CM user.') - parser.add_argument( - '--cm-cluster-name', metavar='name', - help='If CM manages multiple clusters, use this to specify which cluster to use.') - parser.add_argument( - '--use-tls', action='store_true', default=False, - help='Whether to communicate with CM using TLS. This alters the default CM port ' - 'from {clear} to {tls}'.format(clear=CM_CLEAR_PORT, tls=CM_TLS_PORT)) - - def add_ssl_options(parser): group = parser.add_argument_group('SSL Options') group.add_argument( @@ -185,12 +154,7 @@ def add_ssl_options(parser): def create_cluster(args): - if args.cm_host: - cluster = CmCluster( - args.cm_host, port=args.cm_port, user=args.cm_user, password=args.cm_password, - cluster_name=args.cm_cluster_name, ssh_user=args.ssh_user, ssh_port=args.ssh_port, - ssh_key_file=args.ssh_key_file, use_tls=args.use_tls) - elif args.use_hive: + if args.use_hive: cluster = MiniHiveCluster(args.hive_host, args.hive_port) else: cluster = MiniCluster(args.hive_host, args.hive_port, args.minicluster_num_impalads) diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py index 15949c167..ef39e2390 100644 --- a/tests/comparison/cluster.py +++ b/tests/comparison/cluster.py @@ -48,13 +48,6 @@ try: except ImportError: from urlparse import urlparse -try: - from cm_api.api_client import ApiResource as CmApiResource -except ImportError: - # If the cm_api module is not available, we will not be able to use Cloudera Manager. - # This is fine for local testing. - pass - from tests.comparison.db_connection import HiveConnection, ImpalaConnection from tests.common.environ import HIVE_MAJOR_VERSION from tests.common.errors import Timeout @@ -70,9 +63,6 @@ DEFAULT_HIVE_PASSWORD = 'hive' DEFAULT_TIMEOUT = 300 -CM_CLEAR_PORT = 7180 -CM_TLS_PORT = 7183 - class Cluster(with_metaclass(ABCMeta, object)): """This is a base class for clusters. Cluster classes provide various methods for @@ -254,129 +244,6 @@ class MiniHiveCluster(MiniCluster): return os.environ["HIVE_CONF_DIR"] -class CmCluster(Cluster): - - def __init__(self, host_name, port=None, user="admin", password="admin", - cluster_name=None, ssh_user=None, ssh_port=None, ssh_key_file=None, - use_tls=False): - # Initialize strptime() to workaround https://bugs.python.org/issue7980. Apparently - # something in the CM API uses strptime(). - strptime("2015", "%Y") - - Cluster.__init__(self) - # IMPALA-5455: If the caller doesn't specify port, default it based on use_tls - if port is None: - if use_tls: - port = CM_TLS_PORT - else: - port = CM_CLEAR_PORT - self.cm = CmApiResource(host_name, server_port=port, username=user, password=password, - use_tls=use_tls) - clusters = self.cm.get_all_clusters() - if not clusters: - raise Exception("No clusters found in CM at %s" % host_name) - if cluster_name: - clusters_by_name = dict((c.name, c) for c in clusters) - if cluster_name not in clusters_by_name: - raise Exception(("No clusters named %s found in CM at %s." - "Available clusters are %s.") - % (cluster_name, host_name, ", ".join(sorted(clusters_by_name.keys())))) - self.cm_cluster = clusters_by_name[cluster_name] - else: - if len(clusters) > 1: - raise Exception(("Too many clusters found in CM at %s;" - " a cluster name must be provided") - % host_name) - self.cm_cluster = clusters[-1] - - self.ssh_user = ssh_user - self.ssh_port = ssh_port - self.ssh_key_file = ssh_key_file - self._ssh_client_lock = Lock() - self._ssh_clients_by_host_name = defaultdict(list) - - def shell(self, cmd, host_name, timeout_secs=DEFAULT_TIMEOUT): - with self._ssh_client(host_name) as client: - return client.shell(cmd, timeout_secs=timeout_secs) - - @contextmanager - def _ssh_client(self, host_name): - """Returns an SSH client for use in a 'with' block. When the 'with' context exits, - the client will be kept for reuse. - """ - with self._ssh_client_lock: - clients = self._ssh_clients_by_host_name[host_name] - if clients: - client = clients.pop() - else: - # IMPALA-7460: Insulate this import away from the global context so as to avoid - # requiring Paramiko unless it's absolutely needed. - from tests.util.ssh_util import SshClient - LOG.debug("Creating new SSH client for %s", host_name) - client = SshClient() - client.connect(host_name, username=self.ssh_user, key_filename=self.ssh_key_file) - error_occurred = False - try: - yield client - except Exception: - error_occurred = True - raise - finally: - if not error_occurred: - with self._ssh_client_lock: - self._ssh_clients_by_host_name[host_name].append(client) - - def _init_local_hadoop_conf_dir(self): - self._local_hadoop_conf_dir = mkdtemp(prefix='impala_mini_hive_cluster_') - data = BytesIO(self.cm.get("/clusters/%s/services/%s/clientConfig" - % (self.cm_cluster.name, self._find_service("HIVE").name))) - zip_file = ZipFile(data) - for name in zip_file.namelist(): - if name.endswith("/"): - continue - extract_path = os.path.join(self._local_hadoop_conf_dir, os.path.basename(name)) - with open(extract_path, "w") as conf_file: - conf_file.write(zip_file.open(name).read()) - - def _find_service(self, service_type): - """Find a service by its CM API service type. An exception will be raised if no - service is found or multiple services are found. See the CM API documentation for - more details about the service type. - """ - services = [s for s in self.cm_cluster.get_all_services() if s.type == service_type] - if not services: - raise Exception("No service of type %s found in cluster %s" - % (service_type, self.cm_cluster.name)) - if len(services) > 1: - raise Exception("Found %s services in cluster %s; only one is expected." - % len(services, self.cm_cluster.name)) - return services[0] - - def _find_role(self, role_type, service_type): - """Find a role by its CM API role and service type. An exception will be raised if - no roles are found. See the CM API documentation for more details about the - service and role types. - """ - service = self._find_service(service_type) - roles = service.get_roles_by_type(role_type) - if not roles: - raise Exception("No roles of type %s found in service %s" - % (role_type, service.name)) - return roles[0] - - def _init_hdfs(self): - self._hdfs = Hdfs(self, "hdfs") - - def _init_hive(self): - hs2 = self._find_role("HIVESERVER2", "HIVE") - host = self.cm.get_host(hs2.hostRef.hostId) - config = hs2.get_config(view="full")["hs2_thrift_address_port"] - self._hive = Hive(self, str(host.hostname), int(config.value or config.default)) - - def _init_impala(self): - self._impala = CmImpala(self, self._find_service("IMPALA")) - - class Service(object): """This is a base class for cluster services such as HDFS. Service classes will provide an interface for interacting with the service. @@ -646,23 +513,6 @@ class Impala(Service): raise NotImplementedError() -class CmImpala(Impala): - - def __init__(self, cluster, cm_api): - super(CmImpala, self).__init__(cluster, - [CmImpalad(i) for i in cm_api.get_roles_by_type("IMPALAD")]) - self._api = cm_api - - def restart(self): - LOG.info("Restarting Impala") - command = self._api.restart() - command = command.wait(timeout=(60 * 15)) - if command.active: - raise Timeout("Timeout waiting for Impala to restart") - if not command.success: - raise Exception("Failed to restart Impala: %s" % command.resultMessage) - - class Impalad(with_metaclass(ABCMeta, object)): def __init__(self): @@ -724,7 +574,7 @@ class Impalad(with_metaclass(ABCMeta, object)): echo Could not find a running impalad >&2 exit 1 fi - cat /proc/$PID/cmdline""").split(b"\0")[0] + cat /proc/$PID/cmdline""").split("\0")[0] def find_last_crash_message(self, start_time): """Returns a string with various info (backtrace and log messages) if any is found.""" @@ -884,52 +734,3 @@ class MiniClusterImpalad(Impalad): def find_core_dump_dir(self): raise NotImplementedError() - - -class CmImpalad(Impalad): - - def __init__(self, cm_api): - super(CmImpalad, self).__init__() - self._api = cm_api - self._host_name = None - self._hs2_port = None - self._web_ui_port = None - - @property - def host_name(self): - if not self._host_name: - self._host_name = str(self.cluster.cm.get_host(self._api.hostRef.hostId).hostname) - return self._host_name - - @property - def hs2_port(self): - if not self._hs2_port: - self._hs2_port = self._get_cm_config("hs2_port", value_type=int) - return self._hs2_port - - @property - def web_ui_port(self): - if not self._web_ui_port: - self._web_ui_port = self._get_cm_config("impalad_webserver_port", value_type=int) - return self._web_ui_port - - def find_pid(self): - # Get the oldest pid. In a keberized cluster, occasionally two pids could be - # found if -o isn't used. Presumably the second pid is the kerberos ticket - # renewer. - pid = self.shell("pgrep -o impalad || true") - if pid: - return int(pid) - - def find_process_mem_mb_limit(self): - return self._get_cm_config("impalad_memory_limit", value_type=int) // 1024 ** 2 - - def find_core_dump_dir(self): - return self._get_cm_config("core_dump_dir") - - def _get_cm_config(self, config, value_type=None): - config = self._api.get_config(view="full")[config] - value = config.value or config.default - if value_type: - return value_type(value) - return value diff --git a/tests/comparison/data_generator.py b/tests/comparison/data_generator.py index 9e744f96b..b3dfb41b0 100755 --- a/tests/comparison/data_generator.py +++ b/tests/comparison/data_generator.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one @@ -223,7 +223,7 @@ class DbPopulator(object): reducer_count += (table_data_generator.row_count // estimate_rows_per_reducer(table_data_generator, MB_PER_REDUCER)) + 1 mapper_input_data.append(serialize(table_data_generator)) - hdfs.write(mapper_input_file, data='\n'.join(mapper_input_data)) + hdfs.write(mapper_input_file, data=b'\n'.join(mapper_input_data)) files = ['common.py', 'db_types.py', 'data_generator_mapred_common.py', 'data_generator_mapper.py', 'data_generator_reducer.py', diff --git a/tests/comparison/data_generator_mapper.py b/tests/comparison/data_generator_mapper.py old mode 100755 new mode 100644 index 35d412701..b29f05b3e --- a/tests/comparison/data_generator_mapper.py +++ b/tests/comparison/data_generator_mapper.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/comparison/data_generator_mapred_common.py b/tests/comparison/data_generator_mapred_common.py index 2367a0a3f..118acee1a 100644 --- a/tests/comparison/data_generator_mapred_common.py +++ b/tests/comparison/data_generator_mapred_common.py @@ -24,7 +24,7 @@ ''' from __future__ import absolute_import, division, print_function -from base import range +from builtins import range import base64 import pickle from io import BytesIO @@ -71,8 +71,9 @@ class TextTableDataGenerator(object): # has a terminator. Impala and Hive accept this format with the option # 'ROW FORMAT DELIMITED'. self.output_file.write(b"\x01") - self.output_file.write(str(col_val_buffers[col_idx][val_buffer_idx])) - self.output_file.write("\n") + self.output_file.write( + str(col_val_buffers[col_idx][val_buffer_idx]).encode('utf-8')) + self.output_file.write("\n".encode('utf-8')) def _create_val_generator(self, val_type): val_generator = RandomValGenerator().create_val_generator(val_type) diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py index 079850fd4..ce851bf48 100755 --- a/tests/comparison/discrepancy_searcher.py +++ b/tests/comparison/discrepancy_searcher.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/comparison/leopard/controller.py b/tests/comparison/leopard/controller.py index 16abd1ed8..531abcc4d 100755 --- a/tests/comparison/leopard/controller.py +++ b/tests/comparison/leopard/controller.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/comparison/leopard/front_end.py b/tests/comparison/leopard/front_end.py index 5f344ec3f..630510836 100755 --- a/tests/comparison/leopard/front_end.py +++ b/tests/comparison/leopard/front_end.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/comparison/tests/README b/tests/comparison/tests/README index 2859bb2c8..dda0018cf 100644 --- a/tests/comparison/tests/README +++ b/tests/comparison/tests/README @@ -4,7 +4,7 @@ This directory contains tests for the Random Query Generator. How-to -To run the tests we assume you have an impala-python environment already +To run the tests we assume you have an impala-python3 environment already set up. Then: $ cd "${IMPALA_HOME}"/tests/comparison/tests diff --git a/tests/comparison/tests/conftest.py b/tests/comparison/tests/conftest.py index 69e4f7257..996e6aee7 100644 --- a/tests/comparison/tests/conftest.py +++ b/tests/comparison/tests/conftest.py @@ -19,7 +19,7 @@ from __future__ import absolute_import, division, print_function import pytest from tests.comparison import cli_options -from tests.comparison.cluster import CmCluster, MiniCluster +from tests.comparison.cluster import MiniCluster __cluster = None @@ -27,20 +27,12 @@ __cluster = None def pytest_addoption(parser): if not hasattr(parser, "add_argument"): parser.add_argument = parser.addoption - cli_options.add_cm_options(parser) @pytest.fixture def cluster(request): global __cluster if not __cluster: - cm_host = get_option_value(request, "cm_host") - if cm_host: - __cluster = CmCluster(cm_host, port=get_option_value(request, "cm_port"), - user=get_option_value(request, "cm_user"), - password=get_option_value(request, "cm_password"), - cluster_name=get_option_value(request, "cm_cluster_name")) - else: __cluster = MiniCluster() return __cluster diff --git a/tests/comparison/util/verify-oracle-connection.py b/tests/comparison/util/verify-oracle-connection.py index cbc14a757..91a5f3494 100755 --- a/tests/comparison/util/verify-oracle-connection.py +++ b/tests/comparison/util/verify-oracle-connection.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,7 +18,7 @@ # under the License. # This is a script that allows a tester to test his Oracle setup and cx_Oracle -# installation within his impala-python environment. It's meant to be super simple. The +# installation within his impala-python3 environment. It's meant to be super simple. The # emphasis here is on a cx_Oracle connection and cursor with no other distractions, even # command line option parsing/handling. Modify the constants below and run: # diff --git a/tests/custom_cluster/test_auto_scaling.py b/tests/custom_cluster/test_auto_scaling.py index 4ec043be2..e4bdb25f7 100644 --- a/tests/custom_cluster/test_auto_scaling.py +++ b/tests/custom_cluster/test_auto_scaling.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py index a13f56782..3efc7f288 100644 --- a/tests/custom_cluster/test_executor_groups.py +++ b/tests/custom_cluster/test_executor_groups.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/custom_cluster/test_no_delete_stats_in_truncate.py b/tests/custom_cluster/test_no_delete_stats_in_truncate.py index c7b922d4b..ffc82804b 100644 --- a/tests/custom_cluster/test_no_delete_stats_in_truncate.py +++ b/tests/custom_cluster/test_no_delete_stats_in_truncate.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/custom_cluster/test_shared_catalogd.py b/tests/custom_cluster/test_shared_catalogd.py index cdbb14719..a577e5bf1 100644 --- a/tests/custom_cluster/test_shared_catalogd.py +++ b/tests/custom_cluster/test_shared_catalogd.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/metadata/test_column_unicode.py b/tests/metadata/test_column_unicode.py index b905ba8b9..5e59916e5 100644 --- a/tests/metadata/test_column_unicode.py +++ b/tests/metadata/test_column_unicode.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_beeline.py b/tests/shell/test_beeline.py index d6206e067..6b7eea470 100644 --- a/tests/shell/test_beeline.py +++ b/tests/shell/test_beeline.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_cookie_util.py b/tests/shell/test_cookie_util.py index 8c48116ea..06f888b1c 100644 --- a/tests/shell/test_cookie_util.py +++ b/tests/shell/test_cookie_util.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_kerberos_util.py b/tests/shell/test_kerberos_util.py index 6bb546a74..d709d3526 100644 --- a/tests/shell/test_kerberos_util.py +++ b/tests/shell/test_kerberos_util.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_shell_client.py b/tests/shell/test_shell_client.py index fa505b487..97af0e9e7 100644 --- a/tests/shell/test_shell_client.py +++ b/tests/shell/test_shell_client.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py index 0658c0776..4a69c67f5 100644 --- a/tests/shell/test_shell_commandline.py +++ b/tests/shell/test_shell_commandline.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_shell_commandline_jwt_auth.py b/tests/shell/test_shell_commandline_jwt_auth.py index 5dc2412a9..7630ca00d 100644 --- a/tests/shell/test_shell_commandline_jwt_auth.py +++ b/tests/shell/test_shell_commandline_jwt_auth.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_shell_commandline_kerberos_auth.py b/tests/shell/test_shell_commandline_kerberos_auth.py index af6c47fcb..f9ee6ab31 100644 --- a/tests/shell/test_shell_commandline_kerberos_auth.py +++ b/tests/shell/test_shell_commandline_kerberos_auth.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py index 3ce3d6cfe..66d3e9256 100755 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/shell/util.py b/tests/shell/util.py index 30f0afb3d..2770d11db 100755 --- a/tests/shell/util.py +++ b/tests/shell/util.py @@ -1,4 +1,3 @@ -#!/usr/bin/env impala-python # -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py index 9503e530c..0f08403a5 100755 --- a/tests/stress/concurrent_select.py +++ b/tests/stress/concurrent_select.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/stress/extract_min_mem.py b/tests/stress/extract_min_mem.py index 33db7b3f5..ca6036a16 100755 --- a/tests/stress/extract_min_mem.py +++ b/tests/stress/extract_min_mem.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/stress/mem_broker.py b/tests/stress/mem_broker.py index e844cbe05..722958f2e 100644 --- a/tests/stress/mem_broker.py +++ b/tests/stress/mem_broker.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/stress/queries.py b/tests/stress/queries.py index 4ea878f09..ab45c915c 100644 --- a/tests/stress/queries.py +++ b/tests/stress/queries.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/stress/query_retries_stress_runner.py b/tests/stress/query_retries_stress_runner.py index cd89e0cc1..3f80d6741 100755 --- a/tests/stress/query_retries_stress_runner.py +++ b/tests/stress/query_retries_stress_runner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/stress/query_runner.py b/tests/stress/query_runner.py index bdf591f0a..703788832 100644 --- a/tests/stress/query_runner.py +++ b/tests/stress/query_runner.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/stress/runtime_info.py b/tests/stress/runtime_info.py index 5523aa66a..0d0fb3c6d 100644 --- a/tests/stress/runtime_info.py +++ b/tests/stress/runtime_info.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/stress/util.py b/tests/stress/util.py index 1b3d9fe42..de66d8e5c 100644 --- a/tests/stress/util.py +++ b/tests/stress/util.py @@ -1,5 +1,3 @@ -#!/usr/bin/env impala-python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/tests/util/acid_txn.py b/tests/util/acid_txn.py index 3ba00143d..d172d8ade 100644 --- a/tests/util/acid_txn.py +++ b/tests/util/acid_txn.py @@ -56,8 +56,8 @@ hostname = 'localhost' # Utility class for interacting with Hive ACID transactions. # It's basically a facade, i.e. it provides a simplified interface for HMS. # -# You can also use it interactively from impala-python, e.g.: -# $> impala-python +# You can also use it interactively from impala-python3, e.g.: +# $> impala-python3 # >>> from tests.util.acid_txn import AcidTxn # >>> at = AcidTxn() # >>> at.get_open_txns() diff --git a/tests/util/auto_scaler.py b/tests/util/auto_scaler.py index 79b87294b..c6d95d1d8 100755 --- a/tests/util/auto_scaler.py +++ b/tests/util/auto_scaler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/util/concurrent_workload.py b/tests/util/concurrent_workload.py index d44ebcd1b..b0b81cbbd 100755 --- a/tests/util/concurrent_workload.py +++ b/tests/util/concurrent_workload.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/util/run_impyla_http_query.py b/tests/util/run_impyla_http_query.py index 98f95d899..281313707 100755 --- a/tests/util/run_impyla_http_query.py +++ b/tests/util/run_impyla_http_query.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-python +#!/usr/bin/env impala-python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information