diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..5574ffd31 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[settings] +force_alphabetical_sort_within_sections=true +force_sort_within_sections=true +include_trailing_comma=true +known_third_party=builtins +known_first_party=impala_py_lib,impala_shell,impala_thrift_gen,tests +line_length=90 +multi_line_output=3 +no_lines_before=STDLIB +order_by_type=false +skip_gitignore=true +skip_glob=testdata/cluster/cdh* \ No newline at end of file diff --git a/bin/cmake_aux/add_thrift_python_namespace.sh b/bin/cmake_aux/add_thrift_python_namespace.sh new file mode 100755 index 000000000..7ec482d2e --- /dev/null +++ b/bin/cmake_aux/add_thrift_python_namespace.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# add_thrift_python_namespace.sh IN_THRIFT_FILE OUT_THRIFT_FILE +# +# This script reads $IN_THRIFT_FILE and adds a python namespace +# (replacing any existing python namespace) with +# impala_thrift_gen.${BASE_NAME} where the BASE_NAME is the +# thrift filename without the ".thrift". i.e. Foo.thrift uses +# impala_thrift_gen.Foo python namespace. It writes the resulting +# thrift file to $OUT_THRIFT_FILE. +# +# This logic is taken from Impyla's impala/thrift/process_thrift.sh +# script with minor changes. This requires that the source thrift +# file have at least one preexisting non-python namespace. That is +# true for all of the Thrift files that we care about. + +set -eou pipefail + +THRIFT_FILE_IN=$1 +THRIFT_FILE_OUT=$2 + +FILE_NAME=$(basename $THRIFT_FILE_IN) +BASE_NAME=${FILE_NAME%.*} +# Awk script to add the python namespace before the first namespace +# in the thrift file. +ADD_NAMESPACE_PY=" + BEGIN { + n = 0 + } + { + if (\$0 ~ /^namespace/ && n == 0) { + print \"namespace py impala_thrift_gen.$BASE_NAME\"; + n += 1; + } + print \$0; + }" + +# Remove any existing python namespace, then add our namespace +cat $THRIFT_FILE_IN | grep -v "^namespace py" | awk "$ADD_NAMESPACE_PY" > $THRIFT_FILE_OUT diff --git a/bin/impala-isort b/bin/impala-isort new file mode 100755 index 000000000..8dab98af6 --- /dev/null +++ b/bin/impala-isort @@ -0,0 +1,21 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source "$(dirname "$0")/impala-python3-common.sh" +exec "$PY_ENV_DIR/bin/isort" "$@" diff --git a/common/thrift/.gitignore b/common/thrift/.gitignore index 0f6c0c55e..cd3eb435e 100644 --- a/common/thrift/.gitignore +++ b/common/thrift/.gitignore @@ -3,3 +3,4 @@ ErrorCodes.thrift MetricDefs.thrift hive-2-api/TCLIService.thrift hive-3-api/TCLIService.thrift +/thirdparty_thrift/ diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift index dd2fc423d..065c25ec2 100644 --- a/common/thrift/BackendGflags.thrift +++ b/common/thrift/BackendGflags.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.BackendGflags namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt index f86036606..fd77cf82a 100644 --- a/common/thrift/CMakeLists.txt +++ b/common/thrift/CMakeLists.txt @@ -135,18 +135,19 @@ function(THRIFT_GEN_DS VAR) set(${VAR} ${${VAR}} PARENT_SCOPE) endfunction(THRIFT_GEN_DS) +set(THIRDPARTY_THRIFT_DIR "thirdparty_thrift") set(HIVE_THRIFT_SOURCE_DIR "hive-$ENV{IMPALA_HIVE_MAJOR_VERSION}-api") set(TCLI_SERVICE_THRIFT "${HIVE_THRIFT_SOURCE_DIR}/TCLIService.thrift") message("Using Thrift CPP compiler: ${THRIFT_CPP_COMPILER}") message("Using Thrift JAVA compiler: ${THRIFT_JAVA_COMPILER}") message("Using Thrift PY compiler: ${THRIFT_PY_COMPILER}") set(THRIFT_QUIET_WRAPPER "${CMAKE_SOURCE_DIR}/bin/thrift-quiet-wrapper.sh") -set(THRIFT_CPP_INCLUDE_DIR_OPTION -I ${THRIFT_CPP_CONTRIB_DIR} - -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR}) -set(THRIFT_JAVA_INCLUDE_DIR_OPTION -I ${THRIFT_JAVA_CONTRIB_DIR} - -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR}) -set(THRIFT_PY_INCLUDE_DIR_OPTION -I ${THRIFT_PY_CONTRIB_DIR} - -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR}) +set(THRIFT_CPP_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR} + -I ${HIVE_THRIFT_SOURCE_DIR}) +set(THRIFT_JAVA_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR} + -I ${HIVE_THRIFT_SOURCE_DIR}) +set(THRIFT_PY_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR} + -I ${HIVE_THRIFT_SOURCE_DIR}) set(BE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/be/generated-sources) set(FE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/fe/generated-sources) # TODO: avoid duplicating generated java classes @@ -158,12 +159,13 @@ file(MAKE_DIRECTORY ${FE_OUTPUT_DIR}) file(MAKE_DIRECTORY ${EXT_DS_OUTPUT_DIR}) file(MAKE_DIRECTORY ${PYTHON_OUTPUT_DIR}) file(MAKE_DIRECTORY ${HIVE_THRIFT_SOURCE_DIR}) +file(MAKE_DIRECTORY ${THIRDPARTY_THRIFT_DIR}) # Args passed to thrift for Java gen set(JAVA_FE_ARGS ${THRIFT_JAVA_INCLUDE_DIR_OPTION} --gen java -o ${FE_OUTPUT_DIR}) set(JAVA_EXT_DS_ARGS ${THRIFT_JAVA_INCLUDE_DIR_OPTION} --gen java -o ${EXT_DS_OUTPUT_DIR}) set(PYTHON_ARGS ${THRIFT_PY_INCLUDE_DIR_OPTION} -r --gen py:no_utf8strings -o - ${PYTHON_OUTPUT_DIR}) + ${PYTHON_OUTPUT_DIR}) set (EXT_DATA_SRC_FILES ErrorCodes.thrift @@ -211,6 +213,16 @@ set (SRC_FILES ) SET_SOURCE_FILES_PROPERTIES(Status.thrift PROPERTIES OBJECT_DEPENDS ErrorCodes.thrift) +SET_SOURCE_FILES_PROPERTIES(CatalogObjects.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(CatalogService.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(JniCatalog.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(SqlConstraints.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +SET_SOURCE_FILES_PROPERTIES(beeswax.thrift PROPERTIES OBJECT_DEPENDS + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) add_custom_command(OUTPUT ErrorCodes.thrift COMMAND python generate_error_codes.py @@ -232,6 +244,33 @@ add_custom_command(OUTPUT hive-$ENV{IMPALA_HIVE_MAJOR_VERSION}-api/TCLIService.t DEPENDS hive-1-api/TCLIService.thrift ) +# This generates hive_metastore.thrift in the $THIRDPARTY_THRIFT_DIR. The two +# modification are: +# 1. Set the impala_thrift_gen python namespace +# 2. Rearranges the fb303 reference so that it doesn't have the share/fb303/if +# directory structure +add_custom_command(OUTPUT ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift + COMMAND ${CMAKE_SOURCE_DIR}/bin/cmake_aux/add_thrift_python_namespace.sh + $ENV{HIVE_METASTORE_THRIFT_DIR}/hive_metastore.thrift + ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp + COMMAND cat ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp | + sed 's|share/fb303/if/||' > ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift + COMMAND rm ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp + DEPENDS $ENV{HIVE_METASTORE_THRIFT_DIR}/hive_metastore.thrift +) + +# Generate fb303.thrift in the $THIRDPARTY_THRIFT_DIR with the appropriate +# impala_thrift_gen python namespace. +add_custom_command(OUTPUT ${THIRDPARTY_THRIFT_DIR}/fb303.thrift + COMMAND ${CMAKE_SOURCE_DIR}/bin/cmake_aux/add_thrift_python_namespace.sh + ${THRIFT_PY_CONTRIB_DIR}/share/fb303/if/fb303.thrift + ${THIRDPARTY_THRIFT_DIR}/fb303.thrift + DEPENDS ${THRIFT_PY_CONTRIB_DIR}/share/fb303/if/fb303.thrift +) + +SET_SOURCE_FILES_PROPERTIES(${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift + PROPERTIES OBJECT_DEPENDS ${THIRDPARTY_THRIFT_DIR}/fb303.thrift) + # Create a build command for each of the thrift src files and generate # a list of files they produce THRIFT_GEN(THRIFT_ALL_FILES ${SRC_FILES}) @@ -240,11 +279,15 @@ THRIFT_GEN_DS(THRIFT_DATA_SRC_FILES ${EXT_DATA_SRC_FILES}) add_custom_target(thrift-generated-files-error DEPENDS ErrorCodes.thrift) add_custom_target(thrift-generated-files-metrics DEPENDS MetricDefs.thrift) add_custom_target(thrift-generated-files-tcli-service DEPENDS ${TCLI_SERVICE_THRIFT}) +add_custom_target(thrift-generated-files-hive-metastore + DEPENDS ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift) +add_custom_target(thrift-generated-files-fb303 DEPENDS ${THIRDPARTY_THRIFT_DIR}/fb303.thrift) # Add a custom target that generates all the thrift files add_custom_target(thrift-cpp ALL DEPENDS ${THRIFT_ALL_FILES}) add_dependencies(thrift-cpp thrift-generated-files-metrics thrift-generated-files-error - thrift-generated-files-tcli-service) + thrift-generated-files-tcli-service thrift-generated-files-hive-metastore + thrift-generated-files-fb303) add_custom_target(thrift-ext-data-src ALL DEPENDS ${THRIFT_DATA_SRC_FILES}) add_dependencies(thrift-ext-data-src thrift-cpp) diff --git a/common/thrift/CatalogInternalService.thrift b/common/thrift/CatalogInternalService.thrift index 6007f0227..03a1208ad 100644 --- a/common/thrift/CatalogInternalService.thrift +++ b/common/thrift/CatalogInternalService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.CatalogInternalService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift index 6e186d6b9..0afc4f3a1 100644 --- a/common/thrift/CatalogObjects.thrift +++ b/common/thrift/CatalogObjects.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.CatalogObjects namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift index f57f62ec9..6702e3c62 100644 --- a/common/thrift/CatalogService.thrift +++ b/common/thrift/CatalogService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.CatalogService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Data.thrift b/common/thrift/Data.thrift index b68657f3e..999f9330b 100644 --- a/common/thrift/Data.thrift +++ b/common/thrift/Data.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Data namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/DataSinks.thrift b/common/thrift/DataSinks.thrift index 882fe0bee..942863b2c 100644 --- a/common/thrift/DataSinks.thrift +++ b/common/thrift/DataSinks.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.DataSinks namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Descriptors.thrift b/common/thrift/Descriptors.thrift index 2acc31264..83cc597c6 100644 --- a/common/thrift/Descriptors.thrift +++ b/common/thrift/Descriptors.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Descriptors namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ExecStats.thrift b/common/thrift/ExecStats.thrift index f629eaf0a..0a6073c86 100644 --- a/common/thrift/ExecStats.thrift +++ b/common/thrift/ExecStats.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.ExecStats namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Exprs.thrift b/common/thrift/Exprs.thrift index fb02a298f..c2b342956 100644 --- a/common/thrift/Exprs.thrift +++ b/common/thrift/Exprs.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Exprs namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ExternalDataSource.thrift b/common/thrift/ExternalDataSource.thrift index 7ec0d8079..4e8e6d593 100644 --- a/common/thrift/ExternalDataSource.thrift +++ b/common/thrift/ExternalDataSource.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.ExternalDataSource namespace cpp impala.extdatasource namespace java org.apache.impala.extdatasource.thrift diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift index 770017505..5426ba7e6 100644 --- a/common/thrift/Frontend.thrift +++ b/common/thrift/Frontend.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Frontend namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift index c01e537a4..5f324d668 100644 --- a/common/thrift/ImpalaInternalService.thrift +++ b/common/thrift/ImpalaInternalService.thrift @@ -18,6 +18,7 @@ // // This file contains the details of the protocol between coordinators and backends. +namespace py impala_thrift_gen.ImpalaInternalService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift index 88c4de5c4..43886c001 100644 --- a/common/thrift/ImpalaService.thrift +++ b/common/thrift/ImpalaService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.ImpalaService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/JniCatalog.thrift b/common/thrift/JniCatalog.thrift old mode 100755 new mode 100644 index 1b91bec65..62dc72fcd --- a/common/thrift/JniCatalog.thrift +++ b/common/thrift/JniCatalog.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.JniCatalog namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/LineageGraph.thrift b/common/thrift/LineageGraph.thrift index 61b2f5610..5d0f48202 100644 --- a/common/thrift/LineageGraph.thrift +++ b/common/thrift/LineageGraph.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.LineageGraph namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Logging.thrift b/common/thrift/Logging.thrift index 6335667f5..10c7c7837 100644 --- a/common/thrift/Logging.thrift +++ b/common/thrift/Logging.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Logging namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Metrics.thrift b/common/thrift/Metrics.thrift index 8d92bd74b..754cd44d3 100644 --- a/common/thrift/Metrics.thrift +++ b/common/thrift/Metrics.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Metrics namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/NetworkTest.thrift b/common/thrift/NetworkTest.thrift index d52751092..585bf28f5 100644 --- a/common/thrift/NetworkTest.thrift +++ b/common/thrift/NetworkTest.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.NetworkTest namespace cpp impalatest struct ThriftDataParams { diff --git a/common/thrift/Partitions.thrift b/common/thrift/Partitions.thrift index 0d1c860ae..b3fdef8fb 100644 --- a/common/thrift/Partitions.thrift +++ b/common/thrift/Partitions.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Partitions namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift index 3760a6b9f..929dd0008 100644 --- a/common/thrift/PlanNodes.thrift +++ b/common/thrift/PlanNodes.thrift @@ -21,6 +21,7 @@ // of the execution parameters of any one of the backends on which it is running // (those are recorded in TPlanFragmentInstanceCtx). +namespace py impala_thrift_gen.PlanNodes namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Planner.thrift b/common/thrift/Planner.thrift index 36622f97f..a600e6219 100644 --- a/common/thrift/Planner.thrift +++ b/common/thrift/Planner.thrift @@ -18,6 +18,7 @@ // // This file contains structures produced by the planner. +namespace py impala_thrift_gen.Planner namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift index b1ce5fd62..8e3fc5969 100644 --- a/common/thrift/Query.thrift +++ b/common/thrift/Query.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Query namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/ResourceProfile.thrift b/common/thrift/ResourceProfile.thrift index be76edd53..8fe092119 100644 --- a/common/thrift/ResourceProfile.thrift +++ b/common/thrift/ResourceProfile.thrift @@ -19,6 +19,7 @@ // and DataSinks to configure their memory usage. See ResourceProfile.java for more // details. +namespace py impala_thrift_gen.ResourceProfile namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Results.thrift b/common/thrift/Results.thrift index ff5ffd4ff..da639bf57 100644 --- a/common/thrift/Results.thrift +++ b/common/thrift/Results.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Results namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/RuntimeProfile.thrift b/common/thrift/RuntimeProfile.thrift index 02dd1e7ea..1f632b88e 100644 --- a/common/thrift/RuntimeProfile.thrift +++ b/common/thrift/RuntimeProfile.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.RuntimeProfile namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/SqlConstraints.thrift b/common/thrift/SqlConstraints.thrift index ee4a9cf30..7d3882e42 100644 --- a/common/thrift/SqlConstraints.thrift +++ b/common/thrift/SqlConstraints.thrift @@ -16,6 +16,7 @@ // under the License. +namespace py impala_thrift_gen.SqlConstraints namespace cpp impala namespace java org.apache.impala.thrift @@ -28,4 +29,4 @@ struct TSqlConstraints { // Foreign Keys information for the given table. 2: required list foreign_keys -} \ No newline at end of file +} diff --git a/common/thrift/StatestoreService.thrift b/common/thrift/StatestoreService.thrift index 1ae961718..e25ea18d4 100644 --- a/common/thrift/StatestoreService.thrift +++ b/common/thrift/StatestoreService.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.StatestoreService namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Status.thrift b/common/thrift/Status.thrift index 3997e4070..ef74c4965 100644 --- a/common/thrift/Status.thrift +++ b/common/thrift/Status.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Status namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/SystemTables.thrift b/common/thrift/SystemTables.thrift index e1cd1ca06..d1b84d2e0 100644 --- a/common/thrift/SystemTables.thrift +++ b/common/thrift/SystemTables.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.SystemTables namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Types.thrift b/common/thrift/Types.thrift index 0c951ee73..4903dfac0 100644 --- a/common/thrift/Types.thrift +++ b/common/thrift/Types.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Types namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/Zip.thrift b/common/thrift/Zip.thrift index b5f02d1b6..7d97e6c8e 100644 --- a/common/thrift/Zip.thrift +++ b/common/thrift/Zip.thrift @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +namespace py impala_thrift_gen.Zip namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/beeswax.thrift b/common/thrift/beeswax.thrift index 8fdb6c328..f66d06228 100644 --- a/common/thrift/beeswax.thrift +++ b/common/thrift/beeswax.thrift @@ -17,8 +17,8 @@ // Interface for interacting with Beeswax Server +namespace py impala_thrift_gen.beeswax namespace java com.cloudera.beeswax.api -namespace py beeswaxd namespace cpp beeswax include "hive_metastore.thrift" diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py index 80a275f90..b2c00b702 100755 --- a/common/thrift/generate_error_codes.py +++ b/common/thrift/generate_error_codes.py @@ -539,6 +539,7 @@ preamble = """ // IT BY HAND. // +namespace py impala_thrift_gen.ErrorCodes namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/generate_metrics.py b/common/thrift/generate_metrics.py index 6aa02822f..66125ed14 100755 --- a/common/thrift/generate_metrics.py +++ b/common/thrift/generate_metrics.py @@ -88,6 +88,7 @@ THRIFT_PREAMBLE = """ // THIS FILE IS AUTO GENERATED BY generate_metrics.py DO NOT MODIFY IT BY HAND. // +namespace py impala_thrift_gen.Metrics namespace cpp impala namespace java org.apache.impala.thrift diff --git a/common/thrift/hive-1-api/TCLIService.thrift b/common/thrift/hive-1-api/TCLIService.thrift index 09f272e52..777fcae24 100644 --- a/common/thrift/hive-1-api/TCLIService.thrift +++ b/common/thrift/hive-1-api/TCLIService.thrift @@ -32,6 +32,7 @@ // * Service names begin with the letter "T", use a capital letter for each // new word (with no underscores), and end with the word "Service". +namespace py impala_thrift_gen.TCLIService namespace java org.apache.hive.service.cli.thrift namespace cpp apache.hive.service.cli.thrift diff --git a/common/thrift/parquet.thrift b/common/thrift/parquet.thrift index 443493518..675cf7261 100644 --- a/common/thrift/parquet.thrift +++ b/common/thrift/parquet.thrift @@ -20,6 +20,7 @@ /** * File format description for the parquet file format */ +namespace py impala_thrift_gen.parquet namespace cpp parquet namespace java org.apache.parquet.format diff --git a/infra/python/deps/py3-requirements.txt b/infra/python/deps/py3-requirements.txt index c21bb3e92..051226125 100644 --- a/infra/python/deps/py3-requirements.txt +++ b/infra/python/deps/py3-requirements.txt @@ -23,8 +23,9 @@ pylint == 2.10.2 wrapt == 1.12.1 typed-ast == 1.4.3 configparser == 4.0.2 - isort == 4.3.21 + isort == 5.13.2 futures == 3.3.0; python_version == "2.7" + poetry-core == 1.9.1 singledispatch == 3.6.1 toml == 0.10.2 platformdirs == 2.4.1 diff --git a/lib/python/impala_py_lib/profiles.py b/lib/python/impala_py_lib/profiles.py index 4d5ecb731..5425e06ef 100644 --- a/lib/python/impala_py_lib/profiles.py +++ b/lib/python/impala_py_lib/profiles.py @@ -22,9 +22,11 @@ from __future__ import absolute_import, division, print_function import base64 import datetime import zlib + from thrift.protocol import TCompactProtocol from thrift.TSerialization import deserialize -from RuntimeProfile.ttypes import TRuntimeProfileTree + +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileTree def decode_profile_line(line): diff --git a/shell/exec_summary.py b/shell/exec_summary.py index a7897315c..7431cc704 100755 --- a/shell/exec_summary.py +++ b/shell/exec_summary.py @@ -18,7 +18,9 @@ # specific language governing permissions and limitations # under the License. -from ExecStats.ttypes import TExecStats +from __future__ import absolute_import, print_function, unicode_literals + +from impala_thrift_gen.ExecStats.ttypes import TExecStats def build_exec_summary_table(summary, idx, indent_level, new_indent_level, output, diff --git a/shell/impala_client.py b/shell/impala_client.py index 3fc1891e1..96b2554c7 100755 --- a/shell/impala_client.py +++ b/shell/impala_client.py @@ -33,17 +33,19 @@ import traceback from datetime import datetime import uuid -from beeswaxd import BeeswaxService -from beeswaxd.BeeswaxService import QueryState -from ImpalaService import ImpalaService, ImpalaHiveServer2Service -from ImpalaService.ImpalaHiveServer2Service import (TGetRuntimeProfileReq, - TGetExecSummaryReq, TPingImpalaHS2ServiceReq, TCloseImpalaOperationReq) -from ErrorCodes.ttypes import TErrorCode -from Status.ttypes import TStatus -from TCLIService.TCLIService import (TExecuteStatementReq, TOpenSessionReq, - TCloseSessionReq, TProtocolVersion, TStatusCode, TGetOperationStatusReq, - TOperationState, TFetchResultsReq, TFetchOrientation, TGetLogReq, - TGetResultSetMetadataReq, TTypeId, TCancelOperationReq, TCloseOperationReq) +from impala_thrift_gen.beeswax import BeeswaxService +from impala_thrift_gen.beeswax.BeeswaxService import QueryState +from impala_thrift_gen.ImpalaService import ImpalaService, ImpalaHiveServer2Service +from impala_thrift_gen.ImpalaService.ImpalaHiveServer2Service import ( + TGetRuntimeProfileReq, TGetExecSummaryReq, TPingImpalaHS2ServiceReq, + TCloseImpalaOperationReq) +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode +from impala_thrift_gen.Status.ttypes import TStatus +from impala_thrift_gen.TCLIService.TCLIService import (TExecuteStatementReq, + TOpenSessionReq, TCloseSessionReq, TProtocolVersion, TStatusCode, + TGetOperationStatusReq, TOperationState, TFetchResultsReq, TFetchOrientation, + TGetLogReq, TGetResultSetMetadataReq, TTypeId, TCancelOperationReq, + TCloseOperationReq) from ImpalaHttpClient import ImpalaHttpClient from exec_summary import build_exec_summary_table from kerberos_util import get_kerb_host_from_kerberos_host_fqdn diff --git a/shell/value_converter.py b/shell/value_converter.py index 4c33fb004..4c8e5e0a7 100644 --- a/shell/value_converter.py +++ b/shell/value_converter.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from TCLIService.TCLIService import TTypeId +from impala_thrift_gen.TCLIService.TCLIService import TTypeId import sys diff --git a/testdata/bin/wait-for-hiveserver2.py b/testdata/bin/wait-for-hiveserver2.py index 37a2898f2..1b7344596 100755 --- a/testdata/bin/wait-for-hiveserver2.py +++ b/testdata/bin/wait-for-hiveserver2.py @@ -30,7 +30,7 @@ from optparse import OptionParser from tests.util.thrift_util import create_transport # Imports required for HiveServer2 Client -from TCLIService import TCLIService +from impala_thrift_gen.TCLIService import TCLIService from thrift.transport import TTransport, TSocket from thrift.protocol import TBinaryProtocol diff --git a/testdata/bin/wait-for-metastore.py b/testdata/bin/wait-for-metastore.py index ab898fa27..983c40763 100755 --- a/testdata/bin/wait-for-metastore.py +++ b/testdata/bin/wait-for-metastore.py @@ -28,7 +28,7 @@ from optparse import OptionParser from tests.util.thrift_util import create_transport # Imports required for Hive Metastore Client -from hive_metastore import ThriftHiveMetastore +from impala_thrift_gen.hive_metastore import ThriftHiveMetastore from thrift.transport import TTransport, TSocket from thrift.protocol import TBinaryProtocol diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py index 07ce8ad18..594e5ab6e 100644 --- a/tests/authorization/test_authorization.py +++ b/tests/authorization/test_authorization.py @@ -18,17 +18,18 @@ # Client tests for SQL statement authorization from __future__ import absolute_import, division, print_function -import pytest +from getpass import getuser import random import threading import time -from getpass import getuser -from ImpalaService import ImpalaHiveServer2Service -from TCLIService import TCLIService +import pytest +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.file_utils import assert_file_in_dir_contains from tests.common.test_result_verifier import error_msg_equal diff --git a/tests/authorization/test_authorized_proxy.py b/tests/authorization/test_authorized_proxy.py index 20005139b..5e624b3b8 100644 --- a/tests/authorization/test_authorized_proxy.py +++ b/tests/authorization/test_authorized_proxy.py @@ -16,16 +16,17 @@ # under the License. from __future__ import absolute_import, division, print_function -import pytest +import json import os import time -import json -from ImpalaService import ImpalaHiveServer2Service -from TCLIService import TCLIService +import pytest +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.hs2.hs2_test_suite import operation_id_to_query_id diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py index cd1f29bd5..2a7e5b8de 100644 --- a/tests/beeswax/impala_beeswax.py +++ b/tests/beeswax/impala_beeswax.py @@ -26,22 +26,23 @@ # result = client.execute(query_string) # where result is an object of the class ImpalaBeeswaxResult. from __future__ import absolute_import, division, print_function -from builtins import filter, map -import logging -import time -import shlex -import getpass -import re -import sys - -from beeswaxd import BeeswaxService -from beeswaxd.BeeswaxService import QueryState from datetime import datetime -from ImpalaService import ImpalaService -from tests.util.thrift_util import create_transport -from thrift.transport.TTransport import TTransportException +import getpass +import logging +import re +import shlex +import sys +import time + +from builtins import filter, map from thrift.protocol import TBinaryProtocol from thrift.Thrift import TApplicationException +from thrift.transport.TTransport import TTransportException + +from impala_thrift_gen.beeswax import BeeswaxService +from impala_thrift_gen.beeswax.BeeswaxService import QueryState +from impala_thrift_gen.ImpalaService import ImpalaService +from tests.util.thrift_util import create_transport LOG = logging.getLogger('impala_beeswax') # time to sleep in seconds before polling again. This uses a fixed diff --git a/tests/catalog_service/test_catalog_service_client.py b/tests/catalog_service/test_catalog_service_client.py index 22520c4c9..eaf244dc0 100644 --- a/tests/catalog_service/test_catalog_service_client.py +++ b/tests/catalog_service/test_catalog_service_client.py @@ -19,13 +19,13 @@ from __future__ import absolute_import, division, print_function import logging -import pytest -from CatalogService import CatalogService -from CatalogService.CatalogService import TGetFunctionsRequest -from ErrorCodes.ttypes import TErrorCode +import pytest from thrift.protocol import TBinaryProtocol +from impala_thrift_gen.CatalogService import CatalogService +from impala_thrift_gen.CatalogService.CatalogService import TGetFunctionsRequest +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfDockerizedCluster @@ -33,7 +33,6 @@ from tests.common.test_dimensions import create_single_exec_option_dimension from tests.util.filesystem_utils import WAREHOUSE from tests.util.thrift_util import create_transport - LOG = logging.getLogger('test_catalog_service_client') # TODO: Add a test that asserts correct/compatible responses diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py index 50cddfe4d..9ac6bafb0 100644 --- a/tests/common/impala_connection.py +++ b/tests/common/impala_connection.py @@ -21,29 +21,28 @@ from __future__ import absolute_import, division, print_function import abc -from future.utils import with_metaclass import getpass import logging import re import time -from beeswaxd.BeeswaxService import QueryState +from future.utils import with_metaclass import impala.dbapi as impyla import impala.error as impyla_error import impala.hiveserver2 as hs2 -import tests.common -from Query.ttypes import TQueryOptions -from RuntimeProfile.ttypes import TRuntimeProfileFormat + +from impala_thrift_gen.beeswax.BeeswaxService import QueryState +from impala_thrift_gen.Query.ttypes import TQueryOptions +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat from tests.beeswax.impala_beeswax import ( - DEFAULT_SLEEP_INTERVAL, - ImpalaBeeswaxClient, - ImpalaBeeswaxException) + DEFAULT_SLEEP_INTERVAL, + ImpalaBeeswaxClient, + ImpalaBeeswaxException, +) +import tests.common from tests.common.patterns import LOG_FORMAT from tests.common.test_vector import BEESWAX, HS2, HS2_HTTP -from tests.util.thrift_util import ( - op_handle_to_query_id, - session_handle_to_session_id) - +from tests.util.thrift_util import op_handle_to_query_id, session_handle_to_session_id LOG = logging.getLogger(__name__) console_handler = logging.StreamHandler() diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index fe4f4ac50..72126a39a 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -102,11 +102,11 @@ from tests.util.thrift_util import create_transport from tests.util.retry import retry # Imports required for Hive Metastore Client -from hive_metastore import ThriftHiveMetastore +from impala_thrift_gen.hive_metastore import ThriftHiveMetastore from thrift.protocol import TBinaryProtocol # Import to validate query option names -from ImpalaService.ttypes import TImpalaQueryOptions +from impala_thrift_gen.ImpalaService.ttypes import TImpalaQueryOptions # Initializing the logger before conditional imports, since we will need it # for them. diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index ad19590a5..5cee71bd3 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -18,50 +18,58 @@ # Tests admission control from __future__ import absolute_import, division, print_function -from builtins import int, range, round +from copy import deepcopy import itertools import logging import os -import pytest import re import signal import sys import threading -from copy import deepcopy from time import sleep, time +from builtins import int, range, round +import pytest + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.cluster_config import ( - impalad_admission_ctrl_flags, impalad_admission_ctrl_config_args, - RESOURCES_DIR) + impalad_admission_ctrl_flags, + RESOURCES_DIR, +) from tests.common.custom_cluster_test_suite import ( ADMISSIOND_ARGS, + CustomClusterTestSuite, IMPALAD_ARGS, START_ARGS, WORKLOAD_MGMT_IMPALAD_FLAGS, - CustomClusterTestSuite) +) from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties from tests.common.impala_connection import ( - RUNNING, FINISHED, ERROR, - IMPALA_CONNECTION_EXCEPTION) + ERROR, + FINISHED, + IMPALA_CONNECTION_EXCEPTION, + RUNNING, +) from tests.common.resource_pool_config import ResourcePoolConfig -from tests.common.skip import SkipIfFS, SkipIfEC, SkipIfNotHdfsMinicluster +from tests.common.skip import SkipIfEC, SkipIfFS, SkipIfNotHdfsMinicluster from tests.common.test_dimensions import ( - HS2, add_mandatory_exec_option, create_exec_option_dimension, create_single_exec_option_dimension, - create_uncompressed_text_dimension) + create_uncompressed_text_dimension, + HS2, +) from tests.common.test_vector import ImpalaTestDimension from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from tests.util.workload_management import QUERY_TBL_LIVE from tests.util.web_pages_util import ( + get_mem_admitted_backends_debug_page, get_num_completed_backends, - get_mem_admitted_backends_debug_page) +) +from tests.util.workload_management import QUERY_TBL_LIVE from tests.verifiers.mem_usage_verifier import MemUsageVerifier from tests.verifiers.metric_verifier import MetricVerifier -from ImpalaService import ImpalaHiveServer2Service -from TCLIService import TCLIService LOG = logging.getLogger('admission_test') diff --git a/tests/custom_cluster/test_custom_statestore.py b/tests/custom_cluster/test_custom_statestore.py index 8e14f6a82..4a701d8cd 100644 --- a/tests/custom_cluster/test_custom_statestore.py +++ b/tests/custom_cluster/test_custom_statestore.py @@ -19,26 +19,25 @@ # Tests statestore with non-default startup options from __future__ import absolute_import, division, print_function -from builtins import range import logging -import pytest -import uuid import socket - -from tests.common.custom_cluster_test_suite import CustomClusterTestSuite -from tests.common.environ import build_flavor_timeout -from tests.common.skip import SkipIfBuildType -from tests.common.patterns import print_id from time import sleep +import uuid -from Types.ttypes import TNetworkAddress +from builtins import range +import pytest from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -import StatestoreService.StatestoreSubscriber as Subscriber -import StatestoreService.StatestoreService as Statestore -import CatalogService.CatalogService as Catalog -from ErrorCodes.ttypes import TErrorCode +import impala_thrift_gen.CatalogService.CatalogService as Catalog +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode +import impala_thrift_gen.StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber +from impala_thrift_gen.Types.ttypes import TNetworkAddress +from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.environ import build_flavor_timeout +from tests.common.patterns import print_id +from tests.common.skip import SkipIfBuildType LOG = logging.getLogger('custom_statestore_test') STATESTORE_SERVICE_PORT = 24000 diff --git a/tests/custom_cluster/test_delegation.py b/tests/custom_cluster/test_delegation.py index 72660b394..3b3c988ab 100644 --- a/tests/custom_cluster/test_delegation.py +++ b/tests/custom_cluster/test_delegation.py @@ -17,10 +17,12 @@ # from __future__ import absolute_import, division, print_function import getpass + import pytest -from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from TCLIService import TCLIService + +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session USER_NAME = getpass.getuser() PROXY_USER = "proxy_user_name" diff --git a/tests/custom_cluster/test_event_processing_error.py b/tests/custom_cluster/test_event_processing_error.py index cd9a11547..7dfe1d195 100644 --- a/tests/custom_cluster/test_event_processing_error.py +++ b/tests/custom_cluster/test_event_processing_error.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. from __future__ import absolute_import, division, print_function + from builtins import range -from hive_metastore.ttypes import FireEventRequest -from hive_metastore.ttypes import FireEventRequestData +from impala_thrift_gen.hive_metastore.ttypes import FireEventRequest, FireEventRequestData from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.skip import SkipIfCatalogV2, SkipIfFS from tests.metadata.test_event_processing_base import TestEventProcessingBase diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py index fe728a747..1b6f29670 100644 --- a/tests/custom_cluster/test_events_custom_configs.py +++ b/tests/custom_cluster/test_events_custom_configs.py @@ -23,9 +23,9 @@ from os import getenv from time import sleep -from hive_metastore.ttypes import FireEventRequest -from hive_metastore.ttypes import FireEventRequestData -from hive_metastore.ttypes import InsertEventRequestData +from impala_thrift_gen.hive_metastore.ttypes import FireEventRequest +from impala_thrift_gen.hive_metastore.ttypes import FireEventRequestData +from impala_thrift_gen.hive_metastore.ttypes import InsertEventRequestData from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.impala_connection import ERROR, FINISHED from tests.common.impala_test_suite import ImpalaTestSuite diff --git a/tests/custom_cluster/test_hs2.py b/tests/custom_cluster/test_hs2.py index 7c64b885a..e7d24a6d3 100644 --- a/tests/custom_cluster/test_hs2.py +++ b/tests/custom_cluster/test_hs2.py @@ -16,13 +16,13 @@ # under the License. from __future__ import absolute_import, division, print_function -from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from time import sleep import pytest +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.hs2.hs2_test_suite import HS2TestSuite, operation_id_to_query_id -from time import sleep -from TCLIService import TCLIService class TestHS2(CustomClusterTestSuite): diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py index eadb58d24..f185db076 100644 --- a/tests/custom_cluster/test_metastore_service.py +++ b/tests/custom_cluster/test_metastore_service.py @@ -16,24 +16,26 @@ # under the License. from __future__ import absolute_import, division, print_function + from builtins import range import pytest -from hive_metastore.ttypes import Database -from hive_metastore.ttypes import FieldSchema -from hive_metastore.ttypes import FindNextCompactRequest -from hive_metastore.ttypes import GetTableRequest -from hive_metastore.ttypes import GetPartitionsByNamesRequest -from hive_metastore.ttypes import TruncateTableRequest -from hive_metastore.ttypes import Table -from hive_metastore.ttypes import StorageDescriptor -from hive_metastore.ttypes import SerDeInfo -from hive_metastore.ttypes import UpdateTransactionalStatsRequest -from hive_metastore.ttypes import WriteNotificationLogBatchRequest - -from tests.util.event_processor_utils import EventProcessorUtils +from impala_thrift_gen.hive_metastore.ttypes import ( + Database, + FieldSchema, + FindNextCompactRequest, + GetPartitionsByNamesRequest, + GetTableRequest, + SerDeInfo, + StorageDescriptor, + Table, + TruncateTableRequest, + UpdateTransactionalStatsRequest, + WriteNotificationLogBatchRequest, +) from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.impala_test_suite import ImpalaTestSuite +from tests.util.event_processor_utils import EventProcessorUtils from tests.util.filesystem_utils import IS_HDFS, IS_OZONE diff --git a/tests/custom_cluster/test_query_log.py b/tests/custom_cluster/test_query_log.py index 4095e0d7b..23513fbd4 100644 --- a/tests/custom_cluster/test_query_log.py +++ b/tests/custom_cluster/test_query_log.py @@ -16,19 +16,19 @@ # under the License. from __future__ import absolute_import, division, print_function - +from getpass import getuser import os +from random import choice, randint +from signal import SIGRTMIN import string from time import sleep, time -from getpass import getuser -from ImpalaService import ImpalaHiveServer2Service -from random import choice, randint -from signal import SIGRTMIN -from TCLIService import TCLIService +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.cluster_config import impalad_admission_ctrl_config_args from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.impala_connection import FINISHED @@ -39,9 +39,10 @@ from tests.common.wm_test_suite import WorkloadManagementTestSuite from tests.util.retry import retry from tests.util.workload_management import ( assert_query, - WM_DB, QUERY_TBL_LOG, - redaction_rules_file) + redaction_rules_file, + WM_DB, +) class TestQueryLogTableBasic(WorkloadManagementTestSuite): diff --git a/tests/custom_cluster/test_query_retries.py b/tests/custom_cluster/test_query_retries.py index 3357dfe5e..c94bc53b5 100644 --- a/tests/custom_cluster/test_query_retries.py +++ b/tests/custom_cluster/test_query_retries.py @@ -22,21 +22,29 @@ # TODO: Add a test that cancels queries while a retry is running from __future__ import absolute_import, division, print_function -from builtins import map, range -import pytest +from random import randint import re import time -from random import randint +from builtins import map, range +import pytest -from RuntimeProfile.ttypes import TRuntimeProfileFormat -from tests.common.impala_connection import ( - ERROR, FINISHED, IMPALA_CONNECTION_EXCEPTION, RUNNING) -from tests.common.impala_test_suite import ImpalaTestSuite, LOG +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.errors import Timeout -from tests.common.skip import (SkipIfEC, SkipIfBuildType, SkipIfFS, - SkipIfNotHdfsMinicluster) +from tests.common.impala_connection import ( + ERROR, + FINISHED, + IMPALA_CONNECTION_EXCEPTION, + RUNNING, +) +from tests.common.impala_test_suite import ImpalaTestSuite, LOG +from tests.common.skip import ( + SkipIfBuildType, + SkipIfEC, + SkipIfFS, + SkipIfNotHdfsMinicluster, +) from tests.common.test_dimensions import add_mandatory_exec_option # The BE krpc port of the impalad to simulate rpc or disk errors in tests. diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py index 2f798a5c8..f2b3e7a48 100644 --- a/tests/custom_cluster/test_restart_services.py +++ b/tests/custom_cluster/test_restart_services.py @@ -16,28 +16,31 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range import logging import os -import pytest -import psutil import re import signal import socket -import time -import threading - from subprocess import check_call -from tests.common.environ import build_flavor_timeout +import threading +import time from time import sleep +from builtins import range from impala.error import HiveServer2Error -from TCLIService import TCLIService +import psutil +import pytest +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.environ import build_flavor_timeout from tests.common.impala_connection import ( - ERROR, FINISHED, IMPALA_CONNECTION_EXCEPTION, RUNNING) -from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfFS + ERROR, + FINISHED, + IMPALA_CONNECTION_EXCEPTION, + RUNNING, +) +from tests.common.skip import SkipIfFS, SkipIfNotHdfsMinicluster from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session LOG = logging.getLogger(__name__) diff --git a/tests/custom_cluster/test_set_and_unset.py b/tests/custom_cluster/test_set_and_unset.py index 2abe2586b..52c128c19 100644 --- a/tests/custom_cluster/test_set_and_unset.py +++ b/tests/custom_cluster/test_set_and_unset.py @@ -16,12 +16,14 @@ # under the License. from __future__ import absolute_import, division, print_function + import pytest +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from TCLIService import TCLIService -from ImpalaService import ImpalaHiveServer2Service + class TestSetAndUnset(CustomClusterTestSuite, HS2TestSuite): """ diff --git a/tests/custom_cluster/test_statestored_ha.py b/tests/custom_cluster/test_statestored_ha.py index 665ba4d57..9f5f747c1 100644 --- a/tests/custom_cluster/test_statestored_ha.py +++ b/tests/custom_cluster/test_statestored_ha.py @@ -17,23 +17,23 @@ from __future__ import absolute_import, division, print_function import logging -import pytest import time - -from tests.common.custom_cluster_test_suite import CustomClusterTestSuite -from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties -from tests.common.impala_cluster import ( - DEFAULT_CATALOG_SERVICE_PORT, DEFAULT_STATESTORE_SERVICE_PORT) -from tests.common.impala_connection import ( - ERROR, IMPALA_CONNECTION_EXCEPTION, RUNNING) -from tests.common.skip import SkipIfBuildType, SkipIfNotHdfsMinicluster from time import sleep +import pytest from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -import StatestoreService.StatestoreSubscriber as Subscriber -import StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber +from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties +from tests.common.impala_cluster import ( + DEFAULT_CATALOG_SERVICE_PORT, + DEFAULT_STATESTORE_SERVICE_PORT, +) +from tests.common.impala_connection import ERROR, IMPALA_CONNECTION_EXCEPTION, RUNNING +from tests.common.skip import SkipIfBuildType, SkipIfNotHdfsMinicluster LOG = logging.getLogger('statestored_ha_test') diff --git a/tests/custom_cluster/test_workload_mgmt_init.py b/tests/custom_cluster/test_workload_mgmt_init.py index 7991dcd32..4191ff979 100644 --- a/tests/custom_cluster/test_workload_mgmt_init.py +++ b/tests/custom_cluster/test_workload_mgmt_init.py @@ -16,26 +16,26 @@ # under the License. from __future__ import absolute_import, division, print_function - +from logging import getLogger import os import re - from subprocess import CalledProcessError -from logging import getLogger -from SystemTables.ttypes import TQueryTableColumn +from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn from tests.common.custom_cluster_test_suite import ( + CustomClusterTestSuite, WORKLOAD_MGMT_IMPALAD_FLAGS, - CustomClusterTestSuite) +) from tests.common.test_dimensions import hs2_client_protocol_dimension from tests.common.test_vector import HS2 from tests.util.workload_management import ( assert_query, - WM_DB, - QUERY_TBL_LOG_NAME, - QUERY_TBL_LOG, + QUERY_TBL_LIVE, QUERY_TBL_LIVE_NAME, - QUERY_TBL_LIVE) + QUERY_TBL_LOG, + QUERY_TBL_LOG_NAME, + WM_DB, +) LOG = getLogger(__name__) QUERY_TBL_ALL = "{},{}".format(QUERY_TBL_LOG_NAME, QUERY_TBL_LIVE_NAME) diff --git a/tests/custom_cluster/test_workload_mgmt_sql_details.py b/tests/custom_cluster/test_workload_mgmt_sql_details.py index 7938fc03c..b2839d8d1 100644 --- a/tests/custom_cluster/test_workload_mgmt_sql_details.py +++ b/tests/custom_cluster/test_workload_mgmt_sql_details.py @@ -19,7 +19,7 @@ from __future__ import absolute_import, division, print_function import os -from SystemTables.ttypes import TQueryTableColumn +from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.test_dimensions import hs2_client_protocol_dimension from tests.common.wm_test_suite import WorkloadManagementTestSuite diff --git a/tests/hs2/hs2_test_suite.py b/tests/hs2/hs2_test_suite.py index a79743749..a1e50b162 100644 --- a/tests/hs2/hs2_test_suite.py +++ b/tests/hs2/hs2_test_suite.py @@ -18,17 +18,19 @@ # Superclass of all HS2 tests containing commonly used functions. from __future__ import absolute_import, division, print_function -from builtins import range from getpass import getuser -from TCLIService import TCLIService -from ImpalaService import ImpalaHiveServer2Service +import sys +from time import sleep, time + +from builtins import range +from thrift.protocol import TBinaryProtocol from thrift.transport.TSocket import TSocket from thrift.transport.TTransport import TBufferedTransport -from thrift.protocol import TBinaryProtocol -from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT, ImpalaTestSuite from tests.common.test_result_verifier import error_msg_startswith -from time import sleep, time -import sys def add_session_helper(self, protocol_version, conf_overlay, close_session, fn): diff --git a/tests/hs2/test_fetch.py b/tests/hs2/test_fetch.py index 4392ed5eb..7b40ef502 100644 --- a/tests/hs2/test_fetch.py +++ b/tests/hs2/test_fetch.py @@ -17,13 +17,17 @@ # from __future__ import absolute_import, division, print_function -import pytest import re -from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session, - create_op_handle_without_secret) -from TCLIService import TCLIService, constants -from TCLIService.ttypes import TTypeId +import pytest + +from impala_thrift_gen.TCLIService import constants, TCLIService +from impala_thrift_gen.TCLIService.ttypes import TTypeId +from tests.hs2.hs2_test_suite import ( + create_op_handle_without_secret, + HS2TestSuite, + needs_session, +) # Simple test to make sure all the HS2 types are supported for both the row and diff --git a/tests/hs2/test_fetch_first.py b/tests/hs2/test_fetch_first.py index 93f428a9b..c597538cc 100644 --- a/tests/hs2/test_fetch_first.py +++ b/tests/hs2/test_fetch_first.py @@ -21,13 +21,14 @@ # succeed as long all previously fetched rows fit into the bounded result cache. from __future__ import absolute_import, division, print_function + from builtins import range import pytest -from ImpalaService import ImpalaHiveServer2Service -from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session -from TCLIService import TCLIService +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.impala_cluster import ImpalaCluster +from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session class TestFetchFirst(HS2TestSuite): diff --git a/tests/hs2/test_fetch_timeout.py b/tests/hs2/test_fetch_timeout.py index ea0ec5929..148ebf31d 100644 --- a/tests/hs2/test_fetch_timeout.py +++ b/tests/hs2/test_fetch_timeout.py @@ -16,12 +16,11 @@ # under the License. from __future__ import absolute_import, division, print_function -from time import sleep -from time import time -from tests.common.errors import Timeout -from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session) -from TCLIService import TCLIService +from time import sleep, time +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.errors import Timeout +from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session # Tests for the query option FETCH_ROWS_TIMEOUT_MS, which is the maximum amount of # time, in milliseconds, a fetch rows request (TFetchResultsReq) from the client should diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py index 672cb4f52..c5206346b 100644 --- a/tests/hs2/test_hs2.py +++ b/tests/hs2/test_hs2.py @@ -18,33 +18,37 @@ # Client tests for Impala's HiveServer2 interface from __future__ import absolute_import, division, print_function -from builtins import range -from getpass import getuser from contextlib import contextmanager +from getpass import getuser import json import logging -import pytest import random import threading import time import uuid -import impala.dbapi as impyla +from builtins import range +import impala.dbapi as impyla +import pytest + +from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service +from impala_thrift_gen.TCLIService import TCLIService from tests.common.impala_test_suite import IMPALAD_HOSTNAME, IMPALAD_HS2_HTTP_PORT +from tests.common.skip import SkipIfDockerizedCluster +from tests.hs2.hs2_test_suite import ( + create_op_handle_without_secret, + create_session_handle_without_secret, + HS2TestSuite, + needs_session, + needs_session_cluster_properties, + operation_id_to_query_id, +) try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen -from ImpalaService import ImpalaHiveServer2Service -from tests.common.environ import ImpalaTestClusterProperties -from tests.common.skip import SkipIfDockerizedCluster -from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session, - operation_id_to_query_id, create_session_handle_without_secret, - create_op_handle_without_secret, needs_session_cluster_properties) -from TCLIService import TCLIService - LOG = logging.getLogger('test_hs2') SQLSTATE_GENERAL_ERROR = "HY000" diff --git a/tests/hs2/test_json_endpoints.py b/tests/hs2/test_json_endpoints.py index b531fcc45..30c0c1fff 100644 --- a/tests/hs2/test_json_endpoints.py +++ b/tests/hs2/test_json_endpoints.py @@ -19,18 +19,20 @@ from __future__ import absolute_import, division, print_function import json -import pytest from time import time +import pytest + +from impala_thrift_gen.TCLIService import TCLIService +from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER +from tests.common.impala_cluster import ImpalaCluster +from tests.hs2.hs2_test_suite import HS2TestSuite + try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen -from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER -from tests.common.impala_cluster import ImpalaCluster -from tests.hs2.hs2_test_suite import HS2TestSuite -from TCLIService import TCLIService class TestJsonEndpoints(HS2TestSuite): def _get_json_queries(self, http_addr): diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py index 62b91c587..41763e5fa 100644 --- a/tests/metadata/test_compute_stats.py +++ b/tests/metadata/test_compute_stats.py @@ -16,23 +16,28 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range import os -import pytest -from hive_metastore.ttypes import ( - ColumnStatistics, ColumnStatisticsDesc, ColumnStatisticsData, - ColumnStatisticsObj, StringColumnStatsData) +from builtins import range +import pytest + +from impala_thrift_gen.CatalogObjects.ttypes import THdfsCompression +from impala_thrift_gen.hive_metastore.ttypes import ( + ColumnStatistics, + ColumnStatisticsData, + ColumnStatisticsDesc, + ColumnStatisticsObj, + StringColumnStatsData, +) from tests.common.environ import ImpalaTestClusterProperties from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfCatalogV2 +from tests.common.skip import SkipIfCatalogV2, SkipIfFS, SkipIfLocal from tests.common.test_dimensions import ( create_exec_option_dimension, create_single_exec_option_dimension, - create_uncompressed_text_dimension) -from CatalogObjects.ttypes import THdfsCompression - + create_uncompressed_text_dimension, +) IMPALA_TEST_CLUSTER_PROPERTIES = ImpalaTestClusterProperties.get_instance() diff --git a/tests/query_test/test_acid.py b/tests/query_test/test_acid.py index a25cb3867..277bd2da1 100644 --- a/tests/query_test/test_acid.py +++ b/tests/query_test/test_acid.py @@ -19,13 +19,18 @@ from __future__ import absolute_import, division, print_function import os -import pytest +from subprocess import check_call import time -from hive_metastore.ttypes import CommitTxnRequest, LockType, OpenTxnRequest -from subprocess import check_call +import pytest + +from impala_thrift_gen.hive_metastore.ttypes import ( + CommitTxnRequest, + LockType, + OpenTxnRequest, +) from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.skip import SkipIf, SkipIfHive2, SkipIfCatalogV2, SkipIfFS +from tests.common.skip import SkipIf, SkipIfCatalogV2, SkipIfFS, SkipIfHive2 from tests.common.test_dimensions import create_single_exec_option_dimension from tests.util.acid_txn import AcidTxn diff --git a/tests/query_test/test_cancellation.py b/tests/query_test/test_cancellation.py index f6ce4c75b..efda79e8b 100644 --- a/tests/query_test/test_cancellation.py +++ b/tests/query_test/test_cancellation.py @@ -19,18 +19,19 @@ # from __future__ import absolute_import, division, print_function -from builtins import range -import pytest from threading import Thread from time import sleep -from RuntimeProfile.ttypes import TRuntimeProfileFormat + +from builtins import range +import pytest + +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat +from tests.common.impala_connection import MinimalHS2Connection +from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT, ImpalaTestSuite from tests.common.test_dimensions import add_mandatory_exec_option from tests.common.test_vector import ImpalaTestDimension -from tests.common.impala_test_suite import ImpalaTestSuite -from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT from tests.util.cancel_util import cancel_query_and_validate_state from tests.verifiers.metric_verifier import MetricVerifier -from tests.common.impala_connection import MinimalHS2Connection # PRIMARY KEY for lineitem LINEITEM_PK = 'l_orderkey, l_partkey, l_suppkey, l_linenumber' diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py index ad13b1dc0..abc92996b 100644 --- a/tests/query_test/test_iceberg.py +++ b/tests/query_test/test_iceberg.py @@ -16,37 +16,37 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range from collections import defaultdict, namedtuple import datetime +import json import logging import os -import pytest -import pytz import random import re -import time - from subprocess import check_call, check_output -# noinspection PyUnresolvedReferences -from parquet.ttypes import ConvertedType +import time from avro.datafile import DataFileReader from avro.io import DatumReader -import json +from builtins import range +import pytest +import pytz -from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION +# noinspection PyUnresolvedReferences +from impala_thrift_gen.parquet.ttypes import ConvertedType +from tests.common.file_utils import ( + create_iceberg_table_from_directory, + create_table_from_parquet, +) from tests.common.iceberg_test_suite import IcebergTestSuite -from tests.common.skip import SkipIf, SkipIfFS, SkipIfDockerizedCluster +from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION +from tests.common.skip import SkipIf, SkipIfDockerizedCluster, SkipIfFS from tests.common.test_dimensions import add_exec_option_dimension from tests.common.test_result_verifier import error_msg_startswith -from tests.common.file_utils import ( - create_iceberg_table_from_directory, - create_table_from_parquet) from tests.shell.util import run_impala_shell_cmd -from tests.util.filesystem_utils import get_fs_path, IS_HDFS, WAREHOUSE, FILESYSTEM_PREFIX +from tests.util.filesystem_utils import FILESYSTEM_PREFIX, get_fs_path, IS_HDFS, WAREHOUSE from tests.util.get_parquet_metadata import get_parquet_metadata -from tests.util.iceberg_util import cast_ts, quote, get_snapshots, IcebergCatalogs +from tests.util.iceberg_util import cast_ts, get_snapshots, IcebergCatalogs, quote LOG = logging.getLogger(__name__) diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py index a90f456f7..36399feb7 100644 --- a/tests/query_test/test_insert_parquet.py +++ b/tests/query_test/test_insert_parquet.py @@ -18,27 +18,35 @@ # Targeted Impala insert tests from __future__ import absolute_import, division, print_function -from builtins import map, range, round -import os - from collections import namedtuple -from datetime import datetime, date +from datetime import date, datetime from decimal import Decimal +import os from subprocess import check_call -from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder, ConvertedType +from builtins import map, range, round + +from impala_thrift_gen.parquet.ttypes import ( + ColumnOrder, + ConvertedType, + SortingColumn, + TypeDefinedOrder, +) from tests.common.environ import impalad_basedir from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.parametrize import UniqueDatabase from tests.common.skip import SkipIfFS, SkipIfLocal from tests.common.test_dimensions import ( add_exec_option_dimension, - create_exec_option_dimension) + create_exec_option_dimension, +) from tests.common.test_result_verifier import verify_query_result_is_equal from tests.common.test_vector import ImpalaTestDimension from tests.util.filesystem_utils import get_fs_path, WAREHOUSE -from tests.util.get_parquet_metadata import (decode_stats_value, - get_parquet_metadata_from_hdfs_folder) +from tests.util.get_parquet_metadata import ( + decode_stats_value, + get_parquet_metadata_from_hdfs_folder, +) PARQUET_CODECS = ['none', 'snappy', 'gzip', 'zstd', 'lz4'] IMPALA_HOME = os.environ['IMPALA_HOME'] diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py index 5394e22f5..cfdad05ec 100644 --- a/tests/query_test/test_observability.py +++ b/tests/query_test/test_observability.py @@ -18,6 +18,12 @@ from __future__ import absolute_import, division, print_function from collections import defaultdict from datetime import datetime +import re +from time import sleep, time + +import pytest + +from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION from tests.common.impala_test_suite import ImpalaTestSuite @@ -25,10 +31,6 @@ from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfNotHdfsMinicluster from tests.common.test_vector import HS2 from tests.util.filesystem_utils import WAREHOUSE from tests.util.parse_util import get_duration_us_from_str -from time import sleep, time -from RuntimeProfile.ttypes import TRuntimeProfileFormat -import pytest -import re class TestObservability(ImpalaTestSuite): diff --git a/tests/query_test/test_parquet_bloom_filter.py b/tests/query_test/test_parquet_bloom_filter.py index 1db10efe5..112fba2b3 100644 --- a/tests/query_test/test_parquet_bloom_filter.py +++ b/tests/query_test/test_parquet_bloom_filter.py @@ -16,21 +16,18 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range +from collections import namedtuple import math import os - -from collections import namedtuple -from parquet.ttypes import BloomFilterHeader from subprocess import check_call +from builtins import range + +from impala_thrift_gen.parquet.ttypes import BloomFilterHeader from tests.common.file_utils import create_table_and_copy_files from tests.common.impala_test_suite import ImpalaTestSuite from tests.util.filesystem_utils import get_fs_path -from tests.util.get_parquet_metadata import ( - get_parquet_metadata, - read_serialized_object -) +from tests.util.get_parquet_metadata import get_parquet_metadata, read_serialized_object class TestParquetBloomFilter(ImpalaTestSuite): diff --git a/tests/query_test/test_parquet_page_index.py b/tests/query_test/test_parquet_page_index.py index 1d0f1f6b4..e1f735fd0 100644 --- a/tests/query_test/test_parquet_page_index.py +++ b/tests/query_test/test_parquet_page_index.py @@ -18,21 +18,26 @@ # Targeted Impala insert tests from __future__ import absolute_import, division, print_function +from collections import namedtuple import os import random import string - -from collections import namedtuple from subprocess import check_call -from parquet.ttypes import BoundaryOrder, ColumnIndex, OffsetIndex, PageHeader, PageType +from impala_thrift_gen.parquet.ttypes import ( + BoundaryOrder, + ColumnIndex, + OffsetIndex, + PageHeader, + PageType, +) from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfLocal from tests.util.filesystem_utils import get_fs_path from tests.util.get_parquet_metadata import ( decode_stats_value, get_parquet_metadata, - read_serialized_object + read_serialized_object, ) PAGE_INDEX_MAX_STRING_LENGTH = 64 diff --git a/tests/query_test/test_query_opts.py b/tests/query_test/test_query_opts.py index 71a0b73f2..4d81ecfec 100644 --- a/tests/query_test/test_query_opts.py +++ b/tests/query_test/test_query_opts.py @@ -21,12 +21,13 @@ # timeout). from __future__ import absolute_import, division, print_function -from TCLIService import TCLIService +from impala_thrift_gen.TCLIService import TCLIService from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.test_dimensions import create_exec_option_dimension from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session + class TestQueryOptions(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py index ad4bdbe15..410980d8f 100644 --- a/tests/query_test/test_scanners.py +++ b/tests/query_test/test_scanners.py @@ -22,17 +22,19 @@ # explode. from __future__ import absolute_import, division, print_function -from builtins import range +from copy import deepcopy import os -import pytest import random import re -import tempfile -from copy import deepcopy -from parquet.ttypes import ConvertedType from subprocess import check_call +import tempfile +from builtins import range +import pytest + +from impala_thrift_gen.parquet.ttypes import ConvertedType from testdata.common import widetable +from tests.common.file_utils import create_table_and_copy_files, create_table_from_parquet from tests.common.impala_test_suite import ImpalaTestSuite, LOG from tests.common.skip import ( SkipIf, @@ -41,21 +43,18 @@ from tests.common.skip import ( SkipIfHive2, SkipIfHive3, SkipIfLocal, - SkipIfNotHdfsMinicluster) + SkipIfNotHdfsMinicluster, +) from tests.common.test_dimensions import ( add_exec_option_dimension, add_mandatory_exec_option, - create_single_exec_option_dimension, create_exec_option_dimension, - create_uncompressed_text_dimension) -from tests.common.file_utils import ( - create_table_from_parquet, - create_table_and_copy_files) -from tests.common.test_result_verifier import ( - QueryTestResult, - parse_result_rows) + create_single_exec_option_dimension, + create_uncompressed_text_dimension, +) +from tests.common.test_result_verifier import parse_result_rows, QueryTestResult from tests.common.test_vector import ImpalaTestDimension -from tests.util.filesystem_utils import IS_HDFS, get_fs_path +from tests.util.filesystem_utils import get_fs_path, IS_HDFS from tests.util.get_parquet_metadata import get_parquet_metadata from tests.util.parse_util import get_bytes_summary_stats_counter from tests.util.test_file_parser import QueryTestSectionReader diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py index b7a6b28a3..e6417721a 100644 --- a/tests/statestore/test_statestore.py +++ b/tests/statestore/test_statestore.py @@ -16,38 +16,38 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import range from collections import defaultdict import json import logging import socket import threading -import traceback import time +import traceback import uuid +from builtins import range +from thrift.protocol import TBinaryProtocol +from thrift.server.TServer import TServer +from thrift.transport import TSocket, TTransport + +from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode +import impala_thrift_gen.StatestoreService.StatestoreService as Statestore +import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber +from impala_thrift_gen.StatestoreService.StatestoreSubscriber import ( + TTopicRegistration, + TUpdateStateResponse, +) +from impala_thrift_gen.Status.ttypes import TStatus +from impala_thrift_gen.Types.ttypes import TNetworkAddress +from tests.common.base_test_suite import BaseTestSuite +from tests.common.environ import build_flavor_timeout +from tests.common.skip import SkipIfDockerizedCluster + try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen -from Types.ttypes import TNetworkAddress -from thrift.protocol import TBinaryProtocol -from thrift.server.TServer import TServer -from thrift.transport import TSocket -from thrift.transport import TTransport - -import StatestoreService.StatestoreSubscriber as Subscriber -import StatestoreService.StatestoreService as Statestore -from StatestoreService.StatestoreSubscriber import TUpdateStateResponse -from StatestoreService.StatestoreSubscriber import TTopicRegistration -from ErrorCodes.ttypes import TErrorCode -from Status.ttypes import TStatus - -from tests.common.base_test_suite import BaseTestSuite -from tests.common.environ import build_flavor_timeout -from tests.common.skip import SkipIfDockerizedCluster - LOG = logging.getLogger('test_statestore') # Tests for the statestore. The StatestoreSubscriber class is a skeleton implementation of diff --git a/tests/util/acid_txn.py b/tests/util/acid_txn.py index f1830f2db..3e28598a7 100644 --- a/tests/util/acid_txn.py +++ b/tests/util/acid_txn.py @@ -16,14 +16,28 @@ # under the License. from __future__ import absolute_import, division, print_function -from tests.util.thrift_util import create_transport -from hive_metastore import ThriftHiveMetastore -from hive_metastore.ttypes import (AbortTxnRequest, AllocateTableWriteIdsRequest, - CheckLockRequest, CommitTxnRequest, GetValidWriteIdsRequest, HeartbeatRequest, - LockComponent, LockLevel, LockType, LockRequest, OpenTxnRequest, ShowLocksRequest, - TruncateTableRequest, UnlockRequest) + from thrift.protocol import TBinaryProtocol +from impala_thrift_gen.hive_metastore import ThriftHiveMetastore +from impala_thrift_gen.hive_metastore.ttypes import ( + AbortTxnRequest, + AllocateTableWriteIdsRequest, + CheckLockRequest, + CommitTxnRequest, + GetValidWriteIdsRequest, + HeartbeatRequest, + LockComponent, + LockLevel, + LockRequest, + LockType, + OpenTxnRequest, + ShowLocksRequest, + TruncateTableRequest, + UnlockRequest, +) +from tests.util.thrift_util import create_transport + # HMS config metastore_host = "localhost" metastore_port = "9083" diff --git a/tests/util/event_processor_utils.py b/tests/util/event_processor_utils.py index 1b1a23c8e..a270f3eb3 100644 --- a/tests/util/event_processor_utils.py +++ b/tests/util/event_processor_utils.py @@ -20,14 +20,15 @@ # succeeded by querying Impala, or vice versa. from __future__ import absolute_import, division, print_function -import logging -import requests -import time import json -from hive_metastore.ttypes import NotificationEventRequest +import logging +import time -from tests.common.impala_cluster import ImpalaCluster +import requests + +from impala_thrift_gen.hive_metastore.ttypes import NotificationEventRequest from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +from tests.common.impala_cluster import ImpalaCluster LOG = logging.getLogger('event_processor_utils') LOG.setLevel(level=logging.DEBUG) diff --git a/tests/util/get_parquet_metadata.py b/tests/util/get_parquet_metadata.py index 876ffff08..b25f08ede 100644 --- a/tests/util/get_parquet_metadata.py +++ b/tests/util/get_parquet_metadata.py @@ -16,19 +16,20 @@ # under the License. from __future__ import absolute_import, division, print_function -from builtins import map -import os -import struct -import sys - from datetime import date, datetime, time, timedelta from decimal import Decimal from functools import reduce -from parquet.ttypes import ColumnIndex, FileMetaData, OffsetIndex, PageHeader, Type +import os +import struct from subprocess import check_call +import sys + +from builtins import map from thrift.protocol import TCompactProtocol from thrift.transport import TTransport +from impala_thrift_gen.parquet.ttypes import FileMetaData, Type + PARQUET_VERSION_NUMBER = b'PAR1' diff --git a/tests/util/parse_util.py b/tests/util/parse_util.py index 0d2156f51..370ac1d03 100644 --- a/tests/util/parse_util.py +++ b/tests/util/parse_util.py @@ -203,7 +203,7 @@ def get_bytes_summary_stats_counter(counter_name, runtime_profile): """ # This requires the Thrift definitions to be generated. We limit the scope of the import # to allow tools like the stress test to import this file without building Impala. - from RuntimeProfile.ttypes import TSummaryStatsCounter + from impala_thrift_gen.RuntimeProfile.ttypes import TSummaryStatsCounter regex_summary_stat = re.compile(r"""\( Avg:[^\(]*\((?P[0-9]+)\)\s;\s # Matches Avg: [?].[?] [?]B (?) @@ -256,7 +256,7 @@ def get_time_summary_stats_counter(counter_name, runtime_profile): """ # This requires the Thrift definitions to be generated. We limit the scope of the import # to allow tools like the stress test to import this file without building Impala. - from RuntimeProfile.ttypes import TSummaryStatsCounter + from impala_thrift_gen.RuntimeProfile.ttypes import TSummaryStatsCounter regex_summary_stat = re.compile(r"""\( Avg:\s(?P.*)\s;\s # Matches Avg: ? ; diff --git a/tests/util/workload_management.py b/tests/util/workload_management.py index 5053a2a2c..e98249efa 100644 --- a/tests/util/workload_management.py +++ b/tests/util/workload_management.py @@ -24,7 +24,7 @@ import requests from datetime import datetime from time import sleep, time -from SystemTables.ttypes import TQueryTableColumn +from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn from tests.util.assert_time import assert_time_str, convert_to_milliseconds from tests.util.memory import assert_byte_str, convert_to_bytes