IMPALA-11980 (part 1): Put all thrift-generated python code into the impala_thrift_gen package

This puts all of the thrift-generated python code into the
impala_thrift_gen package. This is similar to what Impyla
does for its thrift-generated python code, except that it
uses the impala_thrift_gen package rather than impala._thrift_gen.
This is a preparatory patch for fixing the absolute import
issues.

This patches all of the thrift files to add the python namespace.
This has code to apply the patching to the thirdparty thrift
files (hive_metastore.thrift, fb303.thrift) to do the same.

Putting all the generated python into a package makes it easier
to understand where the imports are getting code. When the
subsequent change rearranges the shell code, the thrift generated
code can stay in a separate directory.

This uses isort to sort the imports for the affected Python files
with the provided .isort.cfg file. This also adds an impala-isort
shell script to make it easy to run.

Testing:
 - Ran a core job

Change-Id: Ie2927f22c7257aa38a78084efe5bd76d566493c0
Reviewed-on: http://gerrit.cloudera.org:8080/20169
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Riza Suminto <riza.suminto@cloudera.com>

.isort.cfg

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[settings]
+force_alphabetical_sort_within_sections=true
+force_sort_within_sections=true
+include_trailing_comma=true
+known_third_party=builtins
+known_first_party=impala_py_lib,impala_shell,impala_thrift_gen,tests
+line_length=90
+multi_line_output=3
+no_lines_before=STDLIB
+order_by_type=false
+skip_gitignore=true
+skip_glob=testdata/cluster/cdh*

bin/cmake_aux/add_thrift_python_namespace.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# add_thrift_python_namespace.sh IN_THRIFT_FILE OUT_THRIFT_FILE
+#
+# This script reads $IN_THRIFT_FILE and adds a python namespace
+# (replacing any existing python namespace) with
+# impala_thrift_gen.${BASE_NAME} where the BASE_NAME is the
+# thrift filename without the ".thrift". i.e. Foo.thrift uses
+# impala_thrift_gen.Foo python namespace. It writes the resulting
+# thrift file to $OUT_THRIFT_FILE.
+#
+# This logic is taken from Impyla's impala/thrift/process_thrift.sh
+# script with minor changes. This requires that the source thrift
+# file have at least one preexisting non-python namespace. That is
+# true for all of the Thrift files that we care about.
+
+set -eou pipefail
+
+THRIFT_FILE_IN=$1
+THRIFT_FILE_OUT=$2
+
+FILE_NAME=$(basename $THRIFT_FILE_IN)
+BASE_NAME=${FILE_NAME%.*}
+# Awk script to add the python namespace before the first namespace
+# in the thrift file.
+ADD_NAMESPACE_PY="
+    BEGIN {
+        n = 0
+    }
+    {
+        if (\$0 ~ /^namespace/ && n == 0) {
+            print \"namespace py impala_thrift_gen.$BASE_NAME\";
+            n += 1;
+        }
+        print \$0;
+    }"
+
+# Remove any existing python namespace, then add our namespace
+cat $THRIFT_FILE_IN | grep -v "^namespace py" | awk "$ADD_NAMESPACE_PY" > $THRIFT_FILE_OUT

bin/impala-isort

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source "$(dirname "$0")/impala-python3-common.sh"
+exec "$PY_ENV_DIR/bin/isort" "$@"

common/thrift/.gitignore

@@ -3,3 +3,4 @@ ErrorCodes.thrift
 MetricDefs.thrift
 hive-2-api/TCLIService.thrift
 hive-3-api/TCLIService.thrift
+/thirdparty_thrift/

common/thrift/BackendGflags.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.BackendGflags
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/CMakeLists.txt

@@ -135,18 +135,19 @@ function(THRIFT_GEN_DS VAR)
   set(${VAR} ${${VAR}} PARENT_SCOPE)
 endfunction(THRIFT_GEN_DS)
 
+set(THIRDPARTY_THRIFT_DIR "thirdparty_thrift")
 set(HIVE_THRIFT_SOURCE_DIR "hive-$ENV{IMPALA_HIVE_MAJOR_VERSION}-api")
 set(TCLI_SERVICE_THRIFT "${HIVE_THRIFT_SOURCE_DIR}/TCLIService.thrift")
 message("Using Thrift CPP compiler: ${THRIFT_CPP_COMPILER}")
 message("Using Thrift JAVA compiler: ${THRIFT_JAVA_COMPILER}")
 message("Using Thrift PY compiler: ${THRIFT_PY_COMPILER}")
 set(THRIFT_QUIET_WRAPPER "${CMAKE_SOURCE_DIR}/bin/thrift-quiet-wrapper.sh")
-set(THRIFT_CPP_INCLUDE_DIR_OPTION -I ${THRIFT_CPP_CONTRIB_DIR}
-  -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR})
-set(THRIFT_JAVA_INCLUDE_DIR_OPTION -I ${THRIFT_JAVA_CONTRIB_DIR}
-  -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR})
-set(THRIFT_PY_INCLUDE_DIR_OPTION -I ${THRIFT_PY_CONTRIB_DIR}
-  -I $ENV{HIVE_METASTORE_THRIFT_DIR} -I ${HIVE_THRIFT_SOURCE_DIR})
+set(THRIFT_CPP_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR}
+  -I ${HIVE_THRIFT_SOURCE_DIR})
+set(THRIFT_JAVA_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR}
+  -I ${HIVE_THRIFT_SOURCE_DIR})
+set(THRIFT_PY_INCLUDE_DIR_OPTION -I ${THIRDPARTY_THRIFT_DIR}
+  -I ${HIVE_THRIFT_SOURCE_DIR})
 set(BE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/be/generated-sources)
 set(FE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/fe/generated-sources)
 # TODO: avoid duplicating generated java classes
@@ -158,12 +159,13 @@ file(MAKE_DIRECTORY ${FE_OUTPUT_DIR})
 file(MAKE_DIRECTORY ${EXT_DS_OUTPUT_DIR})
 file(MAKE_DIRECTORY ${PYTHON_OUTPUT_DIR})
 file(MAKE_DIRECTORY ${HIVE_THRIFT_SOURCE_DIR})
+file(MAKE_DIRECTORY ${THIRDPARTY_THRIFT_DIR})
 
 # Args passed to thrift for Java gen
 set(JAVA_FE_ARGS ${THRIFT_JAVA_INCLUDE_DIR_OPTION} --gen java -o ${FE_OUTPUT_DIR})
 set(JAVA_EXT_DS_ARGS ${THRIFT_JAVA_INCLUDE_DIR_OPTION} --gen java -o ${EXT_DS_OUTPUT_DIR})
 set(PYTHON_ARGS ${THRIFT_PY_INCLUDE_DIR_OPTION} -r --gen py:no_utf8strings -o
-                ${PYTHON_OUTPUT_DIR})
+    ${PYTHON_OUTPUT_DIR})
 
 set (EXT_DATA_SRC_FILES
   ErrorCodes.thrift
@@ -211,6 +213,16 @@ set (SRC_FILES
 )
 
 SET_SOURCE_FILES_PROPERTIES(Status.thrift PROPERTIES OBJECT_DEPENDS ErrorCodes.thrift)
+SET_SOURCE_FILES_PROPERTIES(CatalogObjects.thrift PROPERTIES OBJECT_DEPENDS
+    ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift)
+SET_SOURCE_FILES_PROPERTIES(CatalogService.thrift PROPERTIES OBJECT_DEPENDS
+    ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift)
+SET_SOURCE_FILES_PROPERTIES(JniCatalog.thrift PROPERTIES OBJECT_DEPENDS
+    ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift)
+SET_SOURCE_FILES_PROPERTIES(SqlConstraints.thrift PROPERTIES OBJECT_DEPENDS
+    ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift)
+SET_SOURCE_FILES_PROPERTIES(beeswax.thrift PROPERTIES OBJECT_DEPENDS
+    ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift)
 
 add_custom_command(OUTPUT ErrorCodes.thrift
   COMMAND python generate_error_codes.py
@@ -232,6 +244,33 @@ add_custom_command(OUTPUT hive-$ENV{IMPALA_HIVE_MAJOR_VERSION}-api/TCLIService.t
   DEPENDS hive-1-api/TCLIService.thrift
 )
 
+# This generates hive_metastore.thrift in the $THIRDPARTY_THRIFT_DIR. The two
+# modification are:
+# 1. Set the impala_thrift_gen python namespace
+# 2. Rearranges the fb303 reference so that it doesn't have the share/fb303/if
+#    directory structure
+add_custom_command(OUTPUT ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift
+  COMMAND ${CMAKE_SOURCE_DIR}/bin/cmake_aux/add_thrift_python_namespace.sh
+      $ENV{HIVE_METASTORE_THRIFT_DIR}/hive_metastore.thrift
+      ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp
+  COMMAND cat ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp |
+      sed 's|share/fb303/if/||' > ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift
+  COMMAND rm ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift.tmp
+  DEPENDS $ENV{HIVE_METASTORE_THRIFT_DIR}/hive_metastore.thrift
+)
+
+# Generate fb303.thrift in the $THIRDPARTY_THRIFT_DIR with the appropriate
+# impala_thrift_gen python namespace.
+add_custom_command(OUTPUT ${THIRDPARTY_THRIFT_DIR}/fb303.thrift
+  COMMAND ${CMAKE_SOURCE_DIR}/bin/cmake_aux/add_thrift_python_namespace.sh
+      ${THRIFT_PY_CONTRIB_DIR}/share/fb303/if/fb303.thrift
+      ${THIRDPARTY_THRIFT_DIR}/fb303.thrift
+  DEPENDS ${THRIFT_PY_CONTRIB_DIR}/share/fb303/if/fb303.thrift
+)
+
+SET_SOURCE_FILES_PROPERTIES(${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift
+    PROPERTIES OBJECT_DEPENDS ${THIRDPARTY_THRIFT_DIR}/fb303.thrift)
+
 # Create a build command for each of the thrift src files and generate
 # a list of files they produce
 THRIFT_GEN(THRIFT_ALL_FILES ${SRC_FILES})
@@ -240,11 +279,15 @@ THRIFT_GEN_DS(THRIFT_DATA_SRC_FILES ${EXT_DATA_SRC_FILES})
 add_custom_target(thrift-generated-files-error DEPENDS ErrorCodes.thrift)
 add_custom_target(thrift-generated-files-metrics DEPENDS MetricDefs.thrift)
 add_custom_target(thrift-generated-files-tcli-service DEPENDS ${TCLI_SERVICE_THRIFT})
+add_custom_target(thrift-generated-files-hive-metastore
+  DEPENDS ${THIRDPARTY_THRIFT_DIR}/hive_metastore.thrift)
+add_custom_target(thrift-generated-files-fb303 DEPENDS ${THIRDPARTY_THRIFT_DIR}/fb303.thrift)
 
 # Add a custom target that generates all the thrift files
 add_custom_target(thrift-cpp ALL DEPENDS ${THRIFT_ALL_FILES})
 add_dependencies(thrift-cpp thrift-generated-files-metrics thrift-generated-files-error
-  thrift-generated-files-tcli-service)
+  thrift-generated-files-tcli-service thrift-generated-files-hive-metastore
+  thrift-generated-files-fb303)
 
 add_custom_target(thrift-ext-data-src ALL DEPENDS ${THRIFT_DATA_SRC_FILES})
 add_dependencies(thrift-ext-data-src thrift-cpp)

common/thrift/CatalogInternalService.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.CatalogInternalService
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/CatalogObjects.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.CatalogObjects
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/CatalogService.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.CatalogService
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Data.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Data
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/DataSinks.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.DataSinks
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Descriptors.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Descriptors
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/ExecStats.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.ExecStats
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Exprs.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Exprs
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/ExternalDataSource.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.ExternalDataSource
 namespace cpp impala.extdatasource
 namespace java org.apache.impala.extdatasource.thrift
 

common/thrift/Frontend.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Frontend
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/ImpalaInternalService.thrift

@@ -18,6 +18,7 @@
 //
 // This file contains the details of the protocol between coordinators and backends.
 
+namespace py impala_thrift_gen.ImpalaInternalService
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/ImpalaService.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.ImpalaService
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/JniCatalog.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.JniCatalog
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/LineageGraph.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.LineageGraph
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Logging.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Logging
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Metrics.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Metrics
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/NetworkTest.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.NetworkTest
 namespace cpp impalatest
 
 struct ThriftDataParams {

common/thrift/Partitions.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Partitions
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/PlanNodes.thrift

@@ -21,6 +21,7 @@
 // of the execution parameters of any one of the backends on which it is running
 // (those are recorded in TPlanFragmentInstanceCtx).
 
+namespace py impala_thrift_gen.PlanNodes
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Planner.thrift

@@ -18,6 +18,7 @@
 //
 // This file contains structures produced by the planner.
 
+namespace py impala_thrift_gen.Planner
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Query.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Query
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/ResourceProfile.thrift

@@ -19,6 +19,7 @@
 // and DataSinks to configure their memory usage. See ResourceProfile.java for more
 // details.
 
+namespace py impala_thrift_gen.ResourceProfile
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Results.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Results
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/RuntimeProfile.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.RuntimeProfile
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/SqlConstraints.thrift

@@ -16,6 +16,7 @@
 // under the License.
 
 
+namespace py impala_thrift_gen.SqlConstraints
 namespace cpp impala
 namespace java org.apache.impala.thrift
 
@@ -28,4 +29,4 @@ struct TSqlConstraints {
 
   // Foreign Keys information for the given table.
   2: required list<hive_metastore.SQLForeignKey> foreign_keys
-}
+}

common/thrift/StatestoreService.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.StatestoreService
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Status.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Status
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/SystemTables.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.SystemTables
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Types.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Types
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/Zip.thrift

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+namespace py impala_thrift_gen.Zip
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/beeswax.thrift

@@ -17,8 +17,8 @@
 
 // Interface for interacting with Beeswax Server
 
+namespace py impala_thrift_gen.beeswax
 namespace java com.cloudera.beeswax.api
-namespace py beeswaxd
 namespace cpp beeswax
 
 include "hive_metastore.thrift"

common/thrift/generate_error_codes.py

@@ -539,6 +539,7 @@ preamble = """
 // IT BY HAND.
 //
 
+namespace py impala_thrift_gen.ErrorCodes
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/generate_metrics.py

@@ -88,6 +88,7 @@ THRIFT_PREAMBLE = """
 // THIS FILE IS AUTO GENERATED BY generate_metrics.py DO NOT MODIFY IT BY HAND.
 //
 
+namespace py impala_thrift_gen.Metrics
 namespace cpp impala
 namespace java org.apache.impala.thrift
 

common/thrift/hive-1-api/TCLIService.thrift

@@ -32,6 +32,7 @@
 // * Service names begin with the letter "T", use a capital letter for each
 //   new word (with no underscores), and end with the word "Service".
 
+namespace py impala_thrift_gen.TCLIService
 namespace java org.apache.hive.service.cli.thrift
 namespace cpp apache.hive.service.cli.thrift
 

common/thrift/parquet.thrift

@@ -20,6 +20,7 @@
 /**
  * File format description for the parquet file format
  */
+namespace py impala_thrift_gen.parquet
 namespace cpp parquet
 namespace java org.apache.parquet.format
 

infra/python/deps/py3-requirements.txt

@@ -23,8 +23,9 @@ pylint == 2.10.2
     wrapt == 1.12.1
     typed-ast == 1.4.3
   configparser == 4.0.2
-  isort == 4.3.21
+  isort == 5.13.2
     futures == 3.3.0; python_version == "2.7"
+    poetry-core == 1.9.1
   singledispatch == 3.6.1
   toml == 0.10.2
   platformdirs == 2.4.1

lib/python/impala_py_lib/profiles.py

@@ -22,9 +22,11 @@ from __future__ import absolute_import, division, print_function
 import base64
 import datetime
 import zlib
+
 from thrift.protocol import TCompactProtocol
 from thrift.TSerialization import deserialize
-from RuntimeProfile.ttypes import TRuntimeProfileTree
+
+from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileTree
 
 
 def decode_profile_line(line):

shell/exec_summary.py

@@ -18,7 +18,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from ExecStats.ttypes import TExecStats
+from __future__ import absolute_import, print_function, unicode_literals
+
+from impala_thrift_gen.ExecStats.ttypes import TExecStats
 
 
 def build_exec_summary_table(summary, idx, indent_level, new_indent_level, output,

shell/impala_client.py

@@ -33,17 +33,19 @@ import traceback
 from datetime import datetime
 import uuid
 
-from beeswaxd import BeeswaxService
-from beeswaxd.BeeswaxService import QueryState
-from ImpalaService import ImpalaService, ImpalaHiveServer2Service
-from ImpalaService.ImpalaHiveServer2Service import (TGetRuntimeProfileReq,
-    TGetExecSummaryReq, TPingImpalaHS2ServiceReq, TCloseImpalaOperationReq)
-from ErrorCodes.ttypes import TErrorCode
-from Status.ttypes import TStatus
-from TCLIService.TCLIService import (TExecuteStatementReq, TOpenSessionReq,
-    TCloseSessionReq, TProtocolVersion, TStatusCode, TGetOperationStatusReq,
-    TOperationState, TFetchResultsReq, TFetchOrientation, TGetLogReq,
-    TGetResultSetMetadataReq, TTypeId, TCancelOperationReq, TCloseOperationReq)
+from impala_thrift_gen.beeswax import BeeswaxService
+from impala_thrift_gen.beeswax.BeeswaxService import QueryState
+from impala_thrift_gen.ImpalaService import ImpalaService, ImpalaHiveServer2Service
+from impala_thrift_gen.ImpalaService.ImpalaHiveServer2Service import (
+    TGetRuntimeProfileReq, TGetExecSummaryReq, TPingImpalaHS2ServiceReq,
+    TCloseImpalaOperationReq)
+from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode
+from impala_thrift_gen.Status.ttypes import TStatus
+from impala_thrift_gen.TCLIService.TCLIService import (TExecuteStatementReq,
+    TOpenSessionReq, TCloseSessionReq, TProtocolVersion, TStatusCode,
+    TGetOperationStatusReq, TOperationState, TFetchResultsReq, TFetchOrientation,
+    TGetLogReq, TGetResultSetMetadataReq, TTypeId, TCancelOperationReq,
+    TCloseOperationReq)
 from ImpalaHttpClient import ImpalaHttpClient
 from exec_summary import build_exec_summary_table
 from kerberos_util import get_kerb_host_from_kerberos_host_fqdn

shell/value_converter.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from TCLIService.TCLIService import TTypeId
+from impala_thrift_gen.TCLIService.TCLIService import TTypeId
 
 import sys
 

testdata/bin/wait-for-hiveserver2.py

@@ -30,7 +30,7 @@ from optparse import OptionParser
 from tests.util.thrift_util import create_transport
 
 # Imports required for HiveServer2 Client
-from TCLIService import TCLIService
+from impala_thrift_gen.TCLIService import TCLIService
 from thrift.transport import TTransport, TSocket
 from thrift.protocol import TBinaryProtocol
 

testdata/bin/wait-for-metastore.py

@@ -28,7 +28,7 @@ from optparse import OptionParser
 from tests.util.thrift_util import create_transport
 
 # Imports required for Hive Metastore Client
-from hive_metastore import ThriftHiveMetastore
+from impala_thrift_gen.hive_metastore import ThriftHiveMetastore
 from thrift.transport import TTransport, TSocket
 from thrift.protocol import TBinaryProtocol
 

tests/authorization/test_authorization.py

@@ -18,17 +18,18 @@
 # Client tests for SQL statement authorization
 
 from __future__ import absolute_import, division, print_function
-import pytest
+from getpass import getuser
 import random
 import threading
 import time
 
-from getpass import getuser
-from ImpalaService import ImpalaHiveServer2Service
-from TCLIService import TCLIService
+import pytest
+from thrift.protocol import TBinaryProtocol
 from thrift.transport.TSocket import TSocket
 from thrift.transport.TTransport import TBufferedTransport
-from thrift.protocol import TBinaryProtocol
+
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.file_utils import assert_file_in_dir_contains
 from tests.common.test_result_verifier import error_msg_equal

tests/authorization/test_authorized_proxy.py

@@ -16,16 +16,17 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-import pytest
+import json
 import os
 import time
-import json
 
-from ImpalaService import ImpalaHiveServer2Service
-from TCLIService import TCLIService
+import pytest
+from thrift.protocol import TBinaryProtocol
 from thrift.transport.TSocket import TSocket
 from thrift.transport.TTransport import TBufferedTransport
-from thrift.protocol import TBinaryProtocol
+
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.hs2.hs2_test_suite import operation_id_to_query_id
 

tests/beeswax/impala_beeswax.py

@@ -26,22 +26,23 @@
 #   result = client.execute(query_string)
 #   where result is an object of the class ImpalaBeeswaxResult.
 from __future__ import absolute_import, division, print_function
-from builtins import filter, map
-import logging
-import time
-import shlex
-import getpass
-import re
-import sys
-
-from beeswaxd import BeeswaxService
-from beeswaxd.BeeswaxService import QueryState
 from datetime import datetime
-from ImpalaService import ImpalaService
-from tests.util.thrift_util import create_transport
-from thrift.transport.TTransport import TTransportException
+import getpass
+import logging
+import re
+import shlex
+import sys
+import time
+
+from builtins import filter, map
 from thrift.protocol import TBinaryProtocol
 from thrift.Thrift import TApplicationException
+from thrift.transport.TTransport import TTransportException
+
+from impala_thrift_gen.beeswax import BeeswaxService
+from impala_thrift_gen.beeswax.BeeswaxService import QueryState
+from impala_thrift_gen.ImpalaService import ImpalaService
+from tests.util.thrift_util import create_transport
 
 LOG = logging.getLogger('impala_beeswax')
 # time to sleep in seconds before polling again. This uses a fixed

tests/catalog_service/test_catalog_service_client.py

@@ -19,13 +19,13 @@
 
 from __future__ import absolute_import, division, print_function
 import logging
-import pytest
 
-from CatalogService import CatalogService
-from CatalogService.CatalogService import TGetFunctionsRequest
-from ErrorCodes.ttypes import TErrorCode
+import pytest
 from thrift.protocol import TBinaryProtocol
 
+from impala_thrift_gen.CatalogService import CatalogService
+from impala_thrift_gen.CatalogService.CatalogService import TGetFunctionsRequest
+from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode
 from tests.common.impala_cluster import ImpalaCluster
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.skip import SkipIfDockerizedCluster
@@ -33,7 +33,6 @@ from tests.common.test_dimensions import create_single_exec_option_dimension
 from tests.util.filesystem_utils import WAREHOUSE
 from tests.util.thrift_util import create_transport
 
-
 LOG = logging.getLogger('test_catalog_service_client')
 
 # TODO: Add a test that asserts correct/compatible responses

tests/common/impala_connection.py

@@ -21,29 +21,28 @@
 
 from __future__ import absolute_import, division, print_function
 import abc
-from future.utils import with_metaclass
 import getpass
 import logging
 import re
 import time
 
-from beeswaxd.BeeswaxService import QueryState
+from future.utils import with_metaclass
 import impala.dbapi as impyla
 import impala.error as impyla_error
 import impala.hiveserver2 as hs2
-import tests.common
-from Query.ttypes import TQueryOptions
-from RuntimeProfile.ttypes import TRuntimeProfileFormat
+
+from impala_thrift_gen.beeswax.BeeswaxService import QueryState
+from impala_thrift_gen.Query.ttypes import TQueryOptions
+from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat
 from tests.beeswax.impala_beeswax import (
-  DEFAULT_SLEEP_INTERVAL,
-  ImpalaBeeswaxClient,
-  ImpalaBeeswaxException)
+    DEFAULT_SLEEP_INTERVAL,
+    ImpalaBeeswaxClient,
+    ImpalaBeeswaxException,
+)
+import tests.common
 from tests.common.patterns import LOG_FORMAT
 from tests.common.test_vector import BEESWAX, HS2, HS2_HTTP
-from tests.util.thrift_util import (
-  op_handle_to_query_id,
-  session_handle_to_session_id)
-
+from tests.util.thrift_util import op_handle_to_query_id, session_handle_to_session_id
 
 LOG = logging.getLogger(__name__)
 console_handler = logging.StreamHandler()

tests/common/impala_test_suite.py

@@ -102,11 +102,11 @@ from tests.util.thrift_util import create_transport
 from tests.util.retry import retry
 
 # Imports required for Hive Metastore Client
-from hive_metastore import ThriftHiveMetastore
+from impala_thrift_gen.hive_metastore import ThriftHiveMetastore
 from thrift.protocol import TBinaryProtocol
 
 # Import to validate query option names
-from ImpalaService.ttypes import TImpalaQueryOptions
+from impala_thrift_gen.ImpalaService.ttypes import TImpalaQueryOptions
 
 # Initializing the logger before conditional imports, since we will need it
 # for them.

tests/custom_cluster/test_admission_controller.py

@@ -18,50 +18,58 @@
 # Tests admission control
 
 from __future__ import absolute_import, division, print_function
-from builtins import int, range, round
+from copy import deepcopy
 import itertools
 import logging
 import os
-import pytest
 import re
 import signal
 import sys
 import threading
-from copy import deepcopy
 from time import sleep, time
 
+from builtins import int, range, round
+import pytest
+
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.cluster_config import (
-    impalad_admission_ctrl_flags,
     impalad_admission_ctrl_config_args,
-    RESOURCES_DIR)
+    impalad_admission_ctrl_flags,
+    RESOURCES_DIR,
+)
 from tests.common.custom_cluster_test_suite import (
     ADMISSIOND_ARGS,
+    CustomClusterTestSuite,
     IMPALAD_ARGS,
     START_ARGS,
     WORKLOAD_MGMT_IMPALAD_FLAGS,
-    CustomClusterTestSuite)
+)
 from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties
 from tests.common.impala_connection import (
-    RUNNING, FINISHED, ERROR,
-    IMPALA_CONNECTION_EXCEPTION)
+    ERROR,
+    FINISHED,
+    IMPALA_CONNECTION_EXCEPTION,
+    RUNNING,
+)
 from tests.common.resource_pool_config import ResourcePoolConfig
-from tests.common.skip import SkipIfFS, SkipIfEC, SkipIfNotHdfsMinicluster
+from tests.common.skip import SkipIfEC, SkipIfFS, SkipIfNotHdfsMinicluster
 from tests.common.test_dimensions import (
-    HS2,
     add_mandatory_exec_option,
     create_exec_option_dimension,
     create_single_exec_option_dimension,
-    create_uncompressed_text_dimension)
+    create_uncompressed_text_dimension,
+    HS2,
+)
 from tests.common.test_vector import ImpalaTestDimension
 from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
-from tests.util.workload_management import QUERY_TBL_LIVE
 from tests.util.web_pages_util import (
+    get_mem_admitted_backends_debug_page,
     get_num_completed_backends,
-    get_mem_admitted_backends_debug_page)
+)
+from tests.util.workload_management import QUERY_TBL_LIVE
 from tests.verifiers.mem_usage_verifier import MemUsageVerifier
 from tests.verifiers.metric_verifier import MetricVerifier
-from ImpalaService import ImpalaHiveServer2Service
-from TCLIService import TCLIService
 
 LOG = logging.getLogger('admission_test')
 

tests/custom_cluster/test_custom_statestore.py

@@ -19,26 +19,25 @@
 # Tests statestore with non-default startup options
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
 import logging
-import pytest
-import uuid
 import socket
-
-from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.environ import build_flavor_timeout
-from tests.common.skip import SkipIfBuildType
-from tests.common.patterns import print_id
 from time import sleep
+import uuid
 
-from Types.ttypes import TNetworkAddress
+from builtins import range
+import pytest
 from thrift.protocol import TBinaryProtocol
 from thrift.transport import TSocket, TTransport
 
-import StatestoreService.StatestoreSubscriber as Subscriber
-import StatestoreService.StatestoreService as Statestore
-import CatalogService.CatalogService as Catalog
-from ErrorCodes.ttypes import TErrorCode
+import impala_thrift_gen.CatalogService.CatalogService as Catalog
+from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode
+import impala_thrift_gen.StatestoreService.StatestoreService as Statestore
+import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber
+from impala_thrift_gen.Types.ttypes import TNetworkAddress
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.environ import build_flavor_timeout
+from tests.common.patterns import print_id
+from tests.common.skip import SkipIfBuildType
 
 LOG = logging.getLogger('custom_statestore_test')
 STATESTORE_SERVICE_PORT = 24000

tests/custom_cluster/test_delegation.py

@@ -17,10 +17,12 @@
 #
 from __future__ import absolute_import, division, print_function
 import getpass
+
 import pytest
-from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
-from TCLIService import TCLIService
+
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
 
 USER_NAME = getpass.getuser()
 PROXY_USER = "proxy_user_name"

tests/custom_cluster/test_event_processing_error.py

@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 from __future__ import absolute_import, division, print_function
+
 from builtins import range
 
-from hive_metastore.ttypes import FireEventRequest
-from hive_metastore.ttypes import FireEventRequestData
+from impala_thrift_gen.hive_metastore.ttypes import FireEventRequest, FireEventRequestData
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.skip import SkipIfCatalogV2, SkipIfFS
 from tests.metadata.test_event_processing_base import TestEventProcessingBase

tests/custom_cluster/test_events_custom_configs.py

@@ -23,9 +23,9 @@ from os import getenv
 from time import sleep
 
 
-from hive_metastore.ttypes import FireEventRequest
-from hive_metastore.ttypes import FireEventRequestData
-from hive_metastore.ttypes import InsertEventRequestData
+from impala_thrift_gen.hive_metastore.ttypes import FireEventRequest
+from impala_thrift_gen.hive_metastore.ttypes import FireEventRequestData
+from impala_thrift_gen.hive_metastore.ttypes import InsertEventRequestData
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.impala_connection import ERROR, FINISHED
 from tests.common.impala_test_suite import ImpalaTestSuite

tests/custom_cluster/test_hs2.py

@@ -16,13 +16,13 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from time import sleep
 
 import pytest
 
+from impala_thrift_gen.TCLIService import TCLIService
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.hs2.hs2_test_suite import HS2TestSuite, operation_id_to_query_id
-from time import sleep
-from TCLIService import TCLIService
 
 
 class TestHS2(CustomClusterTestSuite):

tests/custom_cluster/test_metastore_service.py

@@ -16,24 +16,26 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
+
 from builtins import range
 import pytest
 
-from hive_metastore.ttypes import Database
-from hive_metastore.ttypes import FieldSchema
-from hive_metastore.ttypes import FindNextCompactRequest
-from hive_metastore.ttypes import GetTableRequest
-from hive_metastore.ttypes import GetPartitionsByNamesRequest
-from hive_metastore.ttypes import TruncateTableRequest
-from hive_metastore.ttypes import Table
-from hive_metastore.ttypes import StorageDescriptor
-from hive_metastore.ttypes import SerDeInfo
-from hive_metastore.ttypes import UpdateTransactionalStatsRequest
-from hive_metastore.ttypes import WriteNotificationLogBatchRequest
-
-from tests.util.event_processor_utils import EventProcessorUtils
+from impala_thrift_gen.hive_metastore.ttypes import (
+    Database,
+    FieldSchema,
+    FindNextCompactRequest,
+    GetPartitionsByNamesRequest,
+    GetTableRequest,
+    SerDeInfo,
+    StorageDescriptor,
+    Table,
+    TruncateTableRequest,
+    UpdateTransactionalStatsRequest,
+    WriteNotificationLogBatchRequest,
+)
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.util.event_processor_utils import EventProcessorUtils
 from tests.util.filesystem_utils import IS_HDFS, IS_OZONE
 
 

tests/custom_cluster/test_query_log.py

@@ -16,19 +16,19 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-
+from getpass import getuser
 import os
+from random import choice, randint
+from signal import SIGRTMIN
 import string
 from time import sleep, time
 
-from getpass import getuser
-from ImpalaService import ImpalaHiveServer2Service
-from random import choice, randint
-from signal import SIGRTMIN
-from TCLIService import TCLIService
+from thrift.protocol import TBinaryProtocol
 from thrift.transport.TSocket import TSocket
 from thrift.transport.TTransport import TBufferedTransport
-from thrift.protocol import TBinaryProtocol
+
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.cluster_config import impalad_admission_ctrl_config_args
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.impala_connection import FINISHED
@@ -39,9 +39,10 @@ from tests.common.wm_test_suite import WorkloadManagementTestSuite
 from tests.util.retry import retry
 from tests.util.workload_management import (
     assert_query,
-    WM_DB,
     QUERY_TBL_LOG,
-    redaction_rules_file)
+    redaction_rules_file,
+    WM_DB,
+)
 
 
 class TestQueryLogTableBasic(WorkloadManagementTestSuite):

tests/custom_cluster/test_query_retries.py

@@ -22,21 +22,29 @@
 # TODO: Add a test that cancels queries while a retry is running
 
 from __future__ import absolute_import, division, print_function
-from builtins import map, range
-import pytest
+from random import randint
 import re
 import time
 
-from random import randint
+from builtins import map, range
+import pytest
 
-from RuntimeProfile.ttypes import TRuntimeProfileFormat
-from tests.common.impala_connection import (
-    ERROR, FINISHED, IMPALA_CONNECTION_EXCEPTION, RUNNING)
-from tests.common.impala_test_suite import ImpalaTestSuite, LOG
+from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.errors import Timeout
-from tests.common.skip import (SkipIfEC, SkipIfBuildType, SkipIfFS,
-    SkipIfNotHdfsMinicluster)
+from tests.common.impala_connection import (
+    ERROR,
+    FINISHED,
+    IMPALA_CONNECTION_EXCEPTION,
+    RUNNING,
+)
+from tests.common.impala_test_suite import ImpalaTestSuite, LOG
+from tests.common.skip import (
+    SkipIfBuildType,
+    SkipIfEC,
+    SkipIfFS,
+    SkipIfNotHdfsMinicluster,
+)
 from tests.common.test_dimensions import add_mandatory_exec_option
 
 # The BE krpc port of the impalad to simulate rpc or disk errors in tests.

tests/custom_cluster/test_restart_services.py

@@ -16,28 +16,31 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
 import logging
 import os
-import pytest
-import psutil
 import re
 import signal
 import socket
-import time
-import threading
-
 from subprocess import check_call
-from tests.common.environ import build_flavor_timeout
+import threading
+import time
 from time import sleep
 
+from builtins import range
 from impala.error import HiveServer2Error
-from TCLIService import TCLIService
+import psutil
+import pytest
 
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.environ import build_flavor_timeout
 from tests.common.impala_connection import (
-    ERROR, FINISHED, IMPALA_CONNECTION_EXCEPTION, RUNNING)
-from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfFS
+    ERROR,
+    FINISHED,
+    IMPALA_CONNECTION_EXCEPTION,
+    RUNNING,
+)
+from tests.common.skip import SkipIfFS, SkipIfNotHdfsMinicluster
 from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
 
 LOG = logging.getLogger(__name__)

tests/custom_cluster/test_set_and_unset.py

@@ -16,12 +16,14 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
+
 import pytest
 
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
-from TCLIService import TCLIService
-from ImpalaService import ImpalaHiveServer2Service
+
 
 class TestSetAndUnset(CustomClusterTestSuite, HS2TestSuite):
   """

tests/custom_cluster/test_statestored_ha.py

@@ -17,23 +17,23 @@
 
 from __future__ import absolute_import, division, print_function
 import logging
-import pytest
 import time
-
-from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties
-from tests.common.impala_cluster import (
-    DEFAULT_CATALOG_SERVICE_PORT, DEFAULT_STATESTORE_SERVICE_PORT)
-from tests.common.impala_connection import (
-    ERROR, IMPALA_CONNECTION_EXCEPTION, RUNNING)
-from tests.common.skip import SkipIfBuildType, SkipIfNotHdfsMinicluster
 from time import sleep
 
+import pytest
 from thrift.protocol import TBinaryProtocol
 from thrift.transport import TSocket, TTransport
 
-import StatestoreService.StatestoreSubscriber as Subscriber
-import StatestoreService.StatestoreService as Statestore
+import impala_thrift_gen.StatestoreService.StatestoreService as Statestore
+import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties
+from tests.common.impala_cluster import (
+    DEFAULT_CATALOG_SERVICE_PORT,
+    DEFAULT_STATESTORE_SERVICE_PORT,
+)
+from tests.common.impala_connection import ERROR, IMPALA_CONNECTION_EXCEPTION, RUNNING
+from tests.common.skip import SkipIfBuildType, SkipIfNotHdfsMinicluster
 
 LOG = logging.getLogger('statestored_ha_test')
 

tests/custom_cluster/test_workload_mgmt_init.py

@@ -16,26 +16,26 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-
+from logging import getLogger
 import os
 import re
-
 from subprocess import CalledProcessError
-from logging import getLogger
 
-from SystemTables.ttypes import TQueryTableColumn
+from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn
 from tests.common.custom_cluster_test_suite import (
+    CustomClusterTestSuite,
     WORKLOAD_MGMT_IMPALAD_FLAGS,
-    CustomClusterTestSuite)
+)
 from tests.common.test_dimensions import hs2_client_protocol_dimension
 from tests.common.test_vector import HS2
 from tests.util.workload_management import (
     assert_query,
-    WM_DB,
-    QUERY_TBL_LOG_NAME,
-    QUERY_TBL_LOG,
+    QUERY_TBL_LIVE,
     QUERY_TBL_LIVE_NAME,
-    QUERY_TBL_LIVE)
+    QUERY_TBL_LOG,
+    QUERY_TBL_LOG_NAME,
+    WM_DB,
+)
 
 LOG = getLogger(__name__)
 QUERY_TBL_ALL = "{},{}".format(QUERY_TBL_LOG_NAME, QUERY_TBL_LIVE_NAME)

tests/custom_cluster/test_workload_mgmt_sql_details.py

@@ -19,7 +19,7 @@ from __future__ import absolute_import, division, print_function
 
 import os
 
-from SystemTables.ttypes import TQueryTableColumn
+from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.test_dimensions import hs2_client_protocol_dimension
 from tests.common.wm_test_suite import WorkloadManagementTestSuite

tests/hs2/hs2_test_suite.py

@@ -18,17 +18,19 @@
 # Superclass of all HS2 tests containing commonly used functions.
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
 from getpass import getuser
-from TCLIService import TCLIService
-from ImpalaService import ImpalaHiveServer2Service
+import sys
+from time import sleep, time
+
+from builtins import range
+from thrift.protocol import TBinaryProtocol
 from thrift.transport.TSocket import TSocket
 from thrift.transport.TTransport import TBufferedTransport
-from thrift.protocol import TBinaryProtocol
-from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT
+
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
+from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT, ImpalaTestSuite
 from tests.common.test_result_verifier import error_msg_startswith
-from time import sleep, time
-import sys
 
 
 def add_session_helper(self, protocol_version, conf_overlay, close_session, fn):

tests/hs2/test_fetch.py

@@ -17,13 +17,17 @@
 #
 
 from __future__ import absolute_import, division, print_function
-import pytest
 import re
 
-from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session,
-    create_op_handle_without_secret)
-from TCLIService import TCLIService, constants
-from TCLIService.ttypes import TTypeId
+import pytest
+
+from impala_thrift_gen.TCLIService import constants, TCLIService
+from impala_thrift_gen.TCLIService.ttypes import TTypeId
+from tests.hs2.hs2_test_suite import (
+    create_op_handle_without_secret,
+    HS2TestSuite,
+    needs_session,
+)
 
 
 # Simple test to make sure all the HS2 types are supported for both the row and

tests/hs2/test_fetch_first.py

@@ -21,13 +21,14 @@
 # succeed as long all previously fetched rows fit into the bounded result cache.
 
 from __future__ import absolute_import, division, print_function
+
 from builtins import range
 import pytest
 
-from ImpalaService import ImpalaHiveServer2Service
-from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
-from TCLIService import TCLIService
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.impala_cluster import ImpalaCluster
+from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
 
 
 class TestFetchFirst(HS2TestSuite):

tests/hs2/test_fetch_timeout.py

@@ -16,12 +16,11 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from time import sleep
-from time import time
-from tests.common.errors import Timeout
-from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session)
-from TCLIService import TCLIService
+from time import sleep, time
 
+from impala_thrift_gen.TCLIService import TCLIService
+from tests.common.errors import Timeout
+from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
 
 # Tests for the query option FETCH_ROWS_TIMEOUT_MS, which is the maximum amount of
 # time, in milliseconds, a fetch rows request (TFetchResultsReq) from the client should

tests/hs2/test_hs2.py

@@ -18,33 +18,37 @@
 # Client tests for Impala's HiveServer2 interface
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
-from getpass import getuser
 from contextlib import contextmanager
+from getpass import getuser
 import json
 import logging
-import pytest
 import random
 import threading
 import time
 import uuid
-import impala.dbapi as impyla
 
+from builtins import range
+import impala.dbapi as impyla
+import pytest
+
+from impala_thrift_gen.ImpalaService import ImpalaHiveServer2Service
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.impala_test_suite import IMPALAD_HOSTNAME, IMPALAD_HS2_HTTP_PORT
+from tests.common.skip import SkipIfDockerizedCluster
+from tests.hs2.hs2_test_suite import (
+    create_op_handle_without_secret,
+    create_session_handle_without_secret,
+    HS2TestSuite,
+    needs_session,
+    needs_session_cluster_properties,
+    operation_id_to_query_id,
+)
 
 try:
   from urllib.request import urlopen
 except ImportError:
   from urllib2 import urlopen
 
-from ImpalaService import ImpalaHiveServer2Service
-from tests.common.environ import ImpalaTestClusterProperties
-from tests.common.skip import SkipIfDockerizedCluster
-from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session,
-    operation_id_to_query_id, create_session_handle_without_secret,
-    create_op_handle_without_secret, needs_session_cluster_properties)
-from TCLIService import TCLIService
-
 LOG = logging.getLogger('test_hs2')
 
 SQLSTATE_GENERAL_ERROR = "HY000"

tests/hs2/test_json_endpoints.py

@@ -19,18 +19,20 @@
 
 from __future__ import absolute_import, division, print_function
 import json
-import pytest
 from time import time
 
+import pytest
+
+from impala_thrift_gen.TCLIService import TCLIService
+from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER
+from tests.common.impala_cluster import ImpalaCluster
+from tests.hs2.hs2_test_suite import HS2TestSuite
+
 try:
   from urllib.request import urlopen
 except ImportError:
   from urllib2 import urlopen
 
-from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER
-from tests.common.impala_cluster import ImpalaCluster
-from tests.hs2.hs2_test_suite import HS2TestSuite
-from TCLIService import TCLIService
 
 class TestJsonEndpoints(HS2TestSuite):
   def _get_json_queries(self, http_addr):

tests/metadata/test_compute_stats.py

@@ -16,23 +16,28 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
 import os
-import pytest
-from hive_metastore.ttypes import (
-    ColumnStatistics, ColumnStatisticsDesc, ColumnStatisticsData,
-    ColumnStatisticsObj, StringColumnStatsData)
 
+from builtins import range
+import pytest
+
+from impala_thrift_gen.CatalogObjects.ttypes import THdfsCompression
+from impala_thrift_gen.hive_metastore.ttypes import (
+    ColumnStatistics,
+    ColumnStatisticsData,
+    ColumnStatisticsDesc,
+    ColumnStatisticsObj,
+    StringColumnStatsData,
+)
 from tests.common.environ import ImpalaTestClusterProperties
 from tests.common.impala_cluster import ImpalaCluster
 from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfCatalogV2
+from tests.common.skip import SkipIfCatalogV2, SkipIfFS, SkipIfLocal
 from tests.common.test_dimensions import (
     create_exec_option_dimension,
     create_single_exec_option_dimension,
-    create_uncompressed_text_dimension)
-from CatalogObjects.ttypes import THdfsCompression
-
+    create_uncompressed_text_dimension,
+)
 
 IMPALA_TEST_CLUSTER_PROPERTIES = ImpalaTestClusterProperties.get_instance()
 

tests/query_test/test_acid.py

@@ -19,13 +19,18 @@
 
 from __future__ import absolute_import, division, print_function
 import os
-import pytest
+from subprocess import check_call
 import time
 
-from hive_metastore.ttypes import CommitTxnRequest, LockType, OpenTxnRequest
-from subprocess import check_call
+import pytest
+
+from impala_thrift_gen.hive_metastore.ttypes import (
+    CommitTxnRequest,
+    LockType,
+    OpenTxnRequest,
+)
 from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIf, SkipIfHive2, SkipIfCatalogV2, SkipIfFS
+from tests.common.skip import SkipIf, SkipIfCatalogV2, SkipIfFS, SkipIfHive2
 from tests.common.test_dimensions import create_single_exec_option_dimension
 from tests.util.acid_txn import AcidTxn
 

tests/query_test/test_cancellation.py

@@ -19,18 +19,19 @@
 #
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
-import pytest
 from threading import Thread
 from time import sleep
-from RuntimeProfile.ttypes import TRuntimeProfileFormat
+
+from builtins import range
+import pytest
+
+from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat
+from tests.common.impala_connection import MinimalHS2Connection
+from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT, ImpalaTestSuite
 from tests.common.test_dimensions import add_mandatory_exec_option
 from tests.common.test_vector import ImpalaTestDimension
-from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT
 from tests.util.cancel_util import cancel_query_and_validate_state
 from tests.verifiers.metric_verifier import MetricVerifier
-from tests.common.impala_connection import MinimalHS2Connection
 
 # PRIMARY KEY for lineitem
 LINEITEM_PK = 'l_orderkey, l_partkey, l_suppkey, l_linenumber'

tests/query_test/test_iceberg.py

@@ -16,37 +16,37 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
 from collections import defaultdict, namedtuple
 import datetime
+import json
 import logging
 import os
-import pytest
-import pytz
 import random
 import re
-import time
-
 from subprocess import check_call, check_output
-# noinspection PyUnresolvedReferences
-from parquet.ttypes import ConvertedType
+import time
 
 from avro.datafile import DataFileReader
 from avro.io import DatumReader
-import json
+from builtins import range
+import pytest
+import pytz
 
-from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION
+# noinspection PyUnresolvedReferences
+from impala_thrift_gen.parquet.ttypes import ConvertedType
+from tests.common.file_utils import (
+    create_iceberg_table_from_directory,
+    create_table_from_parquet,
+)
 from tests.common.iceberg_test_suite import IcebergTestSuite
-from tests.common.skip import SkipIf, SkipIfFS, SkipIfDockerizedCluster
+from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION
+from tests.common.skip import SkipIf, SkipIfDockerizedCluster, SkipIfFS
 from tests.common.test_dimensions import add_exec_option_dimension
 from tests.common.test_result_verifier import error_msg_startswith
-from tests.common.file_utils import (
-  create_iceberg_table_from_directory,
-  create_table_from_parquet)
 from tests.shell.util import run_impala_shell_cmd
-from tests.util.filesystem_utils import get_fs_path, IS_HDFS, WAREHOUSE, FILESYSTEM_PREFIX
+from tests.util.filesystem_utils import FILESYSTEM_PREFIX, get_fs_path, IS_HDFS, WAREHOUSE
 from tests.util.get_parquet_metadata import get_parquet_metadata
-from tests.util.iceberg_util import cast_ts, quote, get_snapshots, IcebergCatalogs
+from tests.util.iceberg_util import cast_ts, get_snapshots, IcebergCatalogs, quote
 
 LOG = logging.getLogger(__name__)
 

tests/query_test/test_insert_parquet.py

@@ -18,27 +18,35 @@
 # Targeted Impala insert tests
 
 from __future__ import absolute_import, division, print_function
-from builtins import map, range, round
-import os
-
 from collections import namedtuple
-from datetime import datetime, date
+from datetime import date, datetime
 from decimal import Decimal
+import os
 from subprocess import check_call
-from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder, ConvertedType
 
+from builtins import map, range, round
+
+from impala_thrift_gen.parquet.ttypes import (
+    ColumnOrder,
+    ConvertedType,
+    SortingColumn,
+    TypeDefinedOrder,
+)
 from tests.common.environ import impalad_basedir
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.parametrize import UniqueDatabase
 from tests.common.skip import SkipIfFS, SkipIfLocal
 from tests.common.test_dimensions import (
     add_exec_option_dimension,
-    create_exec_option_dimension)
+    create_exec_option_dimension,
+)
 from tests.common.test_result_verifier import verify_query_result_is_equal
 from tests.common.test_vector import ImpalaTestDimension
 from tests.util.filesystem_utils import get_fs_path, WAREHOUSE
-from tests.util.get_parquet_metadata import (decode_stats_value,
-    get_parquet_metadata_from_hdfs_folder)
+from tests.util.get_parquet_metadata import (
+    decode_stats_value,
+    get_parquet_metadata_from_hdfs_folder,
+)
 
 PARQUET_CODECS = ['none', 'snappy', 'gzip', 'zstd', 'lz4']
 IMPALA_HOME = os.environ['IMPALA_HOME']

tests/query_test/test_observability.py

@@ -18,6 +18,12 @@
 from __future__ import absolute_import, division, print_function
 from collections import defaultdict
 from datetime import datetime
+import re
+from time import sleep, time
+
+import pytest
+
+from impala_thrift_gen.RuntimeProfile.ttypes import TRuntimeProfileFormat
 from tests.common.impala_cluster import ImpalaCluster
 from tests.common.impala_connection import IMPALA_CONNECTION_EXCEPTION
 from tests.common.impala_test_suite import ImpalaTestSuite
@@ -25,10 +31,6 @@ from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfNotHdfsMinicluster
 from tests.common.test_vector import HS2
 from tests.util.filesystem_utils import WAREHOUSE
 from tests.util.parse_util import get_duration_us_from_str
-from time import sleep, time
-from RuntimeProfile.ttypes import TRuntimeProfileFormat
-import pytest
-import re
 
 
 class TestObservability(ImpalaTestSuite):

tests/query_test/test_parquet_bloom_filter.py

@@ -16,21 +16,18 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
+from collections import namedtuple
 import math
 import os
-
-from collections import namedtuple
-from parquet.ttypes import BloomFilterHeader
 from subprocess import check_call
 
+from builtins import range
+
+from impala_thrift_gen.parquet.ttypes import BloomFilterHeader
 from tests.common.file_utils import create_table_and_copy_files
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.util.filesystem_utils import get_fs_path
-from tests.util.get_parquet_metadata import (
-    get_parquet_metadata,
-    read_serialized_object
-)
+from tests.util.get_parquet_metadata import get_parquet_metadata, read_serialized_object
 
 
 class TestParquetBloomFilter(ImpalaTestSuite):

tests/query_test/test_parquet_page_index.py

@@ -18,21 +18,26 @@
 # Targeted Impala insert tests
 
 from __future__ import absolute_import, division, print_function
+from collections import namedtuple
 import os
 import random
 import string
-
-from collections import namedtuple
 from subprocess import check_call
-from parquet.ttypes import BoundaryOrder, ColumnIndex, OffsetIndex, PageHeader, PageType
 
+from impala_thrift_gen.parquet.ttypes import (
+    BoundaryOrder,
+    ColumnIndex,
+    OffsetIndex,
+    PageHeader,
+    PageType,
+)
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.skip import SkipIfLocal
 from tests.util.filesystem_utils import get_fs_path
 from tests.util.get_parquet_metadata import (
     decode_stats_value,
     get_parquet_metadata,
-    read_serialized_object
+    read_serialized_object,
 )
 
 PAGE_INDEX_MAX_STRING_LENGTH = 64

tests/query_test/test_query_opts.py

@@ -21,12 +21,13 @@
 #       timeout).
 
 from __future__ import absolute_import, division, print_function
-from TCLIService import TCLIService
 
+from impala_thrift_gen.TCLIService import TCLIService
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.test_dimensions import create_exec_option_dimension
 from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
 
+
 class TestQueryOptions(ImpalaTestSuite):
   @classmethod
   def add_test_dimensions(cls):

tests/query_test/test_scanners.py

@@ -22,17 +22,19 @@
 # explode.
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
+from copy import deepcopy
 import os
-import pytest
 import random
 import re
-import tempfile
-from copy import deepcopy
-from parquet.ttypes import ConvertedType
 from subprocess import check_call
+import tempfile
 
+from builtins import range
+import pytest
+
+from impala_thrift_gen.parquet.ttypes import ConvertedType
 from testdata.common import widetable
+from tests.common.file_utils import create_table_and_copy_files, create_table_from_parquet
 from tests.common.impala_test_suite import ImpalaTestSuite, LOG
 from tests.common.skip import (
     SkipIf,
@@ -41,21 +43,18 @@ from tests.common.skip import (
     SkipIfHive2,
     SkipIfHive3,
     SkipIfLocal,
-    SkipIfNotHdfsMinicluster)
+    SkipIfNotHdfsMinicluster,
+)
 from tests.common.test_dimensions import (
     add_exec_option_dimension,
     add_mandatory_exec_option,
-    create_single_exec_option_dimension,
     create_exec_option_dimension,
-    create_uncompressed_text_dimension)
-from tests.common.file_utils import (
-    create_table_from_parquet,
-    create_table_and_copy_files)
-from tests.common.test_result_verifier import (
-    QueryTestResult,
-    parse_result_rows)
+    create_single_exec_option_dimension,
+    create_uncompressed_text_dimension,
+)
+from tests.common.test_result_verifier import parse_result_rows, QueryTestResult
 from tests.common.test_vector import ImpalaTestDimension
-from tests.util.filesystem_utils import IS_HDFS, get_fs_path
+from tests.util.filesystem_utils import get_fs_path, IS_HDFS
 from tests.util.get_parquet_metadata import get_parquet_metadata
 from tests.util.parse_util import get_bytes_summary_stats_counter
 from tests.util.test_file_parser import QueryTestSectionReader

tests/statestore/test_statestore.py

@@ -16,38 +16,38 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from builtins import range
 from collections import defaultdict
 import json
 import logging
 import socket
 import threading
-import traceback
 import time
+import traceback
 import uuid
 
+from builtins import range
+from thrift.protocol import TBinaryProtocol
+from thrift.server.TServer import TServer
+from thrift.transport import TSocket, TTransport
+
+from impala_thrift_gen.ErrorCodes.ttypes import TErrorCode
+import impala_thrift_gen.StatestoreService.StatestoreService as Statestore
+import impala_thrift_gen.StatestoreService.StatestoreSubscriber as Subscriber
+from impala_thrift_gen.StatestoreService.StatestoreSubscriber import (
+    TTopicRegistration,
+    TUpdateStateResponse,
+)
+from impala_thrift_gen.Status.ttypes import TStatus
+from impala_thrift_gen.Types.ttypes import TNetworkAddress
+from tests.common.base_test_suite import BaseTestSuite
+from tests.common.environ import build_flavor_timeout
+from tests.common.skip import SkipIfDockerizedCluster
+
 try:
   from urllib.request import urlopen
 except ImportError:
   from urllib2 import urlopen
 
-from Types.ttypes import TNetworkAddress
-from thrift.protocol import TBinaryProtocol
-from thrift.server.TServer import TServer
-from thrift.transport import TSocket
-from thrift.transport import TTransport
-
-import StatestoreService.StatestoreSubscriber as Subscriber
-import StatestoreService.StatestoreService as Statestore
-from StatestoreService.StatestoreSubscriber import TUpdateStateResponse
-from StatestoreService.StatestoreSubscriber import TTopicRegistration
-from ErrorCodes.ttypes import TErrorCode
-from Status.ttypes import TStatus
-
-from tests.common.base_test_suite import BaseTestSuite
-from tests.common.environ import build_flavor_timeout
-from tests.common.skip import SkipIfDockerizedCluster
-
 LOG = logging.getLogger('test_statestore')
 
 # Tests for the statestore. The StatestoreSubscriber class is a skeleton implementation of

tests/util/acid_txn.py

@@ -16,14 +16,28 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from tests.util.thrift_util import create_transport
-from hive_metastore import ThriftHiveMetastore
-from hive_metastore.ttypes import (AbortTxnRequest, AllocateTableWriteIdsRequest,
-    CheckLockRequest, CommitTxnRequest, GetValidWriteIdsRequest, HeartbeatRequest,
-    LockComponent, LockLevel, LockType, LockRequest, OpenTxnRequest, ShowLocksRequest,
-    TruncateTableRequest, UnlockRequest)
+
 from thrift.protocol import TBinaryProtocol
 
+from impala_thrift_gen.hive_metastore import ThriftHiveMetastore
+from impala_thrift_gen.hive_metastore.ttypes import (
+    AbortTxnRequest,
+    AllocateTableWriteIdsRequest,
+    CheckLockRequest,
+    CommitTxnRequest,
+    GetValidWriteIdsRequest,
+    HeartbeatRequest,
+    LockComponent,
+    LockLevel,
+    LockRequest,
+    LockType,
+    OpenTxnRequest,
+    ShowLocksRequest,
+    TruncateTableRequest,
+    UnlockRequest,
+)
+from tests.util.thrift_util import create_transport
+
 # HMS config
 metastore_host = "localhost"
 metastore_port = "9083"

tests/util/event_processor_utils.py

@@ -20,14 +20,15 @@
 # succeeded by querying Impala, or vice versa.
 
 from __future__ import absolute_import, division, print_function
-import logging
-import requests
-import time
 import json
-from hive_metastore.ttypes import NotificationEventRequest
+import logging
+import time
 
-from tests.common.impala_cluster import ImpalaCluster
+import requests
+
+from impala_thrift_gen.hive_metastore.ttypes import NotificationEventRequest
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.impala_cluster import ImpalaCluster
 
 LOG = logging.getLogger('event_processor_utils')
 LOG.setLevel(level=logging.DEBUG)

tests/util/get_parquet_metadata.py

@@ -16,19 +16,20 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function
-from builtins import map
-import os
-import struct
-import sys
-
 from datetime import date, datetime, time, timedelta
 from decimal import Decimal
 from functools import reduce
-from parquet.ttypes import ColumnIndex, FileMetaData, OffsetIndex, PageHeader, Type
+import os
+import struct
 from subprocess import check_call
+import sys
+
+from builtins import map
 from thrift.protocol import TCompactProtocol
 from thrift.transport import TTransport
 
+from impala_thrift_gen.parquet.ttypes import FileMetaData, Type
+
 PARQUET_VERSION_NUMBER = b'PAR1'
 
 

tests/util/parse_util.py

@@ -203,7 +203,7 @@ def get_bytes_summary_stats_counter(counter_name, runtime_profile):
   """
   # This requires the Thrift definitions to be generated. We limit the scope of the import
   # to allow tools like the stress test to import this file without building Impala.
-  from RuntimeProfile.ttypes import TSummaryStatsCounter
+  from impala_thrift_gen.RuntimeProfile.ttypes import TSummaryStatsCounter
 
   regex_summary_stat = re.compile(r"""\(
     Avg:[^\(]*\((?P<avg>[0-9]+)\)\s;\s # Matches Avg: [?].[?] [?]B (?)
@@ -256,7 +256,7 @@ def get_time_summary_stats_counter(counter_name, runtime_profile):
   """
   # This requires the Thrift definitions to be generated. We limit the scope of the import
   # to allow tools like the stress test to import this file without building Impala.
-  from RuntimeProfile.ttypes import TSummaryStatsCounter
+  from impala_thrift_gen.RuntimeProfile.ttypes import TSummaryStatsCounter
 
   regex_summary_stat = re.compile(r"""\(
     Avg:\s(?P<avg>.*)\s;\s # Matches Avg: ? ;

tests/util/workload_management.py

@@ -24,7 +24,7 @@ import requests
 from datetime import datetime
 from time import sleep, time
 
-from SystemTables.ttypes import TQueryTableColumn
+from impala_thrift_gen.SystemTables.ttypes import TQueryTableColumn
 from tests.util.assert_time import assert_time_str, convert_to_milliseconds
 from tests.util.memory import assert_byte_str, convert_to_bytes