mirror of
https://github.com/apache/impala.git
synced 2026-02-03 00:00:40 -05:00
4186727fe6
Like IMPALA-8369, this patch adds a compatibility shim in fe so that Impala can interoperate with Hive 3.1.2. we need adds a new Metastoreshim class under compat-apache-hive-3 directory. These shim classes implement method which are different in cdp-hive-3 vs apache-hive-3 and are used by front end code. At the build time, based on the environment variable IMPALA_HIVE_DIST_TYPE one of the two shims is added to as source using the fe/pom.xml build plugin. Some codes that directly use Hive 4 APIs need to be ignored in compilation, eg. fe/src/main/java/org/apache/impala/catalog/metastore/. Use Maven profile to ignore some codes, profile will automatically activated based on the IMPALA_HIVE_DIST_TYPE. Testing: 1. Code compiles and runs against both HMS-3 and ASF-HMS-3 2. Ran full-suite of tests against HMS-3 3. Running full-tests against ASF-HMS-3 will need more work supporting Tez in the mini-cluster (for dataloading) and HMS transaction support. This will be on-going effort and test failures on ASF-Hive-3 will be fixed in additional sub-tasks. Notes: 1. Patch uses a custom build of Apache Hive to be deployed in mini-cluster. This build has the fixes for HIVE-21569, HIVE-20038. This hack will be added to the build script in additional sub-tasks. Change-Id: I9f08db5f6da735ac431819063060941f0941f606 Reviewed-on: http://gerrit.cloudera.org:8080/17774 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
162 lines
6.5 KiB
Diff
162 lines
6.5 KiB
Diff
diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift
|
|
index ad1dc1f769..c92db542bb 100644
|
|
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
|
|
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
|
|
@@ -411,65 +411,13 @@ struct StorageDescriptor {
|
|
12: optional bool storedAsSubDirectories // stored as subdirectories or not
|
|
}
|
|
|
|
-// table information
|
|
-struct Table {
|
|
- 1: string tableName, // name of the table
|
|
- 2: string dbName, // database name ('default')
|
|
- 3: string owner, // owner of this table
|
|
- 4: i32 createTime, // creation time of the table
|
|
- 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
|
|
- 6: i32 retention, // retention time
|
|
- 7: StorageDescriptor sd, // storage descriptor of the table
|
|
- 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
|
|
- 9: map<string, string> parameters, // to store comments or any other user level parameters
|
|
- 10: string viewOriginalText, // original view text, null for non-view
|
|
- 11: string viewExpandedText, // expanded view text, null for non-view
|
|
- 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
|
|
- 13: optional PrincipalPrivilegeSet privileges,
|
|
- 14: optional bool temporary=false,
|
|
- 15: optional bool rewriteEnabled, // rewrite enabled or not
|
|
- 16: optional CreationMetadata creationMetadata, // only for MVs, it stores table names used and txn list at MV creation
|
|
- 17: optional string catName, // Name of the catalog the table is in
|
|
- 18: optional PrincipalType ownerType = PrincipalType.USER // owner type of this table (default to USER for backward compatibility)
|
|
-}
|
|
-
|
|
-struct Partition {
|
|
- 1: list<string> values // string value is converted to appropriate partition key type
|
|
- 2: string dbName,
|
|
- 3: string tableName,
|
|
- 4: i32 createTime,
|
|
- 5: i32 lastAccessTime,
|
|
- 6: StorageDescriptor sd,
|
|
- 7: map<string, string> parameters,
|
|
- 8: optional PrincipalPrivilegeSet privileges,
|
|
- 9: optional string catName
|
|
-}
|
|
-
|
|
-struct PartitionWithoutSD {
|
|
- 1: list<string> values // string value is converted to appropriate partition key type
|
|
- 2: i32 createTime,
|
|
- 3: i32 lastAccessTime,
|
|
- 4: string relativePath,
|
|
- 5: map<string, string> parameters,
|
|
- 6: optional PrincipalPrivilegeSet privileges
|
|
-}
|
|
-
|
|
-struct PartitionSpecWithSharedSD {
|
|
- 1: list<PartitionWithoutSD> partitions,
|
|
- 2: StorageDescriptor sd,
|
|
-}
|
|
-
|
|
-struct PartitionListComposingSpec {
|
|
- 1: list<Partition> partitions
|
|
-}
|
|
-
|
|
-struct PartitionSpec {
|
|
- 1: string dbName,
|
|
- 2: string tableName,
|
|
- 3: string rootPath,
|
|
- 4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec,
|
|
- 5: optional PartitionListComposingSpec partitionList,
|
|
- 6: optional string catName
|
|
+struct CreationMetadata {
|
|
+ 1: required string catName
|
|
+ 2: required string dbName,
|
|
+ 3: required string tblName,
|
|
+ 4: required set<string> tablesUsed,
|
|
+ 5: optional string validTxnList,
|
|
+ 6: optional i64 materializationTime
|
|
}
|
|
|
|
// column statistics
|
|
@@ -567,6 +515,67 @@ struct ColumnStatistics {
|
|
2: required list<ColumnStatisticsObj> statsObj;
|
|
}
|
|
|
|
+// table information
|
|
+struct Table {
|
|
+ 1: string tableName, // name of the table
|
|
+ 2: string dbName, // database name ('default')
|
|
+ 3: string owner, // owner of this table
|
|
+ 4: i32 createTime, // creation time of the table
|
|
+ 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
|
|
+ 6: i32 retention, // retention time
|
|
+ 7: StorageDescriptor sd, // storage descriptor of the table
|
|
+ 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
|
|
+ 9: map<string, string> parameters, // to store comments or any other user level parameters
|
|
+ 10: string viewOriginalText, // original view text, null for non-view
|
|
+ 11: string viewExpandedText, // expanded view text, null for non-view
|
|
+ 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
|
|
+ 13: optional PrincipalPrivilegeSet privileges,
|
|
+ 14: optional bool temporary=false,
|
|
+ 15: optional bool rewriteEnabled, // rewrite enabled or not
|
|
+ 16: optional CreationMetadata creationMetadata, // only for MVs, it stores table names used and txn list at MV creation
|
|
+ 17: optional string catName, // Name of the catalog the table is in
|
|
+ 18: optional PrincipalType ownerType = PrincipalType.USER // owner type of this table (default to USER for backward compatibility)
|
|
+}
|
|
+
|
|
+struct Partition {
|
|
+ 1: list<string> values // string value is converted to appropriate partition key type
|
|
+ 2: string dbName,
|
|
+ 3: string tableName,
|
|
+ 4: i32 createTime,
|
|
+ 5: i32 lastAccessTime,
|
|
+ 6: StorageDescriptor sd,
|
|
+ 7: map<string, string> parameters,
|
|
+ 8: optional PrincipalPrivilegeSet privileges,
|
|
+ 9: optional string catName
|
|
+}
|
|
+
|
|
+struct PartitionWithoutSD {
|
|
+ 1: list<string> values // string value is converted to appropriate partition key type
|
|
+ 2: i32 createTime,
|
|
+ 3: i32 lastAccessTime,
|
|
+ 4: string relativePath,
|
|
+ 5: map<string, string> parameters,
|
|
+ 6: optional PrincipalPrivilegeSet privileges
|
|
+}
|
|
+
|
|
+struct PartitionSpecWithSharedSD {
|
|
+ 1: list<PartitionWithoutSD> partitions,
|
|
+ 2: StorageDescriptor sd,
|
|
+}
|
|
+
|
|
+struct PartitionListComposingSpec {
|
|
+ 1: list<Partition> partitions
|
|
+}
|
|
+
|
|
+struct PartitionSpec {
|
|
+ 1: string dbName,
|
|
+ 2: string tableName,
|
|
+ 3: string rootPath,
|
|
+ 4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec,
|
|
+ 5: optional PartitionListComposingSpec partitionList,
|
|
+ 6: optional string catName
|
|
+}
|
|
+
|
|
struct AggrStats {
|
|
1: required list<ColumnStatisticsObj> colStats,
|
|
2: required i64 partsFound // number of partitions for which stats were found
|
|
@@ -1055,15 +1064,6 @@ struct BasicTxnInfo {
|
|
6: optional string partitionname
|
|
}
|
|
|
|
-struct CreationMetadata {
|
|
- 1: required string catName
|
|
- 2: required string dbName,
|
|
- 3: required string tblName,
|
|
- 4: required set<string> tablesUsed,
|
|
- 5: optional string validTxnList,
|
|
- 6: optional i64 materializationTime
|
|
-}
|
|
-
|
|
struct NotificationEventRequest {
|
|
1: required i64 lastEvent,
|
|
2: optional i32 maxEvents,
|