mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
45995e6892
Defines SystemTable which are in-memory tables that can provide access to Impala state. Adds the 'impala_query_live' to the database 'sys', which already exists for 'sys.impala_query_log'. Implements the 'impala_query_live' table to view active queries across all coordinators sharing the same statestore. SystemTables create new SystemTableScanNodes for their scan node implementation. When computing scan range locations, SystemTableScanNodes creates a scan range for each in the cluster (identified via ClusterMembershipMgr). This produces a plan that looks like: Query: explain select * from sys.impala_query_live +------------------------------------------------------------+ | Explain String | +------------------------------------------------------------+ | Max Per-Host Resource Reservation: Memory=4.00MB Threads=2 | | Per-Host Resource Estimates: Memory=11MB | | WARNING: The following tables are missing relevant table | | and/or column statistics. | | sys.impala_query_live | | | | PLAN-ROOT SINK | | | | | 01:EXCHANGE [UNPARTITIONED] | | | | | 00:SCAN SYSTEM_TABLE [sys.impala_query_live] | | row-size=72B cardinality=20 | +------------------------------------------------------------+ Impala's scheduler checks for whether the query contains fragments that can be scheduled on coordinators, and if present includes an ExecutorGroup containing all coordinators. These are used to schedule scan ranges that are flagged as 'use_coordinator', allowing SystemTableScanNodes to be scheduled on dedicated coordinators and outside the selected executor group. Execution will pull data from ImpalaServer on the backend via a SystemTableScanner implementation based on table name. In the query profile, SYSTEM_TABLE_SCAN_NODE includes ActiveQueryCollectionTime and PendingQueryCollectionTime to track time spent collecting QueryState from ImpalaServer. Grants QueryScanner private access to ImpalaServer, identical to how ImpalaHttpHandler access internal server state. Adds custom cluster tests for impala_query_live, and unit tests for changes to planner and scheduler. Change-Id: Ie2f9a449f0e5502078931e7f1c5df6e0b762c743 Reviewed-on: http://gerrit.cloudera.org:8080/20762 Reviewed-by: Jason Fehr <jfehr@cloudera.com> Reviewed-by: Riza Suminto <riza.suminto@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
86 lines
2.9 KiB
Protocol Buffer
86 lines
2.9 KiB
Protocol Buffer
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
syntax="proto2";
|
|
|
|
package impala;
|
|
|
|
import "common.proto";
|
|
|
|
// Specification of a subsection of a single HDFS file. Corresponds to THdfsFileSpilt and
|
|
// should be kept in sync with it.
|
|
message HdfsFileSplitPB {
|
|
// File name (not the full path). The path is assumed to be relative to the
|
|
// 'location' of the THdfsPartition referenced by partition_id.
|
|
optional string relative_path = 1;
|
|
|
|
// Starting offset.
|
|
optional int64 offset = 2;
|
|
|
|
// Length of split.
|
|
optional int64 length = 3;
|
|
|
|
// ID of partition within the THdfsTable associated with this scan node.
|
|
optional int64 partition_id = 4;
|
|
|
|
// Total size of the hdfs file.
|
|
optional int64 file_length = 5;
|
|
|
|
// Compression type of the hdfs file.
|
|
optional CompressionTypePB file_compression = 6;
|
|
|
|
// Last modified time of the file.
|
|
optional int64 mtime = 7;
|
|
|
|
// Whether this file is erasure-coded.
|
|
optional bool is_erasure_coded = 8;
|
|
|
|
// Hash of the partition's path. This must be hashed with a hash algorithm that is
|
|
// consistent across different processes and machines. This is currently using
|
|
// Java's String.hashCode(), which is consistent. For testing purposes, this can use
|
|
// any consistent hash.
|
|
optional int32 partition_path_hash = 9;
|
|
|
|
// The absolute path of the file, it's used only when data files are outside of
|
|
// the Iceberg table location (IMPALA-11507).
|
|
optional string absolute_path = 10;
|
|
|
|
// Whether this file is encrypted.
|
|
optional bool is_encrypted = 11;
|
|
}
|
|
|
|
// Key range for single THBaseScanNode. Corresponds to THBaseKeyRange and should be kept
|
|
// in sync with it.
|
|
message HBaseKeyRangePB {
|
|
// Inclusive
|
|
optional string startKey = 1;
|
|
|
|
// Exclusive
|
|
optional string stopKey = 2;
|
|
}
|
|
|
|
// Specification of an individual data range which is held in its entirety by a storage
|
|
// server. Corresponds to TScanRange and should be kept in sync with it.
|
|
message ScanRangePB {
|
|
// One of these must be set for every ScanRangePB.
|
|
optional HdfsFileSplitPB hdfs_file_split = 1;
|
|
optional HBaseKeyRangePB hbase_key_range = 2;
|
|
optional bytes kudu_scan_token = 3;
|
|
optional bytes file_metadata = 4;
|
|
optional bool is_system_scan = 5;
|
|
}
|