mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
b5714097e0
This change adds a mechanism to collect host resource usage metrics to
profiles. Metric collection can be controlled through a new query option
'RESOURCE_TRACE_RATIO'. It specifies the probability with which metrics
collection will be enabled. Collection always happens per query for all
executors that run one or more fragment instances of the query.
This mechanism adds a new time series counter class that collects all
measured values and does not re-sample them. It will re-sample values
when printing them into a string profile, preserving up to 64 values,
but Thrift profiles will contain the full list of values.
We add a new section "Per Node Profiles" to the profile to store and
show these values:
Per Node Profiles:
lv-desktop:22000:
CpuIoWaitPercentage (500.000ms): 0, 0
CpuSysPercentage (500.000ms): 1, 1
CpuUserPercentage (500.000ms): 4, 0
- ScratchBytesRead: 0
- ScratchBytesWritten: 0
- ScratchFileUsedBytes: 0
- ScratchReads: 0 (0)
- ScratchWrites: 0 (0)
- TotalEncryptionTime: 0.000ns
- TotalReadBlockTime: 0.000ns
This change also uses the aforementioned mechanism to collect CPU usage
metrics (user, system, and IO wait time).
A future change can then add a tool to decode a Thrift profile and plot
the contained usage metrics, e.g. using matplotlib (IMPALA-8123). Such a
tool is not included in this change because it will require some
reworking of the python dependencies.
This change also includes a few minor improvements to make the resulting
code more readable:
- Extend the PeriodicCounterUpdater to call functions to update global
metrics before updating the counters.
- Expose the scratch profile within the per node resource usage section.
- Improve documentation of the profile counter classes.
- Remove synchronization from StreamingSampler.
- Remove a few pieces of dead code that otherwise would have required
updates.
- Factor some code for profile decoding into the Impala python library
Testing: This change contains a unit test for the system level metrics
collection and e2e tests for the profile changes.
Change-Id: I3aedc20c553ab8d7ed50f72a1a936eba151487d9
Reviewed-on: http://gerrit.cloudera.org:8080/12069
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
#!/usr/bin/env impala-python
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# This file contains library functions to decode and access Impala query profiles.
|
|
|
|
import base64
|
|
import datetime
|
|
import zlib
|
|
from thrift.protocol import TCompactProtocol
|
|
from thrift.TSerialization import deserialize
|
|
from RuntimeProfile.ttypes import TRuntimeProfileTree
|
|
|
|
|
|
def decode_profile_line(line):
|
|
space_separated = line.split(" ")
|
|
if len(space_separated) == 3:
|
|
ts = int(space_separated[0])
|
|
print datetime.datetime.fromtimestamp(ts / 1000.0).isoformat(), space_separated[1]
|
|
base64_encoded = space_separated[2]
|
|
elif len(space_separated) == 1:
|
|
base64_encoded = space_separated[0]
|
|
else:
|
|
raise Exception("Unexpected line: " + line)
|
|
possibly_compressed = base64.b64decode(base64_encoded)
|
|
# Handle both compressed and uncompressed Thrift profiles
|
|
try:
|
|
thrift = zlib.decompress(possibly_compressed)
|
|
except zlib.error:
|
|
thrift = possibly_compressed
|
|
|
|
tree = TRuntimeProfileTree()
|
|
deserialize(tree, thrift, protocol_factory=TCompactProtocol.TCompactProtocolFactory())
|
|
tree.validate()
|
|
|
|
return tree
|