jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-23 21:08:39 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-9627: Update utility scripts for Python 3 (part 2)

Browse Source 
We're starting to see environments where the system Python ('python') is
Python 3. Updates utility and build scripts to work with Python 3, and
updates check-pylint-py3k.sh to check scripts that use system python.

Fixes other issues found during a full build and test run with Python
3.8 as the default for 'python'.

Fixes a impala-shell tip that was supposed to have been two tips (and
had no space after period when they were printed).

Removes out-of-date deploy.py and various Python 2.6 workarounds.

Testing:
- Full build with /usr/bin/python pointed to python3
- run-all-tests passed with python pointed to python3
- ran push_to_asf.py

Change-Id: Idff388aff33817b0629347f5843ec34c78f0d0cb
Reviewed-on: http://gerrit.cloudera.org:8080/19697
Reviewed-by: Michael Smith <michael.smith@cloudera.com>
Tested-by: Michael Smith <michael.smith@cloudera.com>
This commit is contained in:
Michael Smith
2023-03-24 09:31:22 -07:00
parent 5b0864d7b4
commit 0a42185d17
39 changed files with 115 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
docker/monitor.py
View File

@@ -32,6 +32,7 @@
# re.compile(">>> "))
# timeline.create("output.html")
from __future__ import absolute_import, division, print_function
import datetime
import json
import logging
@@ -161,7 +162,7 @@ class ContainerMonitor(object):
# Container may no longer exist.
return None
try:
statcontents = file(os.path.join(dirname, stat)).read()
statcontents = open(os.path.join(dirname, stat)).read()
return statcontents.replace("\n", " ").strip()
except IOError as e:
# Ignore errors; cgroup can disappear on us.
@@ -186,7 +187,7 @@ class ContainerMonitor(object):
self.min_memory_usage_gb = None
self.max_memory_usage_gb = None
with file(self.output_path, "w") as output:
with open(self.output_path, "w") as output:
while self.keep_monitoring:
# Use a single timestamp for a given round of monitoring.
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
@@ -229,7 +230,7 @@ class Timeline(object):
"""
interesting_lines = [
line.strip()
for line in file(container.logfile)
for line in open(container.logfile)
if self.interesting_re.search(line)]
return [(container.name,) + split_timestamp(line) for line in interesting_lines]
@@ -273,15 +274,15 @@ class Timeline(object):
dt = ts - prev_ts
assert type(dt) == float
if dt != 0:
yield ts, container, (user_cpu - prev_user)/dt/USER_HZ,\
(system_cpu - prev_system)/dt/USER_HZ
yield ts, container, (user_cpu - prev_user) // dt // USER_HZ,\
(system_cpu - prev_system) // dt // USER_HZ
prev_by_container[container] = ts, user_cpu, system_cpu
# Now update container totals
for c in self.containers:
if c.id in prev_by_container:
_, u, s = prev_by_container[c.id]
c.total_user_cpu, c.total_system_cpu = u / USER_HZ, s / USER_HZ
c.total_user_cpu, c.total_system_cpu = u // USER_HZ, s // USER_HZ
if c.id in peak_rss_by_container:
c.peak_total_rss = peak_rss_by_container[c.id]
@@ -321,7 +322,7 @@ class Timeline(object):
for c in self.containers:
container_by_id[c.id] = c
for ts, container_id, user, system in self.parse_metrics(file(self.monitor_file)):
for ts, container_id, user, system in self.parse_metrics(open(self.monitor_file)):
container = container_by_id.get(container_id)
if not container:
continue
@@ -337,9 +338,9 @@ class Timeline(object):
metrics_by_container.setdefault(
container.name, []).append((ts - min_ts, user, system))
with file(output, "w") as o:
with open(output, "w") as o:
template_path = os.path.join(os.path.dirname(__file__), "timeline.html.template")
shutil.copyfileobj(file(template_path), o)
shutil.copyfileobj(open(template_path), o)
o.write("\n<script>\nvar data = \n")
json.dump(dict(buildname=self.buildname, timeline=timeline_json,
metrics=metrics_by_container, max_ts=(max_metric_ts - min_ts)), o, indent=2)