mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
0286142b67
The original patch for this Jira relied on a third party python lib for generating Junit XML output, That proved to be limiting because setting up the necessary virtualenv across a variety of dev and test scenarios (private dev environment, jenkins.impala.io, and others) proved to be confusing and messy. This update to the script maintains the same functionality and the same interface, but uses only the python standard library. A symlink has also been added to Impala/bin for convenience. Change-Id: I958ee0d8420b6a4197aaf0a7e0538a566332ea97 Reviewed-on: http://gerrit.cloudera.org:8080/11235 Reviewed-by: David Knupp <dknupp@cloudera.com> Tested-by: David Knupp <dknupp@cloudera.com>
255 lines
8.7 KiB
Python
Executable File
255 lines
8.7 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
A script for generating arbitrary junit XML reports while building Impala.
|
|
These files will be consumed by jenkins.impala.io to generate reports for
|
|
easier triaging of build and setup errors.
|
|
"""
|
|
import argparse
|
|
import errno
|
|
import os
|
|
import textwrap
|
|
from xml.dom import minidom
|
|
from xml.etree import ElementTree as ET
|
|
|
|
from datetime import datetime as dt
|
|
|
|
IMPALA_HOME = os.getenv('IMPALA_HOME', '.')
|
|
SCRIPT_NAME, _ = os.path.splitext(os.path.basename(__file__))
|
|
|
|
|
|
class JunitReport(object):
|
|
"""A Junit XML style report parseable by Jenkins for reporting build status.
|
|
|
|
Generally, a caller who invokes this script doesn't need to do anything
|
|
more than supply the necessary command line parameters. The JunitReport
|
|
class is instantiated using those initial inputs, and a timestamped XML
|
|
file is output to the $IMPALA_HOME/logs/extra_junit_xml_logs/.
|
|
|
|
Log files are timestamped, so they will not overwrite previous files containing
|
|
output of the same step.
|
|
"""
|
|
|
|
def __init__(self, phase, step, error_msg=None, stdout=None, stderr=None,
|
|
elapsed_time=0):
|
|
|
|
self.root_element = None
|
|
self.testsuite_element = None
|
|
self.testcase_element = None
|
|
|
|
self.phase = phase
|
|
self.step = step
|
|
self.error_msg = error_msg
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
self.elapsed_time = elapsed_time
|
|
self.utc_time = dt.utcnow()
|
|
|
|
self.create_root_element()
|
|
self.add_testsuite_element()
|
|
self.add_testcase_element()
|
|
|
|
if self.error_msg is not None:
|
|
self.set_error()
|
|
|
|
if self.stdout is not None:
|
|
self.add_output('out', self.stdout)
|
|
|
|
if self.stderr is not None:
|
|
self.add_output('err', self.stderr)
|
|
|
|
def create_root_element(self):
|
|
"""Create the testsuites root element."""
|
|
self.root_element = ET.Element("testsuites")
|
|
self.root_element.set("time", "{0:.1f}".format(float(self.elapsed_time)))
|
|
self.root_element.set("tests", "1")
|
|
self.root_element.set("failures", "0")
|
|
self.root_element.set("errors", "0")
|
|
|
|
def add_testsuite_element(self):
|
|
"""Create the testsuite element."""
|
|
self.testsuite_element = ET.SubElement(self.root_element, "testsuite")
|
|
self.testsuite_element.set(
|
|
"name", "{}.{}.{}".format(SCRIPT_NAME, self.phase, self.step))
|
|
self.testsuite_element.set(
|
|
"timestamp", "{}+00:00".format(self.utc_time.strftime('%Y-%m-%d %H:%M:%S')))
|
|
self.testsuite_element.set("disabled", "0")
|
|
self.testsuite_element.set("errors", "0")
|
|
self.testsuite_element.set("failures", "0")
|
|
self.testsuite_element.set("skipped", "0")
|
|
self.testsuite_element.set("tests", "1")
|
|
self.testsuite_element.set("time", "0")
|
|
self.testsuite_element.set("file", "None")
|
|
self.testsuite_element.set("log", "None")
|
|
self.testsuite_element.set("url", "None")
|
|
|
|
def add_testcase_element(self):
|
|
"""Create the testcase element."""
|
|
self.testcase_element = ET.SubElement(self.testsuite_element, "testcase")
|
|
self.testcase_element.set("classname", "{}.{}".format(SCRIPT_NAME, self.phase))
|
|
self.testcase_element.set("name", self.step)
|
|
|
|
def set_error(self):
|
|
"""Set an error msg if the step failed, and increment necessary error attributes."""
|
|
error = ET.SubElement(self.testcase_element, "error")
|
|
error.set("message", self.error_msg)
|
|
error.set("type", "error")
|
|
self.testsuite_element.set("errors", "1")
|
|
self.root_element.set("errors", "1")
|
|
|
|
def add_output(self, output_type, file_or_string):
|
|
"""
|
|
Add stdout or stderr content to testcase element.
|
|
|
|
Args:
|
|
output_type: [string] either out or err
|
|
file_or_string: a path to a file containing the content, or a plain string
|
|
"""
|
|
output = ET.SubElement(self.testcase_element, "system-{}".format(output_type))
|
|
output.text = JunitReport.get_xml_content(file_or_string)
|
|
|
|
def to_file(self, junitxml_logdir='.'):
|
|
"""
|
|
Create a timestamped XML report file.
|
|
|
|
Args:
|
|
junitxml_logdir: path to directory where the file will be created
|
|
|
|
Return:
|
|
junit_log_file: path to the generated file
|
|
"""
|
|
filename = '{}.{}.xml'.format(
|
|
self.testsuite_element.attrib['name'],
|
|
self.utc_time.strftime('%Y%m%d_%H_%M_%S')
|
|
)
|
|
junit_log_file = os.path.join(junitxml_logdir, filename)
|
|
|
|
with open(junit_log_file, 'w') as f:
|
|
f.write(str(self))
|
|
|
|
return junit_log_file
|
|
|
|
@staticmethod
|
|
def get_xml_content(file_or_string=None):
|
|
"""
|
|
Derive additional content for the XML report.
|
|
|
|
If the supplied parameter is the path to a file, the contents will be inserted
|
|
into the XML report. If the parameter is just plain string, use that as the
|
|
content for the report.
|
|
|
|
Args:
|
|
file_or_string: a path to a file, or a plain string
|
|
|
|
Returns:
|
|
content as a string
|
|
"""
|
|
if file_or_string is None:
|
|
content = ''
|
|
elif os.path.exists(file_or_string):
|
|
with open(file_or_string, 'r') as f:
|
|
content = f.read()
|
|
else:
|
|
content = file_or_string
|
|
return content
|
|
|
|
def __str__(self):
|
|
"""
|
|
Generate and return a pretty-printable XML string.
|
|
"""
|
|
root_node_str = minidom.parseString(ET.tostring(self.root_element))
|
|
return root_node_str.toprettyxml(indent=' ' * 4)
|
|
|
|
|
|
def get_options():
|
|
"""Parse and return command line options."""
|
|
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
|
|
# Required options
|
|
parser.add_argument("--phase",
|
|
default="buildall",
|
|
help="General build phase or script.")
|
|
parser.add_argument("--step",
|
|
required=True,
|
|
help=textwrap.dedent(
|
|
"""Specific build step or child script being run.
|
|
Each step must be unique for the given build phase.""")
|
|
)
|
|
parser.add_argument("-t", "--time",
|
|
type=float,
|
|
default=0,
|
|
help="If known, the elapsed time in seconds for this step.")
|
|
parser.add_argument("--stdout",
|
|
help=textwrap.dedent(
|
|
"""Standard output to include in the XML report. Can be
|
|
either a string or the path to a file..""")
|
|
)
|
|
parser.add_argument("--stderr",
|
|
help=textwrap.dedent(
|
|
"""Standard error to include in the XML report. Can be
|
|
either a string or the path to a file.""")
|
|
)
|
|
parser.add_argument("--error",
|
|
help=textwrap.dedent(
|
|
"""If specified, the XML report will mark this as an error.
|
|
This should be a brief explanation for the error.""")
|
|
)
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
"""
|
|
Create a report for each invocation of the script, and output the results
|
|
of the test case to an XML file within $IMPALA_HOME/logs/extra_junit_xml_logs.
|
|
The log file name will use "phase" and "step" values provided on the command
|
|
line to structure the report. The XML report filename will follow the form:
|
|
|
|
junitxml_logger.<phase>.<step>.<time_stamp>.xml
|
|
|
|
Phase can be repeated in a given test run, but the step leaf node, which is
|
|
equivalent to a "test case", must be unique within each phase.
|
|
"""
|
|
junitxml_logdir = os.path.join(IMPALA_HOME, 'logs', 'extra_junit_xml_logs')
|
|
|
|
# The equivalent of mkdir -p
|
|
try:
|
|
os.makedirs(junitxml_logdir)
|
|
except OSError as e:
|
|
if e.errno == errno.EEXIST and os.path.isdir(junitxml_logdir):
|
|
pass
|
|
else:
|
|
raise
|
|
|
|
options = get_options()
|
|
|
|
junit_report = JunitReport(phase=options.phase,
|
|
step=options.step,
|
|
error_msg=options.error,
|
|
stdout=options.stdout,
|
|
stderr=options.stderr,
|
|
elapsed_time=options.time)
|
|
|
|
xml_report = junit_report.to_file(junitxml_logdir)
|
|
print("Generated: {}".format(xml_report))
|
|
|
|
|
|
if "__main__" == __name__:
|
|
main()
|