IMPALA-7399: Remove third-party dependencies from junit xml script

The original patch for this Jira relied on a third party python lib
for generating Junit XML output, That proved to be limiting because
setting up the necessary virtualenv across a variety of dev and test
scenarios (private dev environment, jenkins.impala.io, and others)
proved to be confusing and messy.

This update to the script maintains the same functionality and the
same interface, but uses only the python standard library. A symlink
has also been added to Impala/bin for convenience.

Change-Id: I958ee0d8420b6a4197aaf0a7e0538a566332ea97
Reviewed-on: http://gerrit.cloudera.org:8080/11235
Reviewed-by: David Knupp <dknupp@cloudera.com>
Tested-by: David Knupp <dknupp@cloudera.com>

bin/generate_junitxml.py

@@ -0,0 +1 @@
+../lib/python/impala_py_lib/jenkins/generate_junitxml.py

bin/rat_exclude_files.txt

@@ -10,6 +10,7 @@
 */rat_exclude_files.txt
 be/src/testutil/htpasswd
 be/src/testutil/*.key
+bin/generate_junitxml.py
 tests/*/__init__.py
 testdata/common/__init__.py
 fe/src/test/resources/regionservers
@@ -85,6 +86,7 @@ tests/comparison/POSTGRES.txt
 docs/README.md
 docker/README.md
 be/src/thirdparty/pcg-cpp-0.98/README.md
+lib/python/README.md
 
 # http://www.apache.org/legal/src-headers.html: "Test data for which the addition of a
 # source header would cause the tests to fail."

lib/python/README.md

@@ -0,0 +1,9 @@
+# Impala Python Library
+
+A future home for locally pip installable python packages for Impala.
+
+# Installation
+
+```
+$ pip install -e $IMPALA_HOME/lib/python
+```

lib/python/impala_py_lib/jenkins/generate_junitxml.py

@@ -24,55 +24,157 @@ easier triaging of build and setup errors.
 import argparse
 import errno
 import os
-import pytz
 import textwrap
+from xml.dom import minidom
+from xml.etree import ElementTree as ET
 
 from datetime import datetime as dt
-from junit_xml import TestSuite, TestCase
 
 IMPALA_HOME = os.getenv('IMPALA_HOME', '.')
+SCRIPT_NAME, _ = os.path.splitext(os.path.basename(__file__))
 
 
-def get_xml_content(file_or_string=None):
+class JunitReport(object):
+  """A Junit XML style report parseable by Jenkins for reporting build status.
+
+  Generally, a caller who invokes this script doesn't need to do anything
+  more than supply the necessary command line parameters. The JunitReport
+  class is instantiated using those initial inputs, and a timestamped XML
+  file is output to the $IMPALA_HOME/logs/extra_junit_xml_logs/.
+
+  Log files are timestamped, so they will not overwrite previous files containing
+  output of the same step.
   """
-  Derive content for the XML report.
 
-  Args:
-    file_or_string: a path to a file, or a plain string
+  def __init__(self, phase, step, error_msg=None, stdout=None, stderr=None,
+               elapsed_time=0):
 
-  If the supplied parameter is the path to a file, the contents will be inserted
-  into the XML report. If the parameter is just plain string, use that as the
-  content for the report.
-  """
-  if file_or_string is None:
-    content = ''
-  elif os.path.exists(file_or_string):
-    with open(file_or_string, 'r') as f:
-      content = f.read()
-  else:
+    self.root_element = None
+    self.testsuite_element = None
+    self.testcase_element = None
+
+    self.phase = phase
+    self.step = step
+    self.error_msg = error_msg
+    self.stdout = stdout
+    self.stderr = stderr
+    self.elapsed_time = elapsed_time
+    self.utc_time = dt.utcnow()
+
+    self.create_root_element()
+    self.add_testsuite_element()
+    self.add_testcase_element()
+
+    if self.error_msg is not None:
+      self.set_error()
+
+    if self.stdout is not None:
+      self.add_output('out', self.stdout)
+
+    if self.stderr is not None:
+      self.add_output('err', self.stderr)
+
+  def create_root_element(self):
+    """Create the testsuites root element."""
+    self.root_element = ET.Element("testsuites")
+    self.root_element.set("time", "{0:.1f}".format(float(self.elapsed_time)))
+    self.root_element.set("tests", "1")
+    self.root_element.set("failures", "0")
+    self.root_element.set("errors", "0")
+
+  def add_testsuite_element(self):
+    """Create the testsuite element."""
+    self.testsuite_element = ET.SubElement(self.root_element, "testsuite")
+    self.testsuite_element.set(
+      "name", "{}.{}.{}".format(SCRIPT_NAME, self.phase, self.step))
+    self.testsuite_element.set(
+      "timestamp", "{}+00:00".format(self.utc_time.strftime('%Y-%m-%d %H:%M:%S')))
+    self.testsuite_element.set("disabled", "0")
+    self.testsuite_element.set("errors", "0")
+    self.testsuite_element.set("failures", "0")
+    self.testsuite_element.set("skipped", "0")
+    self.testsuite_element.set("tests", "1")
+    self.testsuite_element.set("time", "0")
+    self.testsuite_element.set("file", "None")
+    self.testsuite_element.set("log", "None")
+    self.testsuite_element.set("url", "None")
+
+  def add_testcase_element(self):
+    """Create the testcase element."""
+    self.testcase_element = ET.SubElement(self.testsuite_element, "testcase")
+    self.testcase_element.set("classname", "{}.{}".format(SCRIPT_NAME, self.phase))
+    self.testcase_element.set("name", self.step)
+
+  def set_error(self):
+    """Set an error msg if the step failed, and increment necessary error attributes."""
+    error = ET.SubElement(self.testcase_element, "error")
+    error.set("message", self.error_msg)
+    error.set("type", "error")
+    self.testsuite_element.set("errors", "1")
+    self.root_element.set("errors", "1")
+
+  def add_output(self, output_type, file_or_string):
+    """
+    Add stdout or stderr content to testcase element.
+
+    Args:
+      output_type: [string] either out or err
+      file_or_string: a path to a file containing the content, or a plain string
+    """
+    output = ET.SubElement(self.testcase_element, "system-{}".format(output_type))
+    output.text = JunitReport.get_xml_content(file_or_string)
+
+  def to_file(self, junitxml_logdir='.'):
+    """
+    Create a timestamped XML report file.
+
+    Args:
+      junitxml_logdir: path to directory where the file will be created
+
+    Return:
+      junit_log_file: path to the generated file
+    """
+    filename = '{}.{}.xml'.format(
+        self.testsuite_element.attrib['name'],
+        self.utc_time.strftime('%Y%m%d_%H_%M_%S')
+    )
+    junit_log_file = os.path.join(junitxml_logdir, filename)
+
+    with open(junit_log_file, 'w') as f:
+      f.write(str(self))
+
+    return junit_log_file
+
+  @staticmethod
+  def get_xml_content(file_or_string=None):
+    """
+    Derive additional content for the XML report.
+
+    If the supplied parameter is the path to a file, the contents will be inserted
+    into the XML report. If the parameter is just plain string, use that as the
+    content for the report.
+
+    Args:
+      file_or_string: a path to a file, or a plain string
+
+    Returns:
+      content as a string
+    """
+    if file_or_string is None:
+      content = ''
+    elif os.path.exists(file_or_string):
+      with open(file_or_string, 'r') as f:
+        content = f.read()
+    else:
       content = file_or_string
-  return content
+    return content
 
-
-def generate_xml_file(testsuite, junitxml_logdir='.'):
-  """
-  Create a timestamped XML report file.
-
-  Args:
-    testsuite: junit_xml.TestSuite object
-    junitxml_logdir: path to directory where the file will be created
-
-  Return:
-    junit_log_file: path to the generated file
-  """
-  ts_string = testsuite.timestamp.strftime('%Y%m%d_%H_%M_%S')
-  junit_log_file = os.path.join(junitxml_logdir,
-                                '{}.{}.xml'.format(testsuite.name, ts_string))
-
-  with open(junit_log_file, 'w') as f:
-    TestSuite.to_file(f, [testsuite], prettyprint=True)
-
-  return junit_log_file
+  def __str__(self):
+    """
+    Generate and return a pretty-printable XML string.
+    """
+    root_node_str = minidom.parseString(ET.tostring(self.root_element))
+    return root_node_str.toprettyxml(indent=' ' * 4)
 
 
 def get_options():
@@ -114,7 +216,7 @@ def get_options():
 
 def main():
   """
-  Create a "testcase" for each invocation of the script, and output the results
+  Create a report for each invocation of the script, and output the results
   of the test case to an XML file within $IMPALA_HOME/logs/extra_junit_xml_logs.
   The log file name will use "phase" and "step" values provided on the command
   line to structure the report. The XML report filename will follow the form:
@@ -136,24 +238,15 @@ def main():
       raise
 
   options = get_options()
-  root_name, _ = os.path.splitext(os.path.basename(__file__))
 
-  tc = TestCase(classname='{}.{}'.format(root_name, options.phase),
-                name=options.step,
-                elapsed_sec=options.time,
-                stdout=get_xml_content(options.stdout),
-                stderr=get_xml_content(options.stderr))
+  junit_report = JunitReport(phase=options.phase,
+                             step=options.step,
+                             error_msg=options.error,
+                             stdout=options.stdout,
+                             stderr=options.stderr,
+                             elapsed_time=options.time)
 
-  # Specifying an error message for any step causes the buid to be marked as invalid.
-  if options.error:
-    tc.add_error_info(get_xml_content(options.error))
-    assert tc.is_error()
-
-  testsuite = TestSuite(name='{}.{}.{}'.format(root_name, options.phase, options.step),
-                        timestamp=dt.utcnow().replace(tzinfo=pytz.UTC),
-                        test_cases=[tc])
-
-  xml_report = generate_xml_file(testsuite, junitxml_logdir)
+  xml_report = junit_report.to_file(junitxml_logdir)
   print("Generated: {}".format(xml_report))
 
 

lib/python/requirements.txt

@@ -15,6 +15,4 @@
 # specific language governing permissions and limitations
 # under the License.
 
-junit-xml==1.8
-pytz==2018.5
-six==1.11.0
+# List thirdparty python dependencies for impala_py_lib here