mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
234d641d7b
This adds support for Redhat 9 / Ubuntu 22. It updates to a newer toolchain that has those builds, and it adds supporting code in bootstrap_system.sh. Redhat 9 and Ubuntu 22 use python = python3, which requires various changes to build scripts and tests. Ubuntu 22 uses Python 3.10, which deprecates certain ssl.PROTOCOL_TLS, so this adapts test_client_ssl.py to that change until it can be fully addressed in IMPALA-12219. Various OpenSSL methods have been deprecated. As a workaround until these can be addressed properly, this specifies -Wno-deprecated-declarations. This can be removed once the code is adapted to the non-deprecated APIs in IMPALA-12226. Impala crashes with tcmalloc errors unless we update to a newer gperftools, so this moves to gperftools 2.10. gperftools changed the default for tcmalloc.aggressive_memory_decommit to off, so this adapts our code to set it for backend tests. The gperftools upgrade does not show any performance regression: +----------+-----------------------+---------+------------+------------+----------------+ | Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) | +----------+-----------------------+---------+------------+------------+----------------+ | TPCH(42) | parquet / none / none | 3.08 | -0.64% | 2.20 | -0.37% | +----------+-----------------------+---------+------------+------------+----------------+ With newer Python versions, the impala-virtualenv command fails to create a Python 3 virtualenv. This switches to using Python 3's builtin venv command for Python >=3.6. Kudu needed a newer version and LLVM required a couple patches. Testing: - Ran a core job on Ubuntu 22 and Redhat 9. The tests run to completion without crashing. There are test failures that will be addressed in follow-up JIRAs. - Ran dockerised tests on Ubuntu 22. - Ran dockerised tests on Ubuntu 20 and Rocky 8.5. Change-Id: If1fcdb2f8c635ecd6dc7a8a1db81f5f389c78b86 Reviewed-on: http://gerrit.cloudera.org:8080/20073 Reviewed-by: Michael Smith <michael.smith@cloudera.com> Tested-by: Joe McDonnell <joemcdonnell@cloudera.com>
73 lines
2.8 KiB
Python
Executable File
73 lines
2.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
Some tests that produce JUnitXML include tests that did not run (i.e. status="notrun").
|
|
This script walks through the JUnitXML and removes these elements.
|
|
"""
|
|
from __future__ import absolute_import, division, print_function
|
|
from optparse import OptionParser
|
|
from xml.etree import ElementTree as ET
|
|
from xml.dom import minidom
|
|
|
|
|
|
def junitxml_prune_notrun(junitxml_filename):
|
|
tree = ET.ElementTree()
|
|
root = tree.parse(junitxml_filename)
|
|
|
|
for testsuite in root.findall("testsuite"):
|
|
# Get the list of notrun tests
|
|
notrun_testcases = []
|
|
for testcase in testsuite:
|
|
status = testcase.attrib["status"]
|
|
if status == 'notrun':
|
|
notrun_testcases.append(testcase)
|
|
# Get the total number of tests
|
|
num_tests = int(testsuite.attrib["tests"])
|
|
|
|
# There are two cases.
|
|
# 1. No test from the testsuite ran. The whole testsuite can be pruned.
|
|
# 2. Some test from the testsuite ran. The individual testcases can be pruned.
|
|
if len(notrun_testcases) == num_tests:
|
|
# Remove whole testsuite
|
|
root.remove(testsuite)
|
|
else:
|
|
# Remote individual testcases.
|
|
for testcase in notrun_testcases:
|
|
testsuite.remove(testcase)
|
|
# Fixup the total number of tests
|
|
testsuite.attrib["tests"] = str(num_tests - len(notrun_testcases))
|
|
# Write out the pruned JUnitXML
|
|
# An XML declaration is optional, but it is nice to have. ElementTree.write() does
|
|
# not support an XML declaration on Python 2.6, so use minidom to write the XML.
|
|
root_node_minidom = minidom.parseString(ET.tostring(root))
|
|
junitxml_string = root_node_minidom.toxml(encoding="utf-8")
|
|
with open(junitxml_filename, "wb") as f:
|
|
f.write(junitxml_string)
|
|
|
|
|
|
def main():
|
|
parser = OptionParser()
|
|
parser.add_option("-f", "--file", dest="filename", help="JUnitXML file to prune")
|
|
options, args = parser.parse_args()
|
|
junitxml_prune_notrun(options.filename)
|
|
|
|
|
|
if __name__ == "__main__": main()
|