mirror of
https://github.com/apache/impala.git
synced 2025-12-30 12:02:10 -05:00
ffa7829b70
For files that have a Cloudera copyright (and no other copyright notice), make changes to follow the ASF source file header policy here: http://www.apache.org/legal/src-headers.html#headers Specifically: 1) Remove the Cloudera copyright. 2) Modify NOTICE.txt according to http://www.apache.org/legal/src-headers.html#notice to follow that format and add a line for Cloudera. 3) Replace or add the existing ASF license text with the one given on the website. Much of this change was automatically generated via: git grep -li 'Copyright.*Cloudera' > modified_files.txt cat modified_files.txt | xargs perl -n -i -e 'print unless m#Copyright.*Cloudera#i;' cat modified_files_txt | xargs fix_apache_license.py [1] Some manual fixups were performed following those steps, especially when license text was completely missing from the file. [1] https://gist.github.com/anonymous/ff71292094362fc5c594 with minor modification to ORIG_LICENSE to match Impala's license text. Change-Id: I2e0bd8420945b953e1b806041bea4d72a3943d86 Reviewed-on: http://gerrit.cloudera.org:8080/3779 Reviewed-by: Dan Hecht <dhecht@cloudera.com> Tested-by: Internal Jenkins
115 lines
4.4 KiB
Python
115 lines
4.4 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
import os
|
|
import fnmatch
|
|
import re
|
|
|
|
from tests.performance.query import Query
|
|
from tests.util.test_file_parser import parse_query_test_file
|
|
|
|
class Workload(object):
|
|
"""Represents a workload.
|
|
|
|
A workload is the internal representation for the set of queries on a dataset. It
|
|
consists of the dataset name, and a mapping of query names to query strings.
|
|
|
|
Args:
|
|
name (str): workload name. (Eg. tpch)
|
|
query_name_filters (list of str): List of regular expressions used for matching query
|
|
names
|
|
|
|
Attributes:
|
|
name (str): workload name (Eg. tpch)
|
|
_query_map (dict): contains a query name -> string mapping; mapping of query name to
|
|
section (ex. "TPCH-Q10" -> "select * from...")
|
|
"""
|
|
|
|
WORKLOAD_DIR = os.environ['IMPALA_WORKLOAD_DIR']
|
|
|
|
def __init__(self, name, query_name_filters=None):
|
|
self._name = name
|
|
self._query_map = dict()
|
|
# Build the query name -> string mapping in the c'tor. We want to fail fast and early
|
|
# if the user input is bad.
|
|
self._validate_and_load(query_name_filters)
|
|
|
|
@property
|
|
def name(self):
|
|
return self._name
|
|
|
|
@property
|
|
def query_map(self):
|
|
return self._query_map
|
|
|
|
def _validate_and_load(self, query_name_filters):
|
|
"""Validates that the Workload is legal."""
|
|
query_name_filters = map(str.strip, query_name_filters) if query_name_filters else []
|
|
self._base_dir = os.path.join(Workload.WORKLOAD_DIR, self._name, 'queries')
|
|
# Check whether the workload name corresponds to an existing directory.
|
|
if not os.path.isdir(self._base_dir):
|
|
raise ValueError("Workload %s not found in %s" % (self._name, self._base_dir))
|
|
sections = list()
|
|
# Parse all queries files for the given workload.
|
|
for file_name in self._list_query_files():
|
|
sections.extend(parse_query_test_file(file_name))
|
|
# If the user has specified query names, check whether all the user specified queries
|
|
# exist in the query files.
|
|
all_query_names = [s['QUERY_NAME'] for s in sections if s['QUERY_NAME'].strip()]
|
|
regex = re.compile(r'|'.join(['^%s$' % n for n in query_name_filters]), re.I)
|
|
matched_query_names = filter(lambda x: re.match(regex, x), all_query_names)
|
|
assert len(matched_query_names) > 0, "No matching queries found for %s" % self._name
|
|
# Filter the sections based on the queries the user wants.
|
|
sections = filter(lambda x: x['QUERY_NAME'] in matched_query_names, sections)
|
|
# Add the filtered queries to the query map
|
|
for section in sections:
|
|
self._query_map[section['QUERY_NAME']] = section['QUERY']
|
|
|
|
def _list_query_files(self):
|
|
"""Return a list of all the .test files that contain queries"""
|
|
query_files = list()
|
|
for root, dirs, file_names in os.walk(self._base_dir):
|
|
for file_name in fnmatch.filter(file_names, '*.test'):
|
|
query_files.append(os.path.join(root, file_name))
|
|
assert len(query_files) > 0, "No Query Files found in %s" % self._base_dir
|
|
return query_files
|
|
|
|
def construct_queries(self, test_vector, scale_factor):
|
|
"""Transform a query map into a list of query objects.
|
|
|
|
Transform all the queries in the workload's query map to query objects based on the
|
|
input test vector and scale factor.
|
|
|
|
Args:
|
|
test_vector (?): query vector
|
|
scale_factor (str): eg. "300gb"
|
|
|
|
Returns:
|
|
(list of Query): these will be consumed by ?
|
|
"""
|
|
|
|
queries = list()
|
|
for query_name, query_str in self._query_map.iteritems():
|
|
queries.append(Query(name=query_name,
|
|
query_str=query_str,
|
|
workload=self._name,
|
|
scale_factor=scale_factor,
|
|
test_vector=test_vector))
|
|
return queries
|
|
|