mirror of
https://github.com/apache/impala.git
synced 2026-01-07 09:02:19 -05:00
507bd8be7e
This is intended to be merged at the same time as Part 2 but is
separated out to make the change more reviewable. Part 2 assumes
that it does not need special logic to handle this mode (e.g.
because the old aggs and joins don't use reservation).
Disable the --enable_partitioned_{aggregation,hash_join} options
and remove all product and test code associated with them.
Change-Id: I5ce2236d37c0ced188a4a81f7e00d4b8ac98e7e9
Reviewed-on: http://gerrit.cloudera.org:8080/7102
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins
115 lines
4.8 KiB
Python
115 lines
4.8 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Tests queries with the MT_DOP query option.
|
|
|
|
import pytest
|
|
|
|
from copy import deepcopy
|
|
from tests.common.impala_test_suite import ImpalaTestSuite
|
|
from tests.common.kudu_test_suite import KuduTestSuite
|
|
from tests.common.test_vector import ImpalaTestDimension
|
|
|
|
# COMPUTE STATS on Parquet tables automatically sets MT_DOP=4, so include
|
|
# the value 0 to cover the non-MT path as well.
|
|
MT_DOP_VALUES = [0, 1, 2, 8]
|
|
|
|
class TestMtDop(ImpalaTestSuite):
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestMtDop, cls).add_test_dimensions()
|
|
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('mt_dop', *MT_DOP_VALUES))
|
|
|
|
@classmethod
|
|
def get_workload(cls):
|
|
return 'functional-query'
|
|
|
|
def test_mt_dop(self, vector):
|
|
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
|
|
self.run_test_case('QueryTest/mt-dop', vector)
|
|
|
|
def test_compute_stats(self, vector, unique_database):
|
|
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
|
|
file_format = vector.get_value('table_format').file_format
|
|
fq_table_name = "%s.mt_dop" % unique_database
|
|
|
|
# Different formats need different DDL for creating a test table, and also
|
|
# have different expected results for compute stats.
|
|
expected_results = None
|
|
if file_format == 'kudu':
|
|
# CREATE TABLE LIKE is currently not supported for Kudu tables.
|
|
self.execute_query("create external table %s stored as kudu "
|
|
"tblproperties('kudu.table_name'='impala::functional_kudu.alltypes')"
|
|
% fq_table_name)
|
|
expected_results = "Updated 1 partition(s) and 13 column(s)."
|
|
elif file_format == 'hbase':
|
|
self.execute_query(
|
|
"create external table %s like functional_hbase.alltypes" % fq_table_name)
|
|
expected_results = "Updated 1 partition(s) and 13 column(s)."
|
|
else:
|
|
# Create a second table in the same format pointing to the same data files.
|
|
# This function switches to the format-specific DB in 'vector'.
|
|
table_loc = self._get_table_location("alltypes", vector)
|
|
self.execute_query_using_client(self.client,
|
|
"create external table %s like alltypes location '%s'"
|
|
% (fq_table_name, table_loc), vector)
|
|
# Recover partitions for HDFS tables.
|
|
self.execute_query("alter table %s recover partitions" % fq_table_name)
|
|
expected_results = "Updated 24 partition(s) and 11 column(s)."
|
|
|
|
results = self.execute_query("compute stats %s" % fq_table_name,
|
|
vector.get_value('exec_option'))
|
|
assert expected_results in results.data
|
|
|
|
class TestMtDopParquet(ImpalaTestSuite):
|
|
@classmethod
|
|
def get_workload(cls):
|
|
return 'functional-query'
|
|
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestMtDopParquet, cls).add_test_dimensions()
|
|
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('mt_dop', *MT_DOP_VALUES))
|
|
cls.ImpalaTestMatrix.add_constraint(
|
|
lambda v: v.get_value('table_format').file_format == 'parquet')
|
|
|
|
def test_parquet(self, vector):
|
|
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
|
|
self.run_test_case('QueryTest/mt-dop-parquet', vector)
|
|
|
|
@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
|
|
reason='IMPALA-4641')
|
|
def test_parquet_nested(self, vector):
|
|
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
|
|
self.run_test_case('QueryTest/mt-dop-parquet-nested', vector)
|
|
|
|
def test_parquet_filtering(self, vector):
|
|
"""IMPALA-4624: Test that dictionary filtering eliminates row groups correctly."""
|
|
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
|
|
self.run_test_case('QueryTest/parquet-filtering', vector)
|
|
|
|
class TestMtDopKudu(KuduTestSuite):
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestMtDopKudu, cls).add_test_dimensions()
|
|
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('mt_dop', *MT_DOP_VALUES))
|
|
|
|
def test_kudu(self, vector, unique_database):
|
|
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
|
|
self.run_test_case('QueryTest/mt-dop-kudu', vector, use_db=unique_database)
|
|
|