mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
b6c20b2a40
Allow Impala to start only with a running HMS (and no additional services like HDFS, HBase, Hive, YARN) and use the local file system. Skip all tests that need these services, use HDFS caching or assume that multiple impalads are running. To run Impala with the local filesystem, set TARGET_FILESYSTEM to 'local' and WAREHOUSE_LOCATION_PREFIX to a location on the local filesystem where the current user has permissions since this is the location where the test data will be extracted. Test coverage (with core strategy) in comparison with HDFS and S3: HDFS 1348 tests passed S3 1157 tests passed Local Filesystem 1161 tests passed Change-Id: Ic9718c7e0307273382b1cc6baf203ff2fb2acd03 Reviewed-on: http://gerrit.cloudera.org:8080/1352 Reviewed-by: Alex Behm <alex.behm@cloudera.com> Tested-by: Internal Jenkins Readability: Alex Behm <alex.behm@cloudera.com>
96 lines
3.6 KiB
Python
96 lines
3.6 KiB
Python
# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# Tests Impala properly handles errors when reading and writing data.
|
|
|
|
from tests.common.impala_test_suite import ImpalaTestSuite
|
|
from tests.common.skip import SkipIfS3, SkipIfLocal
|
|
import pytest
|
|
|
|
class TestDataErrors(ImpalaTestSuite):
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestDataErrors, cls).add_test_dimensions()
|
|
|
|
@classmethod
|
|
def get_workload(self):
|
|
return 'functional-query'
|
|
|
|
|
|
@SkipIfS3.qualified_path
|
|
class TestHdfsScanNodeErrors(TestDataErrors):
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestHdfsScanNodeErrors, cls).add_test_dimensions()
|
|
# Only run on delimited text with no compression.
|
|
cls.TestMatrix.add_constraint(lambda v:\
|
|
v.get_value('table_format').file_format != 'hbase' and
|
|
v.get_value('table_format').file_format != 'parquet')
|
|
|
|
def test_hdfs_scan_node_errors(self, vector):
|
|
# TODO: Run each test with abort_on_error=0 and abort_on_error=1.
|
|
vector.get_value('exec_option')['abort_on_error'] = 0
|
|
if (vector.get_value('table_format').file_format != 'text'):
|
|
pytest.xfail("Expected results differ across file formats")
|
|
self.run_test_case('DataErrorsTest/hdfs-scan-node-errors', vector)
|
|
|
|
|
|
@SkipIfS3.qualified_path
|
|
@SkipIfLocal.qualified_path
|
|
class TestHdfsSeqScanNodeErrors(TestHdfsScanNodeErrors):
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestHdfsSeqScanNodeErrors, cls).add_test_dimensions()
|
|
cls.TestMatrix.add_constraint(lambda v:\
|
|
v.get_value('table_format').file_format == 'seq')
|
|
|
|
def test_hdfs_seq_scan_node_errors(self, vector):
|
|
vector.get_value('exec_option')['abort_on_error'] = 0
|
|
self.run_test_case('DataErrorsTest/hdfs-sequence-scan-errors', vector)
|
|
|
|
|
|
@SkipIfS3.qualified_path
|
|
class TestHdfsRcFileScanNodeErrors(TestHdfsScanNodeErrors):
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestHdfsRcFileScanNodeErrors, cls).add_test_dimensions()
|
|
cls.TestMatrix.add_constraint(lambda v:\
|
|
v.get_value('table_format').file_format == 'rc')
|
|
|
|
def test_hdfs_rcfile_scan_node_errors(self, vector):
|
|
vector.get_value('exec_option')['abort_on_error'] = 0
|
|
self.run_test_case('DataErrorsTest/hdfs-rcfile-scan-node-errors', vector)
|
|
|
|
|
|
class TestHBaseDataErrors(TestDataErrors):
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestHBaseDataErrors, cls).add_test_dimensions()
|
|
|
|
# Only run on hbase.
|
|
cls.TestMatrix.add_constraint(lambda v:\
|
|
v.get_value('table_format').file_format == 'hbase' and\
|
|
v.get_value('table_format').compression_codec == 'none')
|
|
|
|
def test_hbase_scan_node_errors(self, vector):
|
|
pytest.xfail("hbasealltypeserror doesn't seem to return any errors")
|
|
|
|
vector.get_value('exec_option')['abort_on_error'] = 0
|
|
self.run_test_case('DataErrorsTest/hbase-scan-node-errors', vector)
|
|
|
|
def test_hbase_insert_errors(self, vector):
|
|
pytest.xfail("hbasealltypeserror doesn't seem to return any errors")
|
|
vector.get_value('exec_option')['abort_on_error'] = 0
|
|
self.run_test_case('DataErrorsTest/hbase-insert-errors', vector)
|