IMPALA-11697: Enable SkipIf.not_hdfs tests for Ozone

Convert SkipIf.not_hdfs to SkipIf.not_dfs for tests that require
filesystem semantics, adding more feature test coverage with Ozone.

Creates a separate not_scratch_fs flag for scratch dir tests as they're
not supported with Ozone yet. Filed IMPALA-11730 to address this.

Preserves not_hdfs for a specific test that uses the dfsadmin CLI to put
it in safemode.

Adds sfs_ofs_unsupported for SmallFileSystem tests. This should work for
many of our filesystems based on
ebb1e2fa99/ql/src/java/org/apache/hadoop/hive/ql/io/SingleFileSystem.java (L62-L87). Makes sfs tests work on S3.

Adds hardcoded_uris for IcebergV2 tests where deletes are implemented as
hardcoded URIs in parquet files. Adding a parquet read/write library for
Python is beyond the scope if this patch.

Change-Id: Iafc1dac52d013e74a459fdc4336c26891a256ef1
Reviewed-on: http://gerrit.cloudera.org:8080/19254
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com>
This commit is contained in:
Michael Smith
2022-11-16 14:13:54 -08:00
parent 87e0077255
commit f8443d9828
12 changed files with 68 additions and 50 deletions

View File

@@ -26,6 +26,8 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIf, SkipIfFS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.util.filesystem_utils import get_fs_path
class TestDataErrors(ImpalaTestSuite):
# batch_size of 1 can expose some interesting corner cases at row batch boundaries.
@@ -42,12 +44,13 @@ class TestDataErrors(ImpalaTestSuite):
def get_workload(self):
return 'functional-query'
# Regression test for IMP-633. Added as a part of IMPALA-5198.
@SkipIf.not_hdfs
@SkipIf.not_dfs
class TestHdfsFileOpenFailErrors(ImpalaTestSuite):
@pytest.mark.execute_serially
def test_hdfs_file_open_fail(self):
absolute_location = "/test-warehouse/file_open_fail"
absolute_location = get_fs_path("/test-warehouse/file_open_fail")
create_stmt = \
"create table file_open_fail (x int) location '" + absolute_location + "'"
insert_stmt = "insert into file_open_fail values(1)"
@@ -64,6 +67,7 @@ class TestHdfsFileOpenFailErrors(ImpalaTestSuite):
assert "Failed to open HDFS file" in str(e)
self.client.execute(drop_stmt)
# Test for IMPALA-5331 to verify that the libHDFS API hdfsGetLastExceptionRootCause()
# works.
@SkipIf.not_hdfs
@@ -161,6 +165,7 @@ class TestAvroErrors(TestDataErrors):
vector.get_value('exec_option')['abort_on_error'] = 0
self.run_test_case('DataErrorsTest/avro-errors', vector)
class TestHBaseDataErrors(TestDataErrors):
@classmethod
def add_test_dimensions(cls):