jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2026-01-10 18:00:14 -05:00
Code Issues Projects Releases Wiki Activity
Files
616eb2fccecef6ff17b4e3fd48e1908aaae7726e
impala/tests/common/skip.py
Sailesh Mukil ed7f5ebf53 IMPALA-1878: Support INSERT and LOAD DATA on S3 and between filesystems 
Previously Impala disallowed LOAD DATA and INSERT on S3. This patch
functionally enables LOAD DATA and INSERT on S3 without making major
changes for the sake of improving performance over S3. This patch also
enables both INSERT and LOAD DATA between file systems.

S3 does not support the rename operation, so the staged files in S3
are copied instead of renamed, which contributes to the slow
performance on S3.

The FinalizeSuccessfulInsert() function now does not make any
underlying assumptions of the filesystem it is on and works across
all supported filesystems. This is done by adding a full URI field to
the base directory for a partition in the TInsertPartitionStatus.
Also, the HdfsOp class now does not assume a single filesystem and
gets connections to the filesystems based on the URI of the file it
is operating on.

Added a python S3 client called 'boto3' to access S3 from the python
tests. A new class called S3Client is introduced which creates
wrappers around the boto3 functions and have the same function
signatures as PyWebHdfsClient by deriving from a base abstract class
BaseFileSystem so that they can be interchangeably through a
'generic_client'. test_load.py is refactored to use this generic
client. The ImpalaTestSuite setup creates a client according to the
TARGET_FILESYSTEM environment variable and assigns it to the
'generic_client'.

P.S: Currently, the test_load.py runs 4x slower on S3 than on
HDFS. Performance needs to be improved in future patches. INSERT
performance is slower than on HDFS too. This is mainly because of an
extra copy that happens between staging and the final location of a
file. However, larger INSERTs come closer to HDFS permformance than
smaller inserts.

ACLs are not taken care of for S3 in this patch. It is something
that still needs to be discussed before implementing.

Change-Id: I94e15ad67752dce21c9b7c1dced6e114905a942d
Reviewed-on: http://gerrit.cloudera.org:8080/2574
Reviewed-by: Sailesh Mukil <sailesh@cloudera.com>
Tested-by: Internal Jenkins
2016-05-12 14:17:49 -07:00

134 lines
6.1 KiB
Python
Raw Blame History

# Copyright (c) 2015 Cloudera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Impala py.test skipif markers. When a test can't be run against S3,
# choose the appropriate reason (or add a new one if needed) and
# annotate the class or test routine with the marker.
#
import re
import os
import pytest
from functools import partial
from tests.util.filesystem_utils import IS_DEFAULT_FS, IS_S3, IS_ISILON, IS_LOCAL
class SkipIfS3:
# These ones are skipped due to product limitations.
caching = pytest.mark.skipif(IS_S3, reason="SET CACHED not implemented for S3")
hive = pytest.mark.skipif(IS_S3, reason="Hive doesn't work with S3")
hdfs_block_size = pytest.mark.skipif(IS_S3, reason="S3 uses it's own block size")
hdfs_acls = pytest.mark.skipif(IS_S3, reason="HDFS acls are not supported on S3")
jira = partial(pytest.mark.skipif, IS_S3)
hdfs_encryption = pytest.mark.skipif(IS_S3,
reason="HDFS encryption is not supported with S3")
hdfs_purge = pytest.mark.skipif(IS_S3,
reason="PURGE has no effect on S3")
# These ones need test infra work to re-enable.
udfs = pytest.mark.skipif(IS_S3, reason="udas/udfs not copied to S3")
datasrc = pytest.mark.skipif(IS_S3, reason="data sources not copied to S3")
hbase = pytest.mark.skipif(IS_S3, reason="HBase not started with S3")
qualified_path = pytest.mark.skipif(IS_S3,
reason="Tests rely on HDFS qualified paths, IMPALA-1872")
class SkipIf:
# Some tests require a non-default filesystem to be present.
default_fs = pytest.mark.skipif(IS_DEFAULT_FS, reason="Non-default filesystem needed")
skip_hbase = pytest.mark.skipif(pytest.config.option.skip_hbase,
reason="--skip_hbase argument specified")
not_default_fs = pytest.mark.skipif(not IS_DEFAULT_FS,
reason="Default filesystem needed")
kudu_not_supported = pytest.mark.skipif(os.environ["KUDU_IS_SUPPORTED"] == "false",
reason="Kudu is not supported")
not_s3 = pytest.mark.skipif(not IS_S3, reason="S3 Filesystem needed")
class SkipIfIsilon:
caching = pytest.mark.skipif(IS_ISILON, reason="SET CACHED not implemented for Isilon")
hbase = pytest.mark.skipif(IS_ISILON, reason="HBase not tested with Isilon")
hive = pytest.mark.skipif(IS_ISILON, reason="Hive not tested with Isilon")
hdfs_acls = pytest.mark.skipif(IS_ISILON, reason="HDFS acls are not supported on Isilon")
hdfs_block_size = pytest.mark.skipif(IS_ISILON,
reason="Isilon uses its own block size")
hdfs_encryption = pytest.mark.skipif(IS_ISILON,
reason="HDFS encryption is not supported with Isilon")
untriaged = pytest.mark.skipif(IS_ISILON,
reason="This Isilon issue has yet to be triaged.")
jira = partial(pytest.mark.skipif, IS_ISILON)
# TODO: looking at TEST_START_CLUSTER_ARGS is a hack. It would be better to add an option
# to pytest.
test_start_cluster_args = os.environ.get("TEST_START_CLUSTER_ARGS","")
old_agg_regex = "enable_partitioned_aggregation=false"
old_hash_join_regex = "enable_partitioned_hash_join=false"
using_old_aggs_joins = re.search(old_agg_regex, test_start_cluster_args) is not None or \
re.search(old_hash_join_regex, test_start_cluster_args) is not None
class SkipIfOldAggsJoins:
nested_types = pytest.mark.skipif(using_old_aggs_joins,
reason="Nested types not supported with old aggs and joins")
passthrough_preagg = pytest.mark.skipif(using_old_aggs_joins,
reason="Passthrough optimization not implemented by old agg")
unsupported = pytest.mark.skipif(using_old_aggs_joins,
reason="Query unsupported with old aggs and joins")
requires_spilling = pytest.mark.skipif(using_old_aggs_joins,
reason="Test case requires spilling to pass")
class SkipIfLocal:
# These ones are skipped due to product limitations.
caching = pytest.mark.skipif(IS_LOCAL,
reason="HDFS caching not supported on local file system")
hdfs_blocks = pytest.mark.skipif(IS_LOCAL,
reason="Files on local filesystem are not split into blocks")
hdfs_encryption = pytest.mark.skipif(IS_LOCAL,
reason="HDFS encryption is not supported on local filesystem")
hive = pytest.mark.skipif(IS_LOCAL,
reason="Hive not started when using local file system")
multiple_impalad = pytest.mark.skipif(IS_LOCAL,
reason="Multiple impalads are not supported when using local file system")
parquet_file_size = pytest.mark.skipif(IS_LOCAL,
reason="Parquet block size incorrectly determined")
# These ones need test infra work to re-enable.
hbase = pytest.mark.skipif(IS_LOCAL,
reason="HBase not started when using local file system")
hdfs_client = pytest.mark.skipif(IS_LOCAL,
reason="HDFS not started when using local file system")
mem_usage_different = pytest.mark.skipif(IS_LOCAL,
reason="Memory limit too low when running single node")
qualified_path = pytest.mark.skipif(IS_LOCAL,
reason="Tests rely on HDFS qualified paths")
root_path = pytest.mark.skipif(IS_LOCAL,
reason="Tests rely on the root directory")
# Try to derive the build type. Assume it's 'latest' by default.
impala_home = os.environ.get("IMPALA_HOME", "")
build_type_regex = re.compile(r'--build_type=(\w+)', re.I)
build_type_search_result = re.search(build_type_regex, test_start_cluster_args)
if build_type_search_result is not None:
build_type = build_type_search_result.groups()[0].lower()
else:
build_type = 'latest'
# Resolve any symlinks in the path.
impalad_basedir = \
os.path.realpath(os.path.join(impala_home, 'be/build', build_type)).rstrip('/')
debug_build = os.path.basename(impalad_basedir).lower() == 'debug'
class SkipIfNotDebugBuild:
debug_only = pytest.mark.skipif(not debug_build,
reason="Tests depends on debug build startup option.")
Reference in New Issue View Git Blame Copy Permalink