impala/tests/common/skip.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Impala py.test skipif markers.  When a test can't be run against S3,
# choose the appropriate reason (or add a new one if needed) and
# annotate the class or test routine with the marker.
#

from __future__ import absolute_import, division, print_function
import pytest
from functools import partial

from tests.common.environ import (ImpalaTestClusterProperties,
                                  IS_DOCKERIZED_TEST_CLUSTER, IS_BUGGY_EL6_KERNEL,
                                  HIVE_MAJOR_VERSION,
                                  IS_APACHE_HIVE, IS_TEST_JDK,
                                  IS_TUPLE_CACHE)
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.kudu_test_suite import get_kudu_master_flag
from tests.util.filesystem_utils import (
    IS_ABFS,
    IS_ADLS,
    IS_GCS,
    IS_COS,
    IS_OSS,
    IS_OBS,
    IS_EC,
    IS_HDFS,
    IS_ISILON,
    IS_LOCAL,
    IS_S3,
    IS_OZONE,
    SECONDARY_FILESYSTEM)

IMPALA_TEST_CLUSTER_PROPERTIES = ImpalaTestClusterProperties.get_instance()
EXPLORATION_STRATEGY_EXHAUSTIVE = 'exhaustive'


class SkipIfFS:
  # These are skipped due to product limitations.
  hdfs_caching = pytest.mark.skipif(not IS_HDFS, reason="SET CACHED not implemented")
  hdfs_encryption = pytest.mark.skipif(not IS_HDFS,
      reason="HDFS encryption is not supported")
  # EC reports block groups of 3 blocks, and the minimum block size is 1MB.
  hdfs_small_block = pytest.mark.skipif(not IS_HDFS or IS_EC,
      reason="Requires tables with 1MB block size")
  hdfs_block_size = pytest.mark.skipif(not IS_HDFS,
      reason="Size of block reported to Impala is not ~128MB")
  hdfs_acls = pytest.mark.skipif(not IS_HDFS, reason="HDFS acls are not supported")

  # Special case product limitations.
  empty_directory = pytest.mark.skipif(IS_S3,
      reason="Empty directories are not supported on S3")
  file_or_folder_name_ends_with_period = pytest.mark.skipif(IS_ABFS,
      reason="ABFS does not support file / directories that end with a period")
  stress_insert_timeouts = pytest.mark.skipif(IS_COS or IS_GCS or IS_OSS or IS_OBS,
      reason="IMPALA-10563, IMPALA-10773")
  shutdown_idle_fails = pytest.mark.skipif(IS_COS or IS_GCS or IS_OSS or IS_OBS,
      reason="IMPALA-10562")
  late_filters = pytest.mark.skipif(IS_ISILON, reason="IMPALA-6998")
  read_past_eof = pytest.mark.skipif(IS_S3 or IS_GCS or (IS_OZONE and IS_EC),
      reason="IMPALA-2512")
  large_block_size = pytest.mark.skipif(IS_OZONE or IS_EC,
      reason="block size is larger than 128MB")
  read_speed_dependent = pytest.mark.skipif(not IS_HDFS or IS_EC,
      reason="success depends on fast scan node performance")
  incorrent_reported_ec = pytest.mark.skipif(IS_OZONE and IS_EC, reason="HDDS-8543")

  # These need test infra work to re-enable.
  hive = pytest.mark.skipif(
      not IS_HDFS, reason="HiveServer2 doesn't work or not started")
  hbase = pytest.mark.skipif(not IS_HDFS, reason="HBase not started")
  qualified_path = pytest.mark.skipif(not IS_HDFS,
      reason="Tests rely on HDFS qualified paths, IMPALA-1872")
  no_partial_listing = pytest.mark.skipif(not IS_HDFS,
      reason="Tests rely on HDFS partial listing.")
  variable_listing_times = pytest.mark.skipif(IS_S3 or IS_GCS or IS_COS or IS_OSS
      or IS_OBS, reason="Flakiness due to unpredictable listing times on S3.")
  eventually_consistent = pytest.mark.skipif(IS_ADLS or IS_COS or IS_OSS or IS_OBS,
      reason="The client is slow to realize changes to file metadata")


class SkipIfKudu:
  """Expose decorators as methods so that kudu web pages can be checked lazily when
     needed, instead of whenever this module is imported. This helps to run Kudu
     unrelated tests without launching the Kudu cluster."""

  @classmethod
  def no_hybrid_clock(cls):
    return pytest.mark.skipif(
        get_kudu_master_flag("--use_hybrid_clock") == "false",
        reason="Test relies on --use_hybrid_clock=true in Kudu.")

  @classmethod
  def hms_integration_enabled(cls):
    return pytest.mark.skipif(
        get_kudu_master_flag("--hive_metastore_uris") != "",
        reason="Test assumes Kudu/HMS integration is not enabled.")


class SkipIf:
  skip_hbase = pytest.mark.skipif(pytest.config.option.skip_hbase,
      reason="--skip_hbase argument specified")
  not_s3 = pytest.mark.skipif(not IS_S3, reason="S3 Filesystem needed")
  not_hdfs = pytest.mark.skipif(not IS_HDFS, reason="HDFS admin needed")
  not_dfs = pytest.mark.skipif(not (IS_HDFS or IS_OZONE),
      reason="HDFS/Ozone Filesystem needed")
  not_scratch_fs = pytest.mark.skipif(not (IS_HDFS or IS_OZONE),
      reason="Scratch dirs for temporary file spilling not supported")
  sfs_unsupported = pytest.mark.skipif(not (IS_HDFS or IS_S3 or IS_ABFS or IS_ADLS
      or IS_GCS), reason="Hive support for sfs+ is limited, HIVE-26757")
  hardcoded_uris = pytest.mark.skipif(not IS_HDFS,
      reason="Iceberg delete files hardcode the full URI in parquet files")
  not_ec = pytest.mark.skipif(not IS_EC, reason="Erasure Coding needed")
  no_secondary_fs = pytest.mark.skipif(not SECONDARY_FILESYSTEM,
      reason="Secondary filesystem needed")
  is_buggy_el6_kernel = pytest.mark.skipif(
      IS_BUGGY_EL6_KERNEL, reason="Kernel is affected by KUDU-1508")
  is_test_jdk = pytest.mark.skipif(IS_TEST_JDK, reason="Testing with different JDK")
  runs_slowly = pytest.mark.skipif(IMPALA_TEST_CLUSTER_PROPERTIES.runs_slowly(),
      reason="Test cluster runs slowly due to enablement of code coverage or sanitizer")
  not_tuple_cache = pytest.mark.skipif(not IS_TUPLE_CACHE,
      reason="Tuple Cache needed")


class SkipIfLocal:
  # These are skipped due to product limitations.
  hdfs_blocks = pytest.mark.skipif(IS_LOCAL,
      reason="Files on local filesystem are not split into blocks")
  multiple_impalad = pytest.mark.skipif(IS_LOCAL,
      reason="Multiple impalads are not supported when using local file system")
  parquet_file_size = pytest.mark.skipif(IS_LOCAL,
      reason="Parquet block size incorrectly determined")
  hdfs_fd_caching = pytest.mark.skipif(IS_LOCAL,
      reason="HDFS file handle caching not supported for local non-HDFS files")

  # These need test infra work to re-enable.
  hdfs_client = pytest.mark.skipif(IS_LOCAL,
      reason="HDFS not started when using local file system")
  root_path = pytest.mark.skipif(IS_LOCAL,
      reason="Tests rely on the root directory")


class SkipIfNotHdfsMinicluster:
  # These are skipped when not running against a local HDFS mini-cluster.
  plans = pytest.mark.skipif(
      not (IS_HDFS or IS_OZONE) or IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster(),
      reason="Test assumes plans from local HDFS mini-cluster")
  tuned_for_minicluster = pytest.mark.skipif(not (IS_HDFS or IS_OZONE)
      or IS_EC or IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster(),
      reason="Test is tuned for 3-node HDFS minicluster with no EC")
  scheduling = pytest.mark.skipif(
      not (IS_HDFS or IS_OZONE) or IS_EC or pytest.config.option.testing_remote_cluster,
      reason="Test is tuned for scheduling decisions made on a 3-node HDFS minicluster "
             "with no EC")


class SkipIfBuildType:
  dev_build = pytest.mark.skipif(IMPALA_TEST_CLUSTER_PROPERTIES.is_dev(),
      reason="Test takes too much time on debug build.")
  not_dev_build = pytest.mark.skipif(not IMPALA_TEST_CLUSTER_PROPERTIES.is_dev(),
      reason="Test depends on debug build startup option.")
  remote = pytest.mark.skipif(IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster(),
      reason="Test depends on running against a local Impala cluster")


class SkipIfEC:
  contain_full_explain = pytest.mark.skipif(IS_EC, reason="Contain full explain output "
              "for hdfs tables.")
  different_scan_split = pytest.mark.skipif(IS_EC, reason="Scan split of row "
              "groups for Parquet tables created in EC mode is different.")
  parquet_file_size = pytest.mark.skipif(IS_EC,
      reason="Fewer parquet files due to large block size, reducing parallelism")


class SkipIfDockerizedCluster:
  catalog_service_not_exposed = pytest.mark.skipif(
      IS_DOCKERIZED_TEST_CLUSTER, reason="Catalog service not exposed.")
  statestore_not_exposed = pytest.mark.skipif(
      IS_DOCKERIZED_TEST_CLUSTER, reason="Statestore service not exposed.")
  internal_hostname = pytest.mark.skipif(
      IS_DOCKERIZED_TEST_CLUSTER, reason="Internal hostname is used, not local hostname.")
  daemon_logs_not_exposed = pytest.mark.skipif(
      IS_DOCKERIZED_TEST_CLUSTER, reason="Daemon logs not exposed in host.")
  accesses_host_filesystem = pytest.mark.skipif(
      IS_DOCKERIZED_TEST_CLUSTER, reason="Daemon would need to access host filesystem.")
  insert_acls = pytest.mark.skipif(IS_DOCKERIZED_TEST_CLUSTER,
      reason="IMPALA-8384: insert ACL tests are broken on dockerised minicluster.")
  insufficient_mem_limit = pytest.mark.skipif(
      IS_DOCKERIZED_TEST_CLUSTER, reason="Test require high per-process mem_limit.")
  runs_slowly = pytest.mark.skipif(
      IS_DOCKERIZED_TEST_CLUSTER, reason="Dockerized env too slow for test.")
  jira = partial(pytest.mark.skipif, IS_DOCKERIZED_TEST_CLUSTER)


class SkipIfHive3:
  col_stat_separated_by_engine = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
      reason="Hive 3 separates column statistics by engine")
  without_hms_not_supported = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
      reason="Instantiating HMS server in embedded mode within Hive client requires more "
             "dependencies of Hive 3, see IMPALA-9287.")
  non_acid = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
      reason="This test expects tables in non-AICD format.")


class SkipIfHive2:
  acid = pytest.mark.skipif(HIVE_MAJOR_VERSION == 2,
      reason="Acid tables are only supported with Hive 3.")
  col_stat_not_separated_by_engine = pytest.mark.skipif(HIVE_MAJOR_VERSION == 2,
      reason="Hive 2 doesnt support separating column statistics by engine")
  create_external_kudu_table = pytest.mark.skipif(HIVE_MAJOR_VERSION == 2,
      reason="Hive 2 does not support creating external.table.purge Kudu tables."
             " See IMPALA-9092 for details.")
  orc = pytest.mark.skipif(HIVE_MAJOR_VERSION <= 2,
      reason="CREATE TABLE LIKE ORC is only supported with Hive version >= 3")
  ranger_auth = pytest.mark.skipif(HIVE_MAJOR_VERSION <= 2,
      reason="Hive 2 doesn't support Ranger authorization.")


class SkipIfCatalogV2:
  """Expose decorators as methods so that is_catalog_v2_cluster() can be evaluated lazily
  when needed, instead of whenever this module is imported."""
  @classmethod
  def stats_pulling_disabled(self):
    return pytest.mark.skipif(
      IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
      reason="Local catalog does not use incremental stats pulling.")

  @classmethod
  def catalog_v1_test(self):
    return pytest.mark.skipif(
      IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
      reason="Test is specific to old implementation of catalog.")

  # TODO: IMPALA-8489: fix this bug.
  @classmethod
  def impala_8489(self):
    return pytest.mark.skipif(
      IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
      reason="IMPALA-8489: TestRecoverPartitions.test_post_invalidate "
             "IllegalStateException.")

  # TODO: IMPALA-7539: fix this bug.
  @classmethod
  def impala_7539(self):
    return pytest.mark.skipif(
      IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
      reason="IMPALA-7539: support HDFS permission checks for LocalCatalog")

  @classmethod
  def hms_event_polling_enabled(self):
    return pytest.mark.skipif(
      IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
      reason="Table isn't invalidated with Local catalog and enabled hms_event_polling.")


class SkipIfApacheHive():
  feature_not_supported = pytest.mark.skipif(IS_APACHE_HIVE,
      reason="Apache Hive does not support this feature")
  data_connector_not_supported = pytest.mark.skipif(
      IS_APACHE_HIVE and HIVE_MAJOR_VERSION <= 3,
      reason="Apache Hive 3.1 or older version do not support DataConnector")


class SkipIfCdpHive():
  deprecated_feature = pytest.mark.skipif(
      not IS_APACHE_HIVE, reason="Deprecated feature on CDP Hive support")


class SkipIfExploration:
  """Exposes decorators as methods so that tests can be skipped based on exploration
     strategy"""

  @classmethod
  def is_not_exhaustive(cls, skip_msg='runs only in exhaustive'):
    return pytest.mark.skipif(
        ImpalaTestSuite.exploration_strategy() != EXPLORATION_STRATEGY_EXHAUSTIVE,
        reason=skip_msg)