mirror of
https://github.com/apache/impala.git
synced 2026-01-07 09:02:19 -05:00
f241782966
This patch fixes a slightly pathological state that occurs when the statestore is under heavy load. The result of the bug is that subscribers cannot successfully re-register because the statestore never marks them as failed. The exact sequence of events is as follows: 1. Subscriber registers with state-store. 2. Statestore does not send heartbeats in timely fashion to subscriber. Subscriber times-out. 3. Subscriber is restarted quickly. Statestore does not detect restart. 4. Subscriber's RegisterSubscriber() call fails, because statestore detects duplicate registration. 5. Subscriber restarts again. Since state-store is slow to send heartbeats, the state-store has not detected the restart and the subscriber receives a heartbeat message from the statestore and does not reject it. 6. Statestore continues to believe subscriber is alive, since the heartbeats are not being rejected. To fix this, we add a registration ID to each successfully registered subscriber that is known to both subscriber and statestore. If the subscriber should restart and re-register, it receives a new registration ID. Whenever a heartbeat arrives, it compares its registration ID to that sent by the statestore with the heartbeat, and rejects the heartbeat if they do not match. We also allow re-registration of existing subscribers (getting rid of the dreaded "Duplicate subscription" message). A new registration overwrites an old one. Change-Id: Ie32df3a586ccb375375ebfbcbec1aaeb930b6bfe Reviewed-on: http://gerrit.ent.cloudera.com:8080/778 Tested-by: jenkins Reviewed-by: Henry Robinson <henry@cloudera.com>
112 lines
4.1 KiB
Python
Executable File
112 lines
4.1 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
|
|
# Impala tests for queries that query metadata and set session settings
|
|
import logging
|
|
import pytest
|
|
from subprocess import call
|
|
from tests.common.test_vector import *
|
|
from tests.common.impala_test_suite import *
|
|
|
|
# TODO: For these tests to pass, all table metadata must be created exhaustively.
|
|
# the tests should be modified to remove that requirement.
|
|
class TestMetadataQueryStatements(ImpalaTestSuite):
|
|
@classmethod
|
|
def get_workload(self):
|
|
return 'functional-query'
|
|
|
|
@classmethod
|
|
def add_test_dimensions(cls):
|
|
super(TestMetadataQueryStatements, cls).add_test_dimensions()
|
|
# There is no reason to run these tests using all dimensions.
|
|
cls.TestMatrix.add_constraint(lambda v:\
|
|
v.get_value('table_format').file_format == 'text' and\
|
|
v.get_value('table_format').compression_codec == 'none')
|
|
|
|
def setup_method(self, method):
|
|
self.cleanup_db('hive_test_db')
|
|
|
|
def teardown_method(self, method):
|
|
self.cleanup_db('hive_test_db')
|
|
|
|
def test_show(self, vector):
|
|
self.run_test_case('QueryTest/show', vector)
|
|
|
|
def test_describe_table(self, vector):
|
|
self.run_test_case('QueryTest/describe', vector)
|
|
|
|
def test_describe_formatted(self, vector):
|
|
# Describe a partitioned table.
|
|
try:
|
|
self.exec_and_compare_hive_and_impala_hs2("describe formatted functional.alltypes")
|
|
self.exec_and_compare_hive_and_impala_hs2(
|
|
"describe formatted functional_text_lzo.alltypes")
|
|
# Describe an unpartitioned table.
|
|
self.exec_and_compare_hive_and_impala_hs2("describe formatted tpch.lineitem")
|
|
self.exec_and_compare_hive_and_impala_hs2("describe formatted functional.jointbl")
|
|
except Exception:
|
|
pytest.xfail(reason="IMPALA-1085, hiveserver2 does not start on occasion")
|
|
|
|
try:
|
|
# Describe a view
|
|
self.exec_and_compare_hive_and_impala_hs2(\
|
|
"describe formatted functional.alltypes_view_sub")
|
|
except AssertionError:
|
|
pytest.xfail("Investigate minor difference in displaying null vs empty values")
|
|
|
|
def test_use_table(self, vector):
|
|
self.run_test_case('QueryTest/use', vector)
|
|
|
|
@pytest.mark.execute_serially
|
|
def test_impala_sees_hive_created_tables_and_databases(self, vector):
|
|
# This scenario is covered as part of the data loading process and doesn't
|
|
# need to be validated with each test run.
|
|
if self.exploration_strategy() != 'exhaustive': pytest.skip()
|
|
db_name = 'hive_test_db'
|
|
tbl_name = 'testtbl'
|
|
self.client.refresh()
|
|
result = self.execute_query("show databases");
|
|
assert db_name not in result.data
|
|
call(["hive", "-e", "CREATE DATABASE %s" % db_name])
|
|
|
|
result = self.execute_query("show databases");
|
|
assert db_name not in result.data
|
|
|
|
self.client.refresh()
|
|
result = self.execute_query("show databases");
|
|
assert db_name in result.data
|
|
|
|
# Make sure no tables show up in the new database
|
|
result = self.execute_query("show tables in %s" % db_name);
|
|
assert len(result.data) == 0
|
|
|
|
self.client.refresh()
|
|
result = self.execute_query("show tables in %s" % db_name);
|
|
assert len(result.data) == 0
|
|
|
|
call(["hive", "-e", "CREATE TABLE %s.%s (i int)" % (db_name, tbl_name)])
|
|
|
|
result = self.execute_query("show tables in %s" % db_name)
|
|
assert tbl_name not in result.data
|
|
|
|
self.client.refresh()
|
|
result = self.execute_query("show tables in %s" % db_name)
|
|
assert tbl_name in result.data
|
|
|
|
# Make sure we can actually use the table
|
|
self.execute_query(("insert overwrite table %s.%s "
|
|
"select 1 from functional.alltypes limit 5"
|
|
% (db_name, tbl_name)))
|
|
result = self.execute_scalar("select count(*) from %s.%s" % (db_name, tbl_name))
|
|
assert int(result) == 5
|
|
|
|
call(["hive", "-e", "DROP TABLE %s.%s " % (db_name, tbl_name)])
|
|
call(["hive", "-e", "DROP DATABASE %s" % db_name])
|
|
|
|
# Requires a refresh to see the dropped database
|
|
result = self.execute_query("show databases");
|
|
assert db_name in result.data
|
|
|
|
self.client.refresh()
|
|
result = self.execute_query("show databases");
|
|
assert db_name not in result.data
|