mirror of
https://github.com/apache/impala.git
synced 2026-02-03 09:00:39 -05:00
86e611bdb2
This patch aims to make the query tear-down path more predictable. Specifically, we want to ensure that Coordinator::ComputeQuerySummary() finishes before the query is unregistered. Before that could happen if cancellation occurred on a different thread from unregistration, e.g. if async_exec_thread_ cancels the query, if an asynchronous Cancel() RPC comes in, if it was cancelled internally because of a timeout, or for some other reason. The race was possible for a couple of reasons: * The synchronization between threads happened on UpdateExecState(), which is called before ComputeQuerySummary(). * The unregistering thread may not call Coordinator::Cancel() if coord_exec_called_ is not set. This patch addresses both problems, by adding synchronization so that Coordinator::Cancel() blocks until ComputeQuerySummary() finishes, and so that coord_exec_called_ is set even if 'async_exec_thread_' is cancelling the query (which ensures that both threads call Coordinator::Cancel() and synchronize there). This is only done on the finalizing thread. Other threads that initiate cancellation should not block on cancellation, so that threads cannot pile up waiting for the query. Testing: Added a regression test that failed before this change. Ran exhaustive tests. Change-Id: I62cacc06d9877d33b79a33aeb3b82195e639b5c4 Reviewed-on: http://gerrit.cloudera.org:8080/15954 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
102 lines
4.5 KiB
Python
102 lines
4.5 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import threading
|
|
from time import sleep
|
|
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
|
|
from tests.common.impala_test_suite import ImpalaTestSuite
|
|
|
|
|
|
def cancel_query_and_validate_state(client, query, exec_option, table_format,
|
|
cancel_delay, join_before_close=False):
|
|
"""Runs the given query asynchronously and then cancels it after the specified delay.
|
|
The query is run with the given 'exec_options' against the specified 'table_format'. A
|
|
separate async thread is launched to fetch the results of the query. The method
|
|
validates that the query was successfully cancelled and that the error messages for the
|
|
calls to ImpalaConnection#fetch and #close are consistent. If 'join_before_close' is
|
|
True the method will join against the fetch results thread before closing the query.
|
|
"""
|
|
if exec_option: client.set_configuration(exec_option)
|
|
if table_format: ImpalaTestSuite.change_database(client, table_format)
|
|
handle = client.execute_async(query)
|
|
|
|
thread = threading.Thread(target=__fetch_results, args=(query, handle))
|
|
thread.start()
|
|
|
|
sleep(cancel_delay)
|
|
assert client.get_state(handle) != client.QUERY_STATES['EXCEPTION']
|
|
cancel_result = client.cancel(handle)
|
|
assert cancel_result.status_code == 0,\
|
|
'Unexpected status code from cancel request: %s' % cancel_result
|
|
|
|
if join_before_close:
|
|
thread.join()
|
|
|
|
close_error = None
|
|
try:
|
|
client.close_query(handle)
|
|
except ImpalaBeeswaxException as e:
|
|
close_error = e
|
|
|
|
# Before accessing fetch_results_error we need to join the fetch thread
|
|
thread.join()
|
|
|
|
# IMPALA-9756: Make sure query summary info has been added to profile for queries
|
|
# that proceeded far enough into execution that it should have been added to profile.
|
|
# The logic in ClientRequestState/Coordinator is convoluted, but the summary info
|
|
# should be added if the query has got to the point where rows can be fetched. We
|
|
# need to do this after both close_query() and fetch() have returned to ensure
|
|
# that the synchronous phase of query unregistration has finished and the profile
|
|
# is final.
|
|
profile = client.get_runtime_profile(handle)
|
|
if ("- Completed admission: " in profile and
|
|
("- First row fetched:" in profile or "- Request finished:" in profile)):
|
|
# TotalBytesRead is a sentinel that will only be created if ComputeQuerySummary()
|
|
# has been run by the cancelling thread.
|
|
assert "- TotalBytesRead:" in profile, profile
|
|
|
|
if thread.fetch_results_error is None:
|
|
# If the fetch rpc didn't result in CANCELLED (and auto-close the query) then
|
|
# the close rpc should have succeeded.
|
|
assert close_error is None
|
|
elif close_error is None:
|
|
# If the close rpc succeeded, then the fetch rpc should have either succeeded,
|
|
# failed with 'Cancelled' or failed with 'Invalid or unknown query handle'
|
|
# (if the close rpc occured before the fetch rpc).
|
|
if thread.fetch_results_error is not None:
|
|
assert 'Cancelled' in str(thread.fetch_results_error) or \
|
|
('Invalid or unknown query handle' in str(thread.fetch_results_error)
|
|
and not join_before_close), str(thread.fetch_results_error)
|
|
else:
|
|
# If the close rpc encountered an exception, then it must be due to fetch
|
|
# noticing the cancellation and doing the auto-close.
|
|
assert 'Invalid or unknown query handle' in str(close_error)
|
|
assert 'Cancelled' in str(thread.fetch_results_error)
|
|
|
|
# TODO: Add some additional verification to check to make sure the query was
|
|
# actually canceled
|
|
|
|
|
|
def __fetch_results(query, handle):
|
|
threading.current_thread().fetch_results_error = None
|
|
threading.current_thread().query_profile = None
|
|
try:
|
|
new_client = ImpalaTestSuite.create_impala_client()
|
|
new_client.fetch(query, handle)
|
|
except ImpalaBeeswaxException as e:
|
|
threading.current_thread().fetch_results_error = e
|