mirror of
https://github.com/apache/impala.git
synced 2026-02-03 09:00:39 -05:00
df365a8e30
Changes the Coordinator to release admitted memory when each Backend completes, rather than waiting for the entire query to complete before releasing admitted memory. When the Coordinator detects that a Backend has completed (via ControlService::ReportExecStatus) it updates the state of the Backend in Coordinator::BackendResourceState. BackendResourceState tracks the state of the admitted resources for each Backend and decides when the resources for a group of Backend states should be released. BackendResourceState defines a state machine to help coordinate the state of the admitted memory for each Backend. It guarantees that by the time the query is shutdown, all Backends release their admitted memory. BackendResourceState implements three rules to control the rate at which the Coordinator releases admitted memory from the AdmissionController: * Resources are released at most once every 1 second, this prevents short lived queries from causing high load on the admission controller * Resources are released at most O(log(num_backends)) times; the BackendResourceStates can release multiple BackendStates from the AdmissionController at a time * All pending resources are released if the only remaining Backend is the Coordinator Backend; this is useful for result spooling where all Backends may complete, except for the Coordinator Backend Exposes the following hidden startup flags to help tune the heuristics above: --batched_release_decay_factor * Defaults to 2 * Controls the base value for the O(log(num_backends)) bound when batching the release of Backends. --release_backend_states_delay_ms * Defaults to 1000 milliseconds * Controls how often Backends can release their resources. Testing: * Ran core tests * Added new tests to test_result_spooling.py and test_admission_controller.py Change-Id: I88bb11e0ede7574568020e0277dd8ac8d2586dc9 Reviewed-on: http://gerrit.cloudera.org:8080/14104 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
53 lines
2.3 KiB
Python
53 lines
2.3 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import json
|
|
|
|
|
|
def get_num_completed_backends(service, query_id):
|
|
"""Get the number of completed backends for the given query_id from the
|
|
'query_backends' web page."""
|
|
query_backend_url = 'query_backends?query_id={0}&json'.format(query_id)
|
|
query_backends_json = json.loads(service.read_debug_webpage(query_backend_url))
|
|
assert 'backend_states' in query_backends_json
|
|
num_complete_backends = 0
|
|
for backend_state in query_backends_json['backend_states']:
|
|
if backend_state['done']: num_complete_backends += 1
|
|
return num_complete_backends
|
|
|
|
|
|
def get_mem_admitted_backends_debug_page(cluster):
|
|
"""Helper method assumes a cluster using a dedicated coordinator. Returns the mem
|
|
admitted to the backends extracted from the backends debug page of the coordinator
|
|
impala daemon. Returns a dictionary with the keys 'coordinator' and 'executor' and
|
|
their respective mem values in bytes. The entry for 'executor' is a list of the mem
|
|
admitted for each executor."""
|
|
# Based on how the cluster is setup, the first impalad in the cluster is the
|
|
# coordinator.
|
|
response_json = cluster.impalads[0].service.get_debug_webpage_json('backends')
|
|
assert 'backends' in response_json
|
|
assert len(response_json['backends']) >= 2
|
|
ret = dict()
|
|
ret['executor'] = []
|
|
from tests.verifiers.mem_usage_verifier import parse_mem_value
|
|
for backend in response_json['backends']:
|
|
if backend['is_coordinator']:
|
|
ret['coordinator'] = parse_mem_value(backend['mem_admitted'])
|
|
else:
|
|
ret['executor'].append(parse_mem_value(backend['mem_admitted']))
|
|
return ret
|