mirror of
https://github.com/apache/impala.git
synced 2025-12-30 03:01:44 -05:00
874a115897
In particular, the new query options: BUFFER_POOL_LIMIT MAX_ROW_SIZE MIN_SPILLABLE_BUFFER_SIZE DEFAULT_SPILLABLE_BUFFER_SIZE Change-Id: I49323f8ffbff3e195058e88762eedbb1fcb1bc0e Reviewed-on: http://gerrit.cloudera.org:8080/8003 Tested-by: Impala Public Jenkins Reviewed-by: John Russell <jrussell@cloudera.com>
98 lines
3.6 KiB
XML
98 lines
3.6 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Licensed to the Apache Software Foundation (ASF) under one
|
|
or more contributor license agreements. See the NOTICE file
|
|
distributed with this work for additional information
|
|
regarding copyright ownership. The ASF licenses this file
|
|
to you under the Apache License, Version 2.0 (the
|
|
"License"); you may not use this file except in compliance
|
|
with the License. You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing,
|
|
software distributed under the License is distributed on an
|
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
KIND, either express or implied. See the License for the
|
|
specific language governing permissions and limitations
|
|
under the License.
|
|
-->
|
|
<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
|
|
<concept id="default_spillable_buffer_size" rev="2.10.0 IMPALA-3200">
|
|
|
|
<title>DEFAULT_SPILLABLE_BUFFER_SIZE Query Option</title>
|
|
<titlealts audience="PDF"><navtitle>DEFAULT_SPILLABLE_BUFFER_SIZE</navtitle></titlealts>
|
|
<prolog>
|
|
<metadata>
|
|
<data name="Category" value="Impala"/>
|
|
<data name="Category" value="Impala Query Options"/>
|
|
<data name="Category" value="Querying"/>
|
|
<data name="Category" value="Developers"/>
|
|
<data name="Category" value="Data Analysts"/>
|
|
</metadata>
|
|
</prolog>
|
|
|
|
<conbody>
|
|
|
|
<p>
|
|
<indexterm audience="hidden">DEFAULT_SPILLABLE_BUFFER_SIZE query option</indexterm>
|
|
Specifies the default size for a memory buffer used when the
|
|
spill-to-disk mechanism is activated, for example for queries against
|
|
a large table with no statistics, or large join operations.
|
|
</p>
|
|
|
|
<p conref="../shared/impala_common.xml#common/type_integer"/>
|
|
|
|
<!-- DEFAULT_SPILLABLE_BUFFER_SIZE: [2097152] -->
|
|
|
|
<p conref="../shared/impala_common.xml#common/default_blurb"/>
|
|
<p>
|
|
<codeph>2097152</codeph> (2 MB)
|
|
</p>
|
|
|
|
<p conref="../shared/impala_common.xml#common/units_blurb"/>
|
|
|
|
<p conref="../shared/impala_common.xml#common/added_in_2100"/>
|
|
|
|
<p conref="../shared/impala_common.xml#common/usage_notes_blurb"/>
|
|
<p>
|
|
This query option sets an upper bound on the size of the internal
|
|
buffer size that can be used during spill-to-disk operations. The
|
|
actual size of the buffer is chosen by the query planner.
|
|
</p>
|
|
<p>
|
|
If overall query performance is limited by the time needed for spilling,
|
|
consider increasing the <codeph>DEFAULT_SPILLABLE_BUFFER_SIZE</codeph> setting.
|
|
Larger buffer sizes result in Impala issuing larger I/O requests to storage
|
|
devices, which might result in higher throughput, particularly on rotational
|
|
disks.
|
|
</p>
|
|
<p>
|
|
The tradeoff with a large value for this setting is increased memory usage during
|
|
spill-to-disk operations. Reducing this value may reduce memory consumption.
|
|
</p>
|
|
<p>
|
|
To determine if the value for this setting is having an effect by capping the
|
|
spillable buffer size, you can see the buffer size chosen by the query planner for
|
|
a particular query. <codeph>EXPLAIN</codeph> the query while the setting
|
|
<codeph>EXPLAIN_LEVEL=2</codeph> is in effect.
|
|
</p>
|
|
|
|
<p conref="../shared/impala_common.xml#common/example_blurb"/>
|
|
|
|
<codeblock><![CDATA[
|
|
set default_spillable_buffer_size=4MB;
|
|
]]>
|
|
</codeblock>
|
|
|
|
<p conref="../shared/impala_common.xml#common/related_info"/>
|
|
<p>
|
|
<xref keyref="buffer_pool_limit"/>,
|
|
<xref keyref="max_row_size"/>,
|
|
<xref keyref="min_spillable_buffer_size"/>,
|
|
<xref keyref="scalability_buffer_pool"/>
|
|
</p>
|
|
|
|
</conbody>
|
|
</concept>
|