mirror of
https://github.com/apache/impala.git
synced 2026-01-10 00:00:16 -05:00
8fc1eccce4
This patch adds changes to the planner to account for memory used by bloom filters at the fragment instance level. Also adds changes to allocate memory for those bloom filters from the buffer pool. Testing: - Modified Planner Tests and end to end tests to account for memory reservation for the runtime filters. - Modified backend tests and benchmarks to use the bufferpool for bloom filter allocation. - Add an end to end test. - Ran rest of the core tests. Change-Id: Iea2759665fb2e8bef9433014a8d42a7ebf99ce1f Reviewed-on: http://gerrit.cloudera.org:8080/8971 Reviewed-by: Bikramjeet Vig <bikramjeet.vig@cloudera.com> Tested-by: Impala Public Jenkins
66 lines
2.5 KiB
Plaintext
66 lines
2.5 KiB
Plaintext
====
|
|
---- QUERY
|
|
# Explain a simple hash join query.
|
|
explain
|
|
select *
|
|
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
|
|
---- RESULTS: VERIFY_IS_EQUAL
|
|
'Max Per-Host Resource Reservation: Memory=35.00MB'
|
|
'Per-Host Resource Estimates: Memory=477.41MB'
|
|
''
|
|
'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
|
|
'| Per-Host Resources: mem-estimate=0B mem-reservation=0B'
|
|
'PLAN-ROOT SINK'
|
|
'| mem-estimate=0B mem-reservation=0B'
|
|
'|'
|
|
'04:EXCHANGE [UNPARTITIONED]'
|
|
'| mem-estimate=0B mem-reservation=0B'
|
|
'| tuple-ids=0,1 row-size=454B cardinality=5757710'
|
|
'|'
|
|
'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
|
|
'Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB'
|
|
'02:HASH JOIN [INNER JOIN, BROADCAST]'
|
|
'| hash predicates: l_orderkey = o_orderkey'
|
|
'| fk/pk conjuncts: l_orderkey = o_orderkey'
|
|
'| runtime filters: RF000[bloom] <- o_orderkey'
|
|
'| mem-estimate=300.41MB mem-reservation=34.00MB spill-buffer=2.00MB'
|
|
'| tuple-ids=0,1 row-size=454B cardinality=5757710'
|
|
'|'
|
|
'|--03:EXCHANGE [BROADCAST]'
|
|
'| | mem-estimate=0B mem-reservation=0B'
|
|
'| | tuple-ids=1 row-size=191B cardinality=1500000'
|
|
'| |'
|
|
'| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
|
|
'| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B'
|
|
'| 01:SCAN HDFS [tpch.orders, RANDOM]'
|
|
row_regex:.*partitions=1/1 files=1 size=.*
|
|
'| stored statistics:'
|
|
row_regex:.*table: rows=1500000 size=.*
|
|
'| columns: all'
|
|
'| extrapolated-rows=disabled'
|
|
'| mem-estimate=88.00MB mem-reservation=0B'
|
|
'| tuple-ids=1 row-size=191B cardinality=1500000'
|
|
'|'
|
|
'00:SCAN HDFS [tpch.lineitem, RANDOM]'
|
|
row_regex:.*partitions=1/1 files=1 size=.*
|
|
' runtime filters: RF000[bloom] -> l_orderkey'
|
|
' stored statistics:'
|
|
row_regex:.*table: rows=6001215 size=.*
|
|
' columns: all'
|
|
' extrapolated-rows=disabled'
|
|
' mem-estimate=88.00MB mem-reservation=0B'
|
|
' tuple-ids=0 row-size=263B cardinality=6001215'
|
|
====
|
|
---- QUERY
|
|
# Tests the warning about missing table stats in the explain header.
|
|
explain select count(t1.int_col), avg(t2.float_col), sum(t3.bigint_col)
|
|
from functional_avro.alltypes t1
|
|
inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id)
|
|
left outer join functional_avro.alltypes t3 on (t2.id = t3.id)
|
|
where t1.month = 1 and t2.year = 2009 and t3.bool_col = false
|
|
---- RESULTS: VERIFY_IS_SUBSET
|
|
'Per-Host Resource Estimates: Memory=4.07GB'
|
|
'WARNING: The following tables are missing relevant table and/or column statistics.'
|
|
'functional_avro.alltypes, functional_parquet.alltypessmall'
|
|
====
|