mirror of
https://github.com/apache/impala.git
synced 2026-01-04 18:00:57 -05:00
e89d7057a6
The main idea of this patch is to use table stats to extrapolate the row counts for new/modified partitions. Existing behavior: - Partitions that lack the row count stat are ignored when estimating the cardinality of HDFS scans. Such partitions effectively have an estimated row count of zero. - We always use the row count stats for partitions that have one. The row count may be innaccurate if data in such partitions has changed significantly. Summary of changes: - Enhance COMPUTE STATS to also store the total number of file bytes in the table. - Use the table-level row count and file bytes stats to estimate the number of rows in a scan. - A new impalad startup flag is added to enable/disable the extrapolation behavior. The feature is disabled by default. Note that even with the feature disabled, COMPUTE STATS stores the file bytes so you can enable the feature without having to run COMPUTE STATS again. Testing: - Added new FE unit test - Added new EE test Change-Id: I972c8a03ed70211734631a7dc9085cb33622ebc4 Reviewed-on: http://gerrit.cloudera.org:8080/6840 Reviewed-by: Alex Behm <alex.behm@cloudera.com> Tested-by: Impala Public Jenkins
63 lines
2.4 KiB
Plaintext
63 lines
2.4 KiB
Plaintext
====
|
|
---- QUERY
|
|
# Explain a simple hash join query.
|
|
explain
|
|
select *
|
|
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
|
|
---- RESULTS: VERIFY_IS_EQUAL
|
|
'Per-Host Resource Reservation: Memory=136.00MB'
|
|
'Per-Host Resource Estimates: Memory=388.41MB'
|
|
''
|
|
'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
|
|
' PLAN-ROOT SINK'
|
|
' | mem-estimate=0B mem-reservation=0B'
|
|
' |'
|
|
' 04:EXCHANGE [UNPARTITIONED]'
|
|
' mem-estimate=0B mem-reservation=0B'
|
|
' tuple-ids=0,1 row-size=454B cardinality=5757710'
|
|
''
|
|
'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
|
|
' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=04, UNPARTITIONED]'
|
|
' | mem-estimate=0B mem-reservation=0B'
|
|
' 02:HASH JOIN [INNER JOIN, BROADCAST]'
|
|
' | hash predicates: l_orderkey = o_orderkey'
|
|
' | runtime filters: RF000 <- o_orderkey'
|
|
' | mem-estimate=300.41MB mem-reservation=136.00MB'
|
|
' | tuple-ids=0,1 row-size=454B cardinality=5757710'
|
|
' |'
|
|
' |--03:EXCHANGE [BROADCAST]'
|
|
' | mem-estimate=0B mem-reservation=0B'
|
|
' | tuple-ids=1 row-size=191B cardinality=1500000'
|
|
' |'
|
|
' 00:SCAN HDFS [tpch.lineitem, RANDOM]'
|
|
row_regex:.*partitions=1/1 files=1 size=.*
|
|
' runtime filters: RF000 -> l_orderkey'
|
|
' stats-rows=6001215 extrapolated-rows=disabled'
|
|
' table stats: rows=6001215 size=718.94MB'
|
|
' column stats: all'
|
|
' mem-estimate=88.00MB mem-reservation=0B'
|
|
' tuple-ids=0 row-size=263B cardinality=6001215'
|
|
''
|
|
'F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
|
|
' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=03, BROADCAST]'
|
|
' | mem-estimate=0B mem-reservation=0B'
|
|
' 01:SCAN HDFS [tpch.orders, RANDOM]'
|
|
row_regex:.*partitions=1/1 files=1 size=.*
|
|
' stats-rows=1500000 extrapolated-rows=disabled'
|
|
' table stats: rows=1500000 size=162.56MB'
|
|
' column stats: all'
|
|
' mem-estimate=88.00MB mem-reservation=0B'
|
|
' tuple-ids=1 row-size=191B cardinality=1500000'
|
|
====
|
|
---- QUERY
|
|
# Tests the warning about missing table stats in the explain header.
|
|
explain select count(t1.int_col), avg(t2.float_col), sum(t3.bigint_col)
|
|
from functional_avro.alltypes t1
|
|
inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id)
|
|
left outer join functional_avro.alltypes t3 on (t2.id = t3.id)
|
|
where t1.month = 1 and t2.year = 2009 and t3.bool_col = false
|
|
---- RESULTS: VERIFY_IS_SUBSET
|
|
'WARNING: The following tables are missing relevant table and/or column statistics.'
|
|
'functional_avro.alltypes, functional_parquet.alltypessmall'
|
|
====
|