mirror of
https://github.com/apache/impala.git
synced 2026-01-01 18:00:30 -05:00
b3d8a507cb
Adds the TABLESAMPLE clause for COMPUTE STATS.
Syntax:
COMPUTE STATS <table> TABLESAMPLE SYSTEM(<number>) [REPEATABLE(<number>)]
Computes and replaces the table-level row count and total file size,
as well as all table-level column statistics. Existing partition-level
row counts are not modified.
The TABLESAMPLE clause can be used to limit the scanned data volume to
a desired percentage. When sampling, the unmodified results of the
COMPUTE STATS queries are sent to the CatalogServer. There, the stats
are extrapolated before storing them into the HMS so as not to confuse
other engines like Hive/SparkSQL which may rely on the shared HMS
fields being accurate.
Limitations
- Only works for HDFS tables
- TABLESAMPLE is not supported for COMPUTE INCREMENTAL STATS
- TABLESAMPLE requires --enable_stats_extrapolation=true
Changes to EXPLAIN
The stored statistics from the HMS are more clearly displayed under
a 'stored statistics' section. Example:
00:SCAN HDFS [functional.alltypes, RANDOM]
partitions=24/24 files=24 size=478.45KB
stored statistics:
table: rows=7300 size=478.45KB
partitions: 24/24 rows=7300
columns: all
Testing:
- added new functional tests
- core/hdfs run passed
Change-Id: I7f3e72471ac563adada4a4156033a85852b7c8b7
Reviewed-on: http://gerrit.cloudera.org:8080/8136
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Impala Public Jenkins
194 lines
7.5 KiB
Plaintext
194 lines
7.5 KiB
Plaintext
====
|
|
---- QUERY
|
|
# First create a partitioned table
|
|
create table t1 like functional.alltypes
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/t1';
|
|
insert into t1 partition(year, month) select * from functional.alltypes;
|
|
compute incremental stats t1;
|
|
show table stats t1;
|
|
---- LABELS
|
|
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
|
|
---- RESULTS
|
|
'2009','1',310,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','2',280,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','3',310,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','4',300,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2009','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','1',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','2',280,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','3',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','4',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'2010','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true',regex:.*
|
|
'Total','',7300,24,'586.84KB','0B','','','',''
|
|
---- TYPES
|
|
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
|
|
====
|
|
---- QUERY
|
|
show column stats t1;
|
|
---- LABELS
|
|
COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
|
|
---- RESULTS
|
|
'id','INT',7300,-1,4,4
|
|
'bool_col','BOOLEAN',2,-1,1,1
|
|
'tinyint_col','TINYINT',10,-1,1,1
|
|
'smallint_col','SMALLINT',10,-1,2,2
|
|
'int_col','INT',10,-1,4,4
|
|
'bigint_col','BIGINT',10,-1,8,8
|
|
'float_col','FLOAT',10,-1,4,4
|
|
'double_col','DOUBLE',10,-1,8,8
|
|
'date_string_col','STRING',736,-1,8,8
|
|
'string_col','STRING',10,-1,1,1
|
|
'timestamp_col','TIMESTAMP',7300,-1,16,16
|
|
'year','INT',2,0,4,4
|
|
'month','INT',12,0,4,4
|
|
---- TYPES
|
|
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
|
|
====
|
|
---- QUERY
|
|
# Show that the truncation removed all files, table stats, and incremental stats,
|
|
# but preserved the partitions.
|
|
truncate table t1;
|
|
show table stats t1;
|
|
---- LABELS
|
|
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
|
|
---- RESULTS
|
|
'2009','1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','5',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','6',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','7',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','8',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','9',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','10',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','11',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2009','12',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','3',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','4',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','5',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','6',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','7',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','8',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','9',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','10',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','11',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'2010','12',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
'Total','',-1,0,'0B','0B','','','',''
|
|
---- TYPES
|
|
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
|
|
====
|
|
---- QUERY
|
|
# Show that the truncation removed the column stats.
|
|
show column stats t1;
|
|
---- LABELS
|
|
COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
|
|
---- RESULTS
|
|
'id','INT',-1,-1,4,4
|
|
'bool_col','BOOLEAN',-1,-1,1,1
|
|
'tinyint_col','TINYINT',-1,-1,1,1
|
|
'smallint_col','SMALLINT',-1,-1,2,2
|
|
'int_col','INT',-1,-1,4,4
|
|
'bigint_col','BIGINT',-1,-1,8,8
|
|
'float_col','FLOAT',-1,-1,4,4
|
|
'double_col','DOUBLE',-1,-1,8,8
|
|
'date_string_col','STRING',-1,-1,-1,-1
|
|
'string_col','STRING',-1,-1,-1,-1
|
|
'timestamp_col','TIMESTAMP',-1,-1,16,16
|
|
'year','INT',2,0,4,4
|
|
'month','INT',12,0,4,4
|
|
---- TYPES
|
|
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
|
|
====
|
|
---- QUERY
|
|
# Create an unpartitioned table.
|
|
create table t2 like functional.tinytable
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/t2';
|
|
insert into t2 select * from functional.tinytable;
|
|
compute incremental stats t2;
|
|
show table stats t2;
|
|
---- LABELS
|
|
#ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
|
|
---- RESULTS
|
|
3,1,'38B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
---- TYPES
|
|
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
|
|
====
|
|
---- QUERY
|
|
show column stats t2;
|
|
---- LABELS
|
|
COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
|
|
---- RESULTS
|
|
'a','STRING',3,-1,8,6.666666507720947
|
|
'b','STRING',3,-1,7,4
|
|
---- TYPES
|
|
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
|
|
====
|
|
---- QUERY
|
|
# Show that the truncation removed all files, table stats, and incremental stats.
|
|
truncate table t2;
|
|
show table stats t2;
|
|
---- LABELS
|
|
#ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
|
|
---- RESULTS
|
|
-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
|
|
---- TYPES
|
|
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
|
|
====
|
|
---- QUERY
|
|
# Show that the truncation removed the column stats.
|
|
show column stats t2;
|
|
---- LABELS
|
|
COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
|
|
---- RESULTS
|
|
'a','STRING',-1,-1,-1,-1
|
|
'b','STRING',-1,-1,-1,-1
|
|
---- TYPES
|
|
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
|
|
====
|
|
---- QUERY
|
|
# TRUNCATE IF EXISTS does not fail on non existent table
|
|
truncate table if exists non_existent;
|
|
---- RESULTS
|
|
====
|
|
---- QUERY
|
|
# Create an unpartitioned table.
|
|
create table t3 like functional.tinytable
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/t3';
|
|
insert into t3 select * from functional.tinytable;
|
|
select count(*) from t3;
|
|
---- RESULTS
|
|
3
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
# TRUNCATE IF EXISTS base scenario
|
|
truncate table if exists t3;
|
|
---- RESULTS
|
|
====
|
|
---- QUERY
|
|
# Verify that truncate was successful
|
|
select count(*) from t3;
|
|
---- RESULTS
|
|
0
|
|
---- TYPES
|
|
BIGINT
|
|
====
|