jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2026-01-02 03:00:32 -05:00
Code Issues Projects Releases Wiki Activity
Files
514dfaf9fdff219256eaa9baf3efcc66bfdfafda
impala/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
Amos Bird 628685ae74 IMPALA-1654: General partition exprs in DDL operations. 
This commit handles partition related DDL in a more general way. We can
now use compound predicates to specify a list of partitions in
statements like ALTER TABLE DROP PARTITION and COMPUTE INCREMENTAL
STATS, etc. It will also make sure some statements only accept one
partition at a time, such as PARTITION SET LOCATION and LOAD DATA. ALTER
TABLE ADD PARTITION remains using the old PartitionKeyValue's logic.

The changed partition related DDLs are as follows,

Table: p (i int) partitioned by (j int, k string)
Partitions:
+-------+---+-------+--------+------+--------------+-------------------+
| j     | k | #Rows | #Files | Size | Bytes Cached | Cache Replication |
+-------+---+-------+--------+------+--------------+-------------------+
| 1     | a | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| 1     | b | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| 1     | c | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| 2     | d | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| 2     | e | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| 2     | f | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| Total |   | -1    | 0      | 0B   | 0B           |                   |
+-------+---+-------+--------+------+--------------+-------------------+

1. show files in p partition (j<2, k='a');
2. alter table p partition (j<2, k in ("b","c") set cached in 'testPool';

// j can appear more than once,
3.1. alter table p partition (j<2, j>0, k<>"d") set uncached;
// it is the same as
3.2. alter table p partition (j<2 and j>0, not k="e") set uncached;
// we can also do 'or'
3.3. alter table p partition (j<2 or j>0, k like "%") set uncached;

// missing 'k' matches all values of k
4. alter table p partition (j<2) set fileformat textfile;
5. alter table p partition (k rlike ".*") set serdeproperties ("k"="v");
6. alter table p partition (j is not null) set tblproperties ("k"="v");
7. alter table p drop partition (j<2);
8. compute incremental stats p partition(j<2);

The remaining old partition related DDLs are as follows,

1. load data inpath '/path/from' into table p partition (j=2, k="d");
2. alter table p add partition (j=2, k="g");
3. alter table p partition (j=2, k="g") set location '/path/to';
4. insert into p partition (j=2, k="g") values (1), (2), (3);

General partition expressions or partially specified partition specs
allows partition predicates to return empty partition set no matter
'IF EXISTS' is specified.

Examples:

[localhost.localdomain:21000] >
alter table p drop partition (j=2, k="f");
Query: alter table p drop partition (j=2, k="f")
+-------------------------+
| summary                 |
+-------------------------+
| Dropped 1 partition(s). |
+-------------------------+
Fetched 1 row(s) in 0.78s
[localhost.localdomain:21000] >
alter table p drop partition (j=2, k<"f");
Query: alter table p drop partition (j=2, k<"f")
+-------------------------+
| summary                 |
+-------------------------+
| Dropped 2 partition(s). |
+-------------------------+
Fetched 1 row(s) in 0.41s
[localhost.localdomain:21000] >
alter table p drop partition (k="a");
Query: alter table p drop partition (k="a")
+-------------------------+
| summary                 |
+-------------------------+
| Dropped 1 partition(s). |
+-------------------------+
Fetched 1 row(s) in 0.25s
[localhost.localdomain:21000] > show partitions p;
Query: show partitions p
+-------+---+-------+--------+------+--------------+-------------------+
| j     | k | #Rows | #Files | Size | Bytes Cached | Cache Replication |
+-------+---+-------+--------+------+--------------+-------------------+
| 1     | b | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| 1     | c | -1    | 0      | 0B   | NOT CACHED   | NOT CACHED        |
| Total |   | -1    | 0      | 0B   | 0B           |                   |
+-------+---+-------+--------+------+--------------+-------------------+
Fetched 3 row(s) in 0.01s

Change-Id: I2c9162fcf9d227b8daf4c2e761d57bab4e26408f
Reviewed-on: http://gerrit.cloudera.org:8080/3942
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Internal Jenkins
2016-11-15 03:27:36 +00:00

388 lines
13 KiB
Plaintext
Raw Blame History

====
---- QUERY
use cachedb
====
---- QUERY
create table cached_tbl_nopart (i int) cached in 'testPool'
====
---- QUERY
insert into cached_tbl_nopart select 1
---- RESULTS
: 1
====
---- QUERY
select * from cached_tbl_nopart
---- RESULTS
1
---- TYPES
INT
====
---- QUERY
# Create a table 'like' cached_tbl_nopart
create table like_cached_tbl_nopart like cached_tbl_nopart
====
---- QUERY
# like_cached_tbl_nopart shouldn't inhert the caching settings
show table stats like_cached_tbl_nopart
---- RESULTS
-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Unpartitioned table shows correct caching
show table stats cached_tbl_nopart
---- RESULTS
-1,1,'2B',regex:.+B,'1','TEXT','false',regex:.*
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Unpartitioned table is no longer cached
alter table cached_tbl_nopart set uncached
====
---- QUERY
show table stats cached_tbl_nopart
---- RESULTS
-1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop table if exists cached_tbl_part
====
---- QUERY
create table cached_tbl_part (i int) partitioned by (j int) cached in 'testPool' with replication = 9
====
---- QUERY
# new partition should inherit the cached property
alter table cached_tbl_part add partition (j=0)
====
---- QUERY
# should be able to override the inherited cached property
alter table cached_tbl_part add partition (j=1) uncached
====
---- QUERY
alter table cached_tbl_part add partition (j=2) cached in 'testPool'
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'9','TEXT','false',regex:.*
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'9','TEXT','false',regex:.*
'Total',-1,0,'0B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop table if exists cached_tbl_part
====
---- QUERY
create table cached_tbl_part (i int) partitioned by (j int) cached in 'testPool'
====
---- QUERY
# new partition should inherit the cached property
alter table cached_tbl_part add partition (j=0)
====
---- QUERY
# should be able to override the inherited cached property
alter table cached_tbl_part add partition (j=1) uncached
====
---- QUERY
alter table cached_tbl_part add partition (j=2) cached in 'testPool'
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'Total',-1,0,'0B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# uncache one of the partitions
alter table cached_tbl_part partition (j=2) set uncached
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'Total',-1,0,'0B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Can uncache the same partition twice without an error.
alter table cached_tbl_part partition (j=2) set uncached
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'Total',-1,0,'0B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# mark an uncached partition as cached
alter table cached_tbl_part partition (j=1) set cached in 'testPool'
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'Total',-1,0,'0B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# dynamic partition insert inherits table properties
insert into cached_tbl_part partition(j) values(3, 3), (4, 4);
---- RESULTS
j=3/: 1
j=4/: 1
====
---- QUERY
# Modify partition cache replication
alter table cached_tbl_part partition (j=3) set cached in 'testPool' with replication = 4
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'4','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'1','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Set uncached clears all cache requests
alter table cached_tbl_part set uncached
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'3',-1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'4',-1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Can call set uncached multiple times on the same partitioned table
alter table cached_tbl_part set uncached
====
---- QUERY
# Set cached caches all partitions
alter table cached_tbl_part set cached in 'testPool'
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'1','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'1','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Can call set cached multiple times on the same partitioned table.
alter table cached_tbl_part set cached in 'testPool'
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'1','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'1','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Modify partition cache replication in preparation for table level alteration
alter table cached_tbl_part partition (j=3) set cached in 'testPool' with replication = 4
====
---- QUERY
# Uncache a partition in preparation for table level alteration
alter table cached_tbl_part partition (j=1) set uncached
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'4','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'1','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Set replication factor for all partitions regardless of their current state
alter table cached_tbl_part set cached in 'testPool' with replication = 8
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Add partition with different replication
alter table cached_tbl_part add partition(j=5) cached in 'testPool' with replication = 3
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'5',-1,0,'0B',regex:.+B,'3','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Change replication for a partition
alter table cached_tbl_part partition(j=2) set cached in 'testPool' with replication = 3
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'3','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'5',-1,0,'0B',regex:.+B,'3','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Add partition with inherited replication from table
alter table cached_tbl_part add partition(j=6) cached in 'testPool'
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'3','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'5',-1,0,'0B',regex:.+B,'3','TEXT','false',regex:.*
'6',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Add a partition with uncacheable location and it should fail
alter table cached_tbl_part add partition(j=7)
location 'file://$IMPALA_HOME/testdata/data/cached_tbl_part'
---- CATCH
Location 'file:$IMPALA_HOME/testdata/data/cached_tbl_part' cannot be cached. Please retry without caching: ALTER TABLE cachedb.cached_tbl_part ADD PARTITION ... UNCACHED
====
---- QUERY
# Add a partition with uncacheable location and it should succeed with UNCACHED specified
alter table cached_tbl_part add partition(j=7)
location 'file://$IMPALA_HOME/testdata/data/cache_tbl_part' uncached
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'2',-1,0,'0B',regex:.+B,'3','TEXT','false',regex:.*
'3',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'4',-1,1,'2B',regex:.+B,'8','TEXT','false',regex:.*
'5',-1,0,'0B',regex:.+B,'3','TEXT','false',regex:.*
'6',-1,0,'0B',regex:.+B,'8','TEXT','false',regex:.*
'7',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'Total',-1,2,'4B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Create a table which resides on the local file system
create table uncached_tbl_local(i int)
location 'file://$IMPALA_HOME/testdata/data/uncached_tbl_local'
====
---- QUERY
# Turning on caching should fail for this table
alter table uncached_tbl_local set cached in 'testPool'
---- CATCH
Table cachedb.uncached_tbl_local cannot be cached. Please check if the table or partitions are on a filesystem which supports caching.
====
---- QUERY
# Create a new table on HDFS
create table cached_tbl_local(i int) partitioned by (j int)
====
---- QUERY
# Add a partition which resides on the local filesystem
alter table cached_tbl_local add partition(j=0)
location 'file://$IMPALA_HOME/testdata/data/cached_local_part'
====
---- QUERY
# Turning on caching for this new partition should fail
alter table cached_tbl_local partition(j=0) set cached in 'testPool'
---- CATCH
Partition(s) (j=0) cannot be cached. Please check if the table or partitions are on a filesystem which supports caching.
====
---- QUERY
# Turning on caching should also fail for this table
alter table cached_tbl_local set cached in 'testPool'
---- CATCH
Table cachedb.cached_tbl_local cannot be cached. Please check if the table or partitions are on a filesystem which supports caching.
====
---- QUERY
# Adding a cacheable partition should be allowed even if other partitions are not cacheable.
alter table cached_tbl_local add partition(j=1) cached in 'testPool'
====
---- QUERY
# Turning on caching should still fail for this table
alter table cached_tbl_local set cached in 'testPool'
---- CATCH
Table cachedb.cached_tbl_local cannot be cached. Please check if the table or partitions are on a filesystem which supports caching.
====
---- QUERY
show partitions cached_tbl_local;
---- RESULTS
'0',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false',regex:.*
'1',-1,0,'0B',regex:.+B,'1','TEXT','false',regex:.*
'Total',-1,0,'0B',regex:.+B,'','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
Reference in New Issue View Git Blame Copy Permalink