mirror of
https://github.com/apache/impala.git
synced 2026-01-08 12:02:54 -05:00
5a6f53db16
The following changes are included in this commit: 1. Modified the alltypesagg table to include an additional partition key that has nulls. 2. Added a number of tests in hdfs.test that exercise the partition pruning logic (see IMPALA-887). 3. Modified all the tests that are affected by the change in alltypesagg. Change-Id: I1a769375aaa71273341522eb94490ba5e4c6f00d Reviewed-on: http://gerrit.ent.cloudera.com:8080/2874 Reviewed-by: Dimitris Tsirogiannis <dtsirogiannis@cloudera.com> Tested-by: jenkins Reviewed-on: http://gerrit.ent.cloudera.com:8080/3236
483 lines
13 KiB
Plaintext
483 lines
13 KiB
Plaintext
# full scan of string typed row-key
|
|
select * from functional_hbase.stringids
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
====
|
|
# predicate on row key doesn't get transformed into scan parameter, because
|
|
# it's mapped as an int (but stored in ascii and ordered lexicographically)
|
|
select * from functional_hbase.alltypessmall
|
|
where id < 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
predicates: id < 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60201 <unbounded>:3
|
|
HBASE KEYRANGE port=60202 3:7
|
|
HBASE KEYRANGE port=60203 7:<unbounded>
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
predicates: id < 5
|
|
====
|
|
# if the row key is mapped as a string col, range predicates are applied to the scan
|
|
select * from functional_hbase.stringids
|
|
where id = '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 5
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 5:5\0
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 5
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id > '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 5\0
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 5\0:7
|
|
HBASE KEYRANGE port=60203 7:<unbounded>
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 5\0
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id >= '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 5
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 5:7
|
|
HBASE KEYRANGE port=60203 7:<unbounded>
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 5
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id < '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
stop key: 5
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60201 <unbounded>:3
|
|
HBASE KEYRANGE port=60202 3:5
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
stop key: 5
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id <= '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id > '4' and id < '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4\0
|
|
stop key: 5
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 4\0:5
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4\0
|
|
stop key: 5
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id >= '4' and id < '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 4:5
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id > '4' and id <= '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4\0
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 4\0:5\0
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4\0
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
====
|
|
select * from functional_hbase.stringids
|
|
where id >= '4' and id <= '5'
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 4:5\0
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
====
|
|
# with aggregation
|
|
select int_col, count(*)
|
|
from functional_hbase.alltypessmall
|
|
where id < 5
|
|
group by 1
|
|
---- PLAN
|
|
01:AGGREGATE [FINALIZE]
|
|
| output: count(*)
|
|
| group by: int_col
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
predicates: id < 5
|
|
---- DISTRIBUTEDPLAN
|
|
04:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
03:AGGREGATE [MERGE FINALIZE]
|
|
| output: sum(count(*))
|
|
| group by: int_col
|
|
|
|
|
02:EXCHANGE [HASH(int_col)]
|
|
|
|
|
01:AGGREGATE
|
|
| output: count(*)
|
|
| group by: int_col
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
predicates: id < 5
|
|
====
|
|
# predicates on string columns against a constant string are converted to HBase filters
|
|
select * from functional_hbase.alltypessmall where string_col = '4'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col EQUAL '4'
|
|
predicates: string_col = '4'
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col EQUAL '4'
|
|
predicates: string_col = '4'
|
|
====
|
|
# test all comparison ops
|
|
select * from functional_hbase.alltypessmall where string_col != '4'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col NOT_EQUAL '4'
|
|
predicates: string_col != '4'
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col NOT_EQUAL '4'
|
|
predicates: string_col != '4'
|
|
====
|
|
select * from functional_hbase.alltypessmall where string_col < '4'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col LESS '4'
|
|
predicates: string_col < '4'
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col LESS '4'
|
|
predicates: string_col < '4'
|
|
====
|
|
select * from functional_hbase.alltypessmall where string_col > '4'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col GREATER '4'
|
|
predicates: string_col > '4'
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col GREATER '4'
|
|
predicates: string_col > '4'
|
|
====
|
|
select * from functional_hbase.alltypessmall where string_col <= '4'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col LESS_OR_EQUAL '4'
|
|
predicates: string_col <= '4'
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col LESS_OR_EQUAL '4'
|
|
predicates: string_col <= '4'
|
|
====
|
|
select * from functional_hbase.alltypessmall where string_col >= '4'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col GREATER_OR_EQUAL '4'
|
|
predicates: string_col >= '4'
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col GREATER_OR_EQUAL '4'
|
|
predicates: string_col >= '4'
|
|
====
|
|
# test multiple filters
|
|
select * from functional_hbase.alltypessmall
|
|
where string_col >= '4' and string_col != '2' and date_string_col = '04/03/09'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters:
|
|
d:string_col GREATER_OR_EQUAL '4'
|
|
d:string_col NOT_EQUAL '2'
|
|
d:date_string_col EQUAL '04/03/09'
|
|
predicates: string_col >= '4', string_col != '2', date_string_col = '04/03/09'
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters:
|
|
d:string_col GREATER_OR_EQUAL '4'
|
|
d:string_col NOT_EQUAL '2'
|
|
d:date_string_col EQUAL '04/03/09'
|
|
predicates: string_col >= '4', string_col != '2', date_string_col = '04/03/09'
|
|
====
|
|
# mix of predicates and functional_hbase. filters
|
|
select * from functional_hbase.alltypessmall where string_col = '4' and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
hbase filters: d:string_col EQUAL '4'
|
|
predicates: string_col = '4', tinyint_col = 5
|
|
====
|
|
# mix of predicates, functional_hbase. filters and start/stop keys
|
|
select * from functional_hbase.stringids
|
|
where string_col = '4' and tinyint_col = 5 and id >= '4' and id <= '5'
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5\0
|
|
hbase filters: d:string_col EQUAL '4'
|
|
predicates: string_col = '4', tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 4:5\0
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5\0
|
|
hbase filters: d:string_col EQUAL '4'
|
|
predicates: string_col = '4', tinyint_col = 5
|
|
====
|
|
# predicates involving casts (ie, non-string comparisons) cannot be turned into filters
|
|
select * from functional_hbase.alltypessmall where cast(string_col as int) >= 4
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
predicates: CAST(string_col AS INT) >= 4
|
|
====
|
|
# non-const comparisons cannot be turned into filters
|
|
select * from functional_hbase.alltypessmall where string_col >= date_string_col
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypessmall]
|
|
predicates: string_col >= date_string_col
|
|
====
|
|
# IMP-1188 - row key predicate is a constant expr.
|
|
select * from functional_hbase.stringids
|
|
where id = concat('', '5')
|
|
and tinyint_col = 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 5
|
|
stop key: 5\0
|
|
predicates: tinyint_col = 5
|
|
====
|
|
# IMP-1188 - row key predicate is a constant expr.
|
|
select * from functional_hbase.stringids
|
|
where string_col = '4' and tinyint_col = 5
|
|
and id >= concat('', '4') and id <= concat('5', '')
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5\0
|
|
hbase filters: d:string_col EQUAL '4'
|
|
predicates: string_col = '4', tinyint_col = 5
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60202 4:5\0
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
start key: 4
|
|
stop key: 5\0
|
|
hbase filters: d:string_col EQUAL '4'
|
|
predicates: string_col = '4', tinyint_col = 5
|
|
====
|
|
# IMP-1188 - row key predicate is null.
|
|
select * from functional_hbase.stringids where id = null
|
|
---- PLAN
|
|
empty scan node
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
empty scan node
|
|
====
|
|
# IMP-1188 - row key lower bound is bigger than upper bound.
|
|
select * from functional_hbase.stringids where id > 'b' and id < 'a'
|
|
---- PLAN
|
|
empty scan node
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
empty scan node
|
|
====
|
|
# IMP-1188 - casting row key to non-string type disables predicate from being pushed
|
|
# into HBase
|
|
select * from functional_hbase.stringids
|
|
where cast(id as int) < 5
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.stringids]
|
|
predicates: CAST(id AS INT) < 5
|
|
====
|
|
# The following test cases test plan generation for queries executed against HBase
|
|
# and have 'IS NULL/IS NOT NULL' predicates as well as conjunctive predicates.
|
|
# (IMPALA-642)
|
|
#
|
|
# HBase scan query with an 'IS NULL' predicate
|
|
select * from functional_hbase.alltypesagg
|
|
where bigint_col is null
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col IS NULL
|
|
====
|
|
# HBase scan query with projection and an 'IS NULL' predicate on one of the
|
|
# projected columns
|
|
select bigint_col, day from functional_hbase.alltypesagg
|
|
where bigint_col is null
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col IS NULL
|
|
====
|
|
# HBase scan query with 'IS NOT NULL' predicate
|
|
select * from functional_hbase.alltypesagg
|
|
where bigint_col is not null
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col IS NOT NULL
|
|
====
|
|
# HBase scan query with conjunctive predicates one of which is an 'IS NULL'
|
|
select * from functional_hbase.alltypesagg
|
|
where bigint_col is null and day = 1
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col IS NULL, day = 1
|
|
====
|
|
# HBase scan query with conjunctive predicates one of which is an 'IS NOT NULL'
|
|
select * from functional_hbase.alltypesagg
|
|
where bigint_col is not null and bool_col = true
|
|
---- PLAN
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col IS NOT NULL, bool_col = TRUE
|
|
---- SCANRANGELOCATIONS
|
|
NODE 0:
|
|
HBASE KEYRANGE port=60201 <unbounded>:3
|
|
HBASE KEYRANGE port=60202 3:7
|
|
HBASE KEYRANGE port=60203 7:<unbounded>
|
|
---- DISTRIBUTEDPLAN
|
|
01:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col IS NOT NULL, bool_col = TRUE
|
|
====
|
|
# HBase scan query with an aggregation and a single predicate
|
|
select count(*) from functional_hbase.alltypesagg
|
|
where bigint_col = 10
|
|
---- PLAN
|
|
01:AGGREGATE [FINALIZE]
|
|
| output: count(*)
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col = 10
|
|
---- DISTRIBUTEDPLAN
|
|
03:AGGREGATE [MERGE FINALIZE]
|
|
| output: sum(count(*))
|
|
|
|
|
02:EXCHANGE [UNPARTITIONED]
|
|
|
|
|
01:AGGREGATE
|
|
| output: count(*)
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col = 10
|
|
====
|
|
# HBase scan query with an aggregation and conjunctive predicates
|
|
select count(*) from functional_hbase.alltypesagg
|
|
where bigint_col = 10 and day = 1
|
|
---- PLAN
|
|
01:AGGREGATE [FINALIZE]
|
|
| output: count(*)
|
|
|
|
|
00:SCAN HBASE [functional_hbase.alltypesagg]
|
|
predicates: bigint_col = 10, day = 1
|
|
====
|