jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2026-01-06 15:01:43 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-3089: Perform static partition pruning in the FE with disjunctive

Browse Source 
BETWEEN predicates

This commit fixes an issue where the slow path is employed during static
partition pruning for disjunctive BETWEEN predicates, inroducing
significant latency during planning, especially for tables with large
number of partitions.

Change-Id: I66ef566fa176a859d126d49152921a176a491b0a
Reviewed-on: http://gerrit.cloudera.org:8080/2320
Reviewed-by: Dimitris Tsirogiannis <dtsirogiannis@cloudera.com>
Tested-by: Internal Jenkins
This commit is contained in:
Dimitris Tsirogiannis
2016-02-25 17:37:01 -08:00
committed by Harrison Sheinblatt
parent 5d45fb18ec
commit 2c37d99fed
2 changed files with 20 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/src/main/java/com/cloudera/impala/planner/HdfsPartitionPruner.java
View File

@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudera.impala.analysis.Analyzer;
import com.cloudera.impala.analysis.BetweenPredicate;
import com.cloudera.impala.analysis.BinaryPredicate;
import com.cloudera.impala.analysis.BinaryPredicate.Operator;
import com.cloudera.impala.analysis.CompoundPredicate;
@@ -204,6 +205,9 @@ public class HdfsPartitionPruner {
if (!(expr.getChild(i).isLiteral())) return false;
}
return true;
} else if (expr instanceof BetweenPredicate) {
return canEvalUsingPartitionMd(((BetweenPredicate) expr).getRewrittenPredicate(),
analyzer);
}
return false;
}
@@ -399,8 +403,8 @@ public class HdfsPartitionPruner {
* Evaluate a slot binding predicate on a partition key using the partition
* key values; return the matching partition ids. An empty set is returned
* if there are no matching partitions. This function can evaluate the following
* types of predicates: BinaryPredicate, CompoundPredicate, IsNullPredicate, and
* InPredicate.
* types of predicates: BinaryPredicate, CompoundPredicate, IsNullPredicate,
* InPredicate, and BetweenPredicate.
*/
private HashSet<Long> evalSlotBindingFilter(Expr expr) {
Preconditions.checkNotNull(expr);
@@ -423,6 +427,8 @@ public class HdfsPartitionPruner {
return evalInPredicate(expr);
} else if (expr instanceof IsNullPredicate) {
return evalIsNullPredicate(expr);
} else if (expr instanceof BetweenPredicate) {
return evalSlotBindingFilter(((BetweenPredicate) expr).getRewrittenPredicate());
}
return null;
}

12
testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test vendored
View File

@@ -325,6 +325,18 @@ NODE 0:
HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=7/090701.txt 0:20853
HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=8/090801.txt 0:20853
====
# disjunction between predicates with complex constant exprs
select * from functional.alltypes
where year = 2009 and (month between 5+1 and 8-1 or month between 9-2 and 1+7)
---- PLAN
00:SCAN HDFS [functional.alltypes]
partitions=3/24 files=3 size=60.43KB
---- SCANRANGELOCATIONS
NODE 0:
HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=6/090601.txt 0:20179
HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=7/090701.txt 0:20853
HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=8/090801.txt 0:20853
====
# slot binding still determined
select * from functional.alltypes where year - 1 = 2009
---- PLAN