From 2c37d99fedfb18d07e6f7340846d351f5eccfc50 Mon Sep 17 00:00:00 2001 From: Dimitris Tsirogiannis Date: Thu, 25 Feb 2016 17:37:01 -0800 Subject: [PATCH] IMPALA-3089: Perform static partition pruning in the FE with disjunctive BETWEEN predicates This commit fixes an issue where the slow path is employed during static partition pruning for disjunctive BETWEEN predicates, inroducing significant latency during planning, especially for tables with large number of partitions. Change-Id: I66ef566fa176a859d126d49152921a176a491b0a Reviewed-on: http://gerrit.cloudera.org:8080/2320 Reviewed-by: Dimitris Tsirogiannis Tested-by: Internal Jenkins --- .../cloudera/impala/planner/HdfsPartitionPruner.java | 10 ++++++++-- .../functional-planner/queries/PlannerTest/hdfs.test | 12 ++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/com/cloudera/impala/planner/HdfsPartitionPruner.java b/fe/src/main/java/com/cloudera/impala/planner/HdfsPartitionPruner.java index 89489e714..30020d156 100644 --- a/fe/src/main/java/com/cloudera/impala/planner/HdfsPartitionPruner.java +++ b/fe/src/main/java/com/cloudera/impala/planner/HdfsPartitionPruner.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.cloudera.impala.analysis.Analyzer; +import com.cloudera.impala.analysis.BetweenPredicate; import com.cloudera.impala.analysis.BinaryPredicate; import com.cloudera.impala.analysis.BinaryPredicate.Operator; import com.cloudera.impala.analysis.CompoundPredicate; @@ -204,6 +205,9 @@ public class HdfsPartitionPruner { if (!(expr.getChild(i).isLiteral())) return false; } return true; + } else if (expr instanceof BetweenPredicate) { + return canEvalUsingPartitionMd(((BetweenPredicate) expr).getRewrittenPredicate(), + analyzer); } return false; } @@ -399,8 +403,8 @@ public class HdfsPartitionPruner { * Evaluate a slot binding predicate on a partition key using the partition * key values; return the matching partition ids. An empty set is returned * if there are no matching partitions. This function can evaluate the following - * types of predicates: BinaryPredicate, CompoundPredicate, IsNullPredicate, and - * InPredicate. + * types of predicates: BinaryPredicate, CompoundPredicate, IsNullPredicate, + * InPredicate, and BetweenPredicate. */ private HashSet evalSlotBindingFilter(Expr expr) { Preconditions.checkNotNull(expr); @@ -423,6 +427,8 @@ public class HdfsPartitionPruner { return evalInPredicate(expr); } else if (expr instanceof IsNullPredicate) { return evalIsNullPredicate(expr); + } else if (expr instanceof BetweenPredicate) { + return evalSlotBindingFilter(((BetweenPredicate) expr).getRewrittenPredicate()); } return null; } diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test index 945b36bd2..880017b82 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test @@ -325,6 +325,18 @@ NODE 0: HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=7/090701.txt 0:20853 HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=8/090801.txt 0:20853 ==== +# disjunction between predicates with complex constant exprs +select * from functional.alltypes +where year = 2009 and (month between 5+1 and 8-1 or month between 9-2 and 1+7) +---- PLAN +00:SCAN HDFS [functional.alltypes] + partitions=3/24 files=3 size=60.43KB +---- SCANRANGELOCATIONS +NODE 0: + HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=6/090601.txt 0:20179 + HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=7/090701.txt 0:20853 + HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypes/year=2009/month=8/090801.txt 0:20853 +==== # slot binding still determined select * from functional.alltypes where year - 1 = 2009 ---- PLAN