diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index 7b9b639ab..40409a252 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -278,6 +278,8 @@ DEFINE_validator(ssl_minimum_version, [](const char* flagname, const string& val return false; }); +DEFINE_bool(use_calcite_planner, false, "By default this flag is false. If true, " + "the Calcite planner will be used to compile queries."); DEFINE_int32(idle_session_timeout, 0, "The time, in seconds, that a session may be idle" " for before it is closed (and all running queries cancelled) by Impala. If 0, idle" " sessions are never expired. It can be overridden by the query option" @@ -2018,6 +2020,7 @@ void ImpalaServer::InitializeConfigVariables() { // Set idle_session_timeout here to let the SET command return the value of // the command line option FLAGS_idle_session_timeout default_query_options_.__set_idle_session_timeout(FLAGS_idle_session_timeout); + default_query_options_.__set_use_calcite_planner(FLAGS_use_calcite_planner); // The next query options used to be set with flags. Setting them in // default_query_options_ here in order to make default_query_options // take precedence over the legacy flags. diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py index 6dad0c923..e6042417f 100755 --- a/bin/start-impala-cluster.py +++ b/bin/start-impala-cluster.py @@ -703,9 +703,7 @@ def build_impalad_arg_lists(cluster_size, num_coordinators, use_exclusive_coordi args = "-allow_tuple_caching=true {args}".format(args=args) if options.use_calcite_planner.lower() == 'true': - args = "-jni_frontend_class={jni_frontend_class} {args}".format( - jni_frontend_class="org/apache/impala/calcite/service/CalciteJniFrontend", - args=args) + args = "-use_calcite_planner=true {args}".format(args=args) os.environ["USE_CALCITE_PLANNER"] = "true" if options.enable_ranger_authz: diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java index 781fba6bb..ee209c487 100644 --- a/fe/src/main/java/org/apache/impala/service/Frontend.java +++ b/fe/src/main/java/org/apache/impala/service/Frontend.java @@ -149,6 +149,7 @@ import org.apache.impala.catalog.iceberg.IcebergMetadataTable; import org.apache.impala.catalog.paimon.FePaimonTable; import org.apache.impala.catalog.paimon.FeShowFileStmtSupport; import org.apache.impala.common.AnalysisException; +import org.apache.impala.common.UnsupportedFeatureException; import org.apache.impala.common.UserCancelledException; import org.apache.impala.common.FileSystemUtil; import org.apache.impala.common.ImpalaException; @@ -2399,7 +2400,7 @@ public class Frontend { try { request = getTExecRequest(compilerFactory, planCtx, timeline); } catch (Exception e) { - if (!shouldFallbackToRegularPlanner(planCtx)) { + if (!shouldFallbackToRegularPlanner(planCtx, e)) { throw e; } LOG.info("Calcite planner failed: ", e); @@ -2417,17 +2418,20 @@ public class Frontend { return request; } - private boolean shouldFallbackToRegularPlanner(PlanCtx planCtx) { + private boolean shouldFallbackToRegularPlanner(PlanCtx planCtx, Exception e) { // TODO: Need a fallback flag for various modes. In production, we will most // likely want to fallback to the original planner, but in testing, we might want // the query to fail. // There are some cases where we will always want to fallback, e.g. if the statement // fails at parse time because it is not a select statement. + if (e instanceof UnsupportedFeatureException) { + return true; + } TQueryCtx queryCtx = planCtx.getQueryContext(); try { return !(Parser.parse(queryCtx.client_request.stmt, queryCtx.client_request.query_options) instanceof QueryStmt); - } catch (Exception e) { + } catch (Exception f) { return false; } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java index 05cb65722..d2c19b7f8 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java @@ -156,8 +156,18 @@ public class CalciteAnalysisDriver implements AnalysisDriver { validatedNode_ = sqlValidator_.validate(parsedStmt_.getParsedSqlNode()); return new CalciteAnalysisResult(this); } catch (ImpalaException e) { + try { + UnsupportedChecker.throwUnsupportedIfKnownException(e, stmtTableCache_); + } catch (ImpalaException u) { + e = u; + } return new CalciteAnalysisResult(this, e); } catch (CalciteContextException e) { + try { + UnsupportedChecker.throwUnsupportedIfKnownException(e, stmtTableCache_); + } catch (ImpalaException u) { + return new CalciteAnalysisResult(this, u); + } return new CalciteAnalysisResult(this, new AnalysisException(e.getMessage(), e.getCause())); } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java index 7edf6def1..a72b7c9c3 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java @@ -23,7 +23,11 @@ import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.impala.calcite.parser.ImpalaSqlParserImpl; import org.apache.impala.calcite.validate.ImpalaConformance; +import org.apache.impala.common.ImpalaException; import org.apache.impala.common.ParseException; +import org.apache.impala.common.UnsupportedFeatureException; + +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +63,11 @@ public class CalciteQueryParser implements CompilerStep { SqlNode sqlNode = parser.parseQuery(); return sqlNode; } catch (SqlParseException e) { + try { + UnsupportedChecker.throwUnsupportedIfKnownException(e); + } catch (ImpalaException u) { + throw new ParseException(u.getMessage()); + } throw new ParseException(e.getMessage()); } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java new file mode 100644 index 000000000..b9b7e3455 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.calcite.service; + +import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache; +import org.apache.impala.common.ImpalaException; +import org.apache.impala.common.ParseException; +import org.apache.impala.common.UnsupportedFeatureException; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * UnsupportedChecker containts methods which determine if a feature + * is unsupported for the Calcite planner. + */ +public class UnsupportedChecker { + + private static Pattern LEFT_SEMI = Pattern.compile(".*\\bleft\\ssemi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern RIGHT_SEMI = Pattern.compile(".*\\bright\\ssemi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern LEFT_ANTI = Pattern.compile(".*\\bleft\\santi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern RIGHT_ANTI = Pattern.compile(".*\\bright\\santi\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern INPUT_FILE_NAME = Pattern.compile(".*\\binput__file__name\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern FILE_POSITION = Pattern.compile(".*\\bfile__position\\b.*", + Pattern.CASE_INSENSITIVE); + + private static Pattern TABLE_NOT_FOUND = + Pattern.compile(".*\\bTable '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE); + + private static Pattern COLUMN_NOT_FOUND = + Pattern.compile(".*\\bColumn '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE); + + public static void throwUnsupportedIfKnownException(Exception e) + throws ImpalaException { + String s = e.toString().replace("\n"," "); + if (LEFT_ANTI.matcher(s).matches() || RIGHT_ANTI.matcher(s).matches()) { + throw new UnsupportedFeatureException("Anti joins not supported."); + } + if (LEFT_SEMI.matcher(s).matches() || RIGHT_SEMI.matcher(s).matches()) { + throw new UnsupportedFeatureException("Semi joins not supported."); + } + if (INPUT_FILE_NAME.matcher(s).matches() || FILE_POSITION.matcher(s).matches()) { + throw new UnsupportedFeatureException("Virtual columns not supported."); + } + } + + public static void throwUnsupportedIfKnownException(Exception e, + StmtTableCache stmtTableCache) throws ImpalaException { + throwUnsupportedIfKnownException(e); + String s = e.toString().replace("\n"," "); + Matcher m = TABLE_NOT_FOUND.matcher(s); + + // If the error given is "table/column not found", it is possible that the message + // was generated by a complex column that looks like a table + // (e.g. mytbl.my_complex_column) which is currently not supported. We check for + // this possibility by seeing if the 'table not found' is identified as a column + // within one of the tables in the query. This check isn't fool-proof in that + // it might actually be a table that also is a column name in another table. + // However, that case should be extremely rare, and the result would be that + // the wrong error message will show up. + if (m.matches()) { + if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) { + throw new UnsupportedFeatureException( + "Complex column " + m.group(1) + " not supported."); + } + } + + m = COLUMN_NOT_FOUND.matcher(s); + if (m.matches()) { + if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) { + throw new UnsupportedFeatureException( + "Complex column " + m.group(1) + " not supported."); + } + } + } +}