IMPALA-14408: Use regular path for Calcite planner instead of CalciteJniFrontend

When the --use_calcite_planner=true option is set at the server level,
the queries will no longer go through CalciteJniFrontend. Instead, they
will go through the regular JniFrontend, which is the path that is used
when the query option for "use_calcite_planner" is set.

The CalciteJniFrontend will be removed in a later commit.

This commit also enables fallback to the original planner when an unsupported
feature exception is thrown. This needed to be added to allow the tests to run
properly. During initial database load, there are queries that access complex
columns which throws the unsupported exception.

Change-Id: I732516ca8f7ea64f73484efd67071910c9b62c8f
Reviewed-on: http://gerrit.cloudera.org:8080/23523
Reviewed-by: Steve Carlin <scarlin@cloudera.com>
Tested-by: Steve Carlin <scarlin@cloudera.com>

be/src/service/impala-server.cc

@@ -278,6 +278,8 @@ DEFINE_validator(ssl_minimum_version, [](const char* flagname, const string& val
   return false;
 });
 
+DEFINE_bool(use_calcite_planner, false, "By default this flag is false. If true, "
+    "the Calcite planner will be used to compile queries.");
 DEFINE_int32(idle_session_timeout, 0, "The time, in seconds, that a session may be idle"
     " for before it is closed (and all running queries cancelled) by Impala. If 0, idle"
     " sessions are never expired. It can be overridden by the query option"
@@ -2018,6 +2020,7 @@ void ImpalaServer::InitializeConfigVariables() {
   // Set idle_session_timeout here to let the SET command return the value of
   // the command line option FLAGS_idle_session_timeout
   default_query_options_.__set_idle_session_timeout(FLAGS_idle_session_timeout);
+  default_query_options_.__set_use_calcite_planner(FLAGS_use_calcite_planner);
   // The next query options used to be set with flags. Setting them in
   // default_query_options_ here in order to make default_query_options
   // take precedence over the legacy flags.

bin/start-impala-cluster.py

@@ -703,9 +703,7 @@ def build_impalad_arg_lists(cluster_size, num_coordinators, use_exclusive_coordi
       args = "-allow_tuple_caching=true {args}".format(args=args)
 
     if options.use_calcite_planner.lower() == 'true':
-      args = "-jni_frontend_class={jni_frontend_class} {args}".format(
-          jni_frontend_class="org/apache/impala/calcite/service/CalciteJniFrontend",
-          args=args)
+      args = "-use_calcite_planner=true {args}".format(args=args)
       os.environ["USE_CALCITE_PLANNER"] = "true"
 
     if options.enable_ranger_authz:

fe/src/main/java/org/apache/impala/service/Frontend.java

@@ -149,6 +149,7 @@ import org.apache.impala.catalog.iceberg.IcebergMetadataTable;
 import org.apache.impala.catalog.paimon.FePaimonTable;
 import org.apache.impala.catalog.paimon.FeShowFileStmtSupport;
 import org.apache.impala.common.AnalysisException;
+import org.apache.impala.common.UnsupportedFeatureException;
 import org.apache.impala.common.UserCancelledException;
 import org.apache.impala.common.FileSystemUtil;
 import org.apache.impala.common.ImpalaException;
@@ -2399,7 +2400,7 @@ public class Frontend {
       try {
         request = getTExecRequest(compilerFactory, planCtx, timeline);
       } catch (Exception e) {
-        if (!shouldFallbackToRegularPlanner(planCtx)) {
+        if (!shouldFallbackToRegularPlanner(planCtx, e)) {
           throw e;
         }
         LOG.info("Calcite planner failed: ", e);
@@ -2417,17 +2418,20 @@ public class Frontend {
     return request;
   }
 
-  private boolean shouldFallbackToRegularPlanner(PlanCtx planCtx) {
+  private boolean shouldFallbackToRegularPlanner(PlanCtx planCtx, Exception e) {
     // TODO: Need a fallback flag for various modes. In production, we will most
     // likely want to fallback to the original planner, but in testing, we might want
     // the query to fail.
     // There are some cases where we will always want to fallback, e.g. if the statement
     // fails at parse time because it is not a select statement.
+    if (e instanceof UnsupportedFeatureException) {
+      return true;
+    }
     TQueryCtx queryCtx = planCtx.getQueryContext();
     try {
       return !(Parser.parse(queryCtx.client_request.stmt,
           queryCtx.client_request.query_options) instanceof QueryStmt);
-    } catch (Exception e) {
+    } catch (Exception f) {
       return false;
     }
   }

java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteAnalysisDriver.java

@@ -156,8 +156,18 @@ public class CalciteAnalysisDriver implements AnalysisDriver {
       validatedNode_ = sqlValidator_.validate(parsedStmt_.getParsedSqlNode());
       return new CalciteAnalysisResult(this);
     } catch (ImpalaException e) {
+      try {
+        UnsupportedChecker.throwUnsupportedIfKnownException(e, stmtTableCache_);
+      } catch (ImpalaException u) {
+        e = u;
+      }
       return new CalciteAnalysisResult(this, e);
     } catch (CalciteContextException e) {
+      try {
+        UnsupportedChecker.throwUnsupportedIfKnownException(e, stmtTableCache_);
+      } catch (ImpalaException u) {
+        return new CalciteAnalysisResult(this, u);
+      }
       return new CalciteAnalysisResult(this,
           new AnalysisException(e.getMessage(), e.getCause()));
     }

java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java

@@ -23,7 +23,11 @@ import org.apache.calcite.sql.parser.SqlParser;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.impala.calcite.parser.ImpalaSqlParserImpl;
 import org.apache.impala.calcite.validate.ImpalaConformance;
+import org.apache.impala.common.ImpalaException;
 import org.apache.impala.common.ParseException;
+import org.apache.impala.common.UnsupportedFeatureException;
+
+import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -59,6 +63,11 @@ public class CalciteQueryParser implements CompilerStep {
       SqlNode sqlNode = parser.parseQuery();
       return sqlNode;
     } catch (SqlParseException e) {
+      try {
+        UnsupportedChecker.throwUnsupportedIfKnownException(e);
+      } catch (ImpalaException u) {
+        throw new ParseException(u.getMessage());
+      }
       throw new ParseException(e.getMessage());
     }
   }

java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java

@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.calcite.service;
+
+import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.common.ParseException;
+import org.apache.impala.common.UnsupportedFeatureException;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * UnsupportedChecker containts methods which determine if a feature
+ * is unsupported for the Calcite planner.
+ */
+public class UnsupportedChecker {
+
+  private static Pattern LEFT_SEMI = Pattern.compile(".*\\bleft\\ssemi\\b.*",
+      Pattern.CASE_INSENSITIVE);
+
+  private static Pattern RIGHT_SEMI = Pattern.compile(".*\\bright\\ssemi\\b.*",
+      Pattern.CASE_INSENSITIVE);
+
+  private static Pattern LEFT_ANTI = Pattern.compile(".*\\bleft\\santi\\b.*",
+      Pattern.CASE_INSENSITIVE);
+
+  private static Pattern RIGHT_ANTI = Pattern.compile(".*\\bright\\santi\\b.*",
+      Pattern.CASE_INSENSITIVE);
+
+  private static Pattern INPUT_FILE_NAME = Pattern.compile(".*\\binput__file__name\\b.*",
+      Pattern.CASE_INSENSITIVE);
+
+  private static Pattern FILE_POSITION = Pattern.compile(".*\\bfile__position\\b.*",
+      Pattern.CASE_INSENSITIVE);
+
+  private static Pattern TABLE_NOT_FOUND =
+      Pattern.compile(".*\\bTable '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE);
+
+  private static Pattern COLUMN_NOT_FOUND =
+      Pattern.compile(".*\\bColumn '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE);
+
+  public static void throwUnsupportedIfKnownException(Exception e)
+      throws ImpalaException {
+    String s = e.toString().replace("\n"," ");
+    if (LEFT_ANTI.matcher(s).matches() || RIGHT_ANTI.matcher(s).matches()) {
+      throw new UnsupportedFeatureException("Anti joins not supported.");
+    }
+    if (LEFT_SEMI.matcher(s).matches() || RIGHT_SEMI.matcher(s).matches()) {
+      throw new UnsupportedFeatureException("Semi joins not supported.");
+    }
+    if (INPUT_FILE_NAME.matcher(s).matches() || FILE_POSITION.matcher(s).matches()) {
+      throw new UnsupportedFeatureException("Virtual columns not supported.");
+    }
+  }
+
+  public static void throwUnsupportedIfKnownException(Exception e,
+      StmtTableCache stmtTableCache) throws ImpalaException {
+    throwUnsupportedIfKnownException(e);
+    String s = e.toString().replace("\n"," ");
+    Matcher m = TABLE_NOT_FOUND.matcher(s);
+
+    // If the error given is "table/column not found", it is possible that the message
+    // was generated by a complex column that looks like a table
+    // (e.g. mytbl.my_complex_column) which is currently not supported. We check for
+    // this possibility by seeing if the 'table not found' is identified as a column
+    // within one of the tables in the query. This check isn't fool-proof in that
+    // it might actually be a table that also is a column name in another table.
+    // However, that case should be extremely rare, and the result would be that
+    // the wrong error message will show up.
+    if (m.matches()) {
+      if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) {
+        throw new UnsupportedFeatureException(
+            "Complex column " + m.group(1) + " not supported.");
+      }
+    }
+
+    m = COLUMN_NOT_FOUND.matcher(s);
+    if (m.matches()) {
+      if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) {
+        throw new UnsupportedFeatureException(
+            "Complex column " + m.group(1) + " not supported.");
+      }
+    }
+  }
+}