From 1cab95066d856ab949b2777c0bc0f4ebe7eabf4f Mon Sep 17 00:00:00 2001 From: Nong Li Date: Fri, 11 Apr 2014 13:21:39 -0700 Subject: [PATCH] Add the return type as a column for SHOW FUNCTIONS. Also includes some misc pattern matching cleanup. Change-Id: I6c9ec78b094a73864b4d669afbd75a48c9bf9585 Reviewed-on: http://gerrit.ent.cloudera.com:8080/2199 Tested-by: jenkins Reviewed-by: Nong Li Reviewed-on: http://gerrit.ent.cloudera.com:8080/2271 --- be/src/service/query-exec-state.cc | 16 ++- be/src/service/query-exec-state.h | 2 + common/thrift/Frontend.thrift | 1 + .../com/cloudera/impala/catalog/Catalog.java | 36 +----- .../impala/catalog/CatalogServiceCatalog.java | 5 +- .../java/com/cloudera/impala/catalog/Db.java | 36 ++---- .../com/cloudera/impala/service/Frontend.java | 13 +- .../cloudera/impala/service/JniFrontend.java | 13 +- .../cloudera/impala/service/MetadataOp.java | 49 ++----- .../cloudera/impala/util/PatternMatcher.java | 89 +++++++++++++ .../cloudera/impala/catalog/CatalogTest.java | 34 +++-- .../queries/QueryTest/functions-ddl.test | 120 +++++++++--------- tests/common/impala_test_suite.py | 7 +- 13 files changed, 238 insertions(+), 183 deletions(-) create mode 100644 fe/src/main/java/com/cloudera/impala/util/PatternMatcher.java diff --git a/be/src/service/query-exec-state.cc b/be/src/service/query-exec-state.cc index 45eaaf76f..e81814cfa 100644 --- a/be/src/service/query-exec-state.cc +++ b/be/src/service/query-exec-state.cc @@ -213,7 +213,7 @@ Status ImpalaServer::QueryExecState::ExecLocalCatalogOp( params->__isset.show_pattern ? (¶ms->show_pattern) : NULL; RETURN_IF_ERROR(frontend_->GetFunctions( params->type, params->db, fn_pattern, &query_ctxt_.session, &functions)); - SetResultSet(functions.fn_signatures); + SetResultSet(functions.fn_ret_types, functions.fn_signatures); return Status::OK; } case TCatalogOpType::DESCRIBE: { @@ -727,6 +727,20 @@ void ImpalaServer::QueryExecState::SetResultSet(const vector& results) { } } +void ImpalaServer::QueryExecState::SetResultSet(const vector& col1, + const vector& col2) { + DCHECK_EQ(col1.size(), col2.size()); + + request_result_set_.reset(new vector); + request_result_set_->resize(col1.size()); + for (int i = 0; i < col1.size(); ++i) { + (*request_result_set_.get())[i].__isset.colVals = true; + (*request_result_set_.get())[i].colVals.resize(2); + (*request_result_set_.get())[i].colVals[0].__set_stringVal(col1[i]); + (*request_result_set_.get())[i].colVals[1].__set_stringVal(col2[i]); + } +} + void ImpalaServer::QueryExecState::SetCreateTableAsSelectResultSet() { DCHECK(ddl_type() == TDdlType::CREATE_TABLE_AS_SELECT); int total_num_rows_inserted = 0; diff --git a/be/src/service/query-exec-state.h b/be/src/service/query-exec-state.h index d9bc45a68..11a4545e3 100644 --- a/be/src/service/query-exec-state.h +++ b/be/src/service/query-exec-state.h @@ -320,6 +320,8 @@ class ImpalaServer::QueryExecState { // Copies results into request_result_set_ void SetResultSet(const std::vector& results); + void SetResultSet(const std::vector& col1, + const std::vector& col2); // Sets the result set for a CREATE TABLE AS SELECT statement. The results will not be // ready until all BEs complete execution. This can be called as part of Wait(), diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift index 37b36b43d..9cabe2909 100644 --- a/common/thrift/Frontend.thrift +++ b/common/thrift/Frontend.thrift @@ -181,6 +181,7 @@ struct TGetFunctionsParams { // getFunctions() returns a list of function signatures struct TGetFunctionsResult { 1: list fn_signatures + 2: list fn_ret_types } // Parameters for the USE db command diff --git a/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java b/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java index a9e791cda..12f0de0c8 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/Catalog.java @@ -15,14 +15,11 @@ package com.cloudera.impala.catalog; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import com.cloudera.impala.analysis.ArithmeticExpr; import com.cloudera.impala.analysis.BinaryPredicate; @@ -35,9 +32,9 @@ import com.cloudera.impala.builtins.ScalarBuiltins; import com.cloudera.impala.catalog.MetaStoreClientPool.MetaStoreClient; import com.cloudera.impala.thrift.TCatalogObject; import com.cloudera.impala.thrift.TFunction; -import com.cloudera.impala.thrift.TFunctionType; import com.cloudera.impala.thrift.TPartitionKeyValue; import com.cloudera.impala.thrift.TTableName; +import com.cloudera.impala.util.PatternMatcher; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -221,18 +218,6 @@ public abstract class Catalog { return db.removeFunction(desc); } - /** - * Returns all the function for 'type' in this DB. - */ - public List getFunctionSignatures(TFunctionType type, String dbName, - String pattern) throws DatabaseNotFoundException { - Db db = getDb(dbName); - if (db == null) { - throw new DatabaseNotFoundException("Database '" + dbName + "' not found"); - } - return filterStringsByPattern(db.getAllFunctionSignatures(type), pattern); - } - /** * Returns true if there is a function with this function name. Parameters * are ignored. @@ -271,24 +256,9 @@ public abstract class Catalog { if (matchPattern == null) { filtered = Lists.newArrayList(candidates); } else { - List patterns = Lists.newArrayList(); - // Hive ignores pretty much all metacharacters, so we have to escape them. - final String metaCharacters = "+?.^()]\\/{}"; - final Pattern regex = Pattern.compile("([" + Pattern.quote(metaCharacters) + "])"); - - for (String pattern: Arrays.asList(matchPattern.split("\\|"))) { - Matcher matcher = regex.matcher(pattern); - pattern = matcher.replaceAll("\\\\$1").replace("*", ".*"); - patterns.add(pattern); - } - + PatternMatcher matcher = PatternMatcher.createHivePatternMatcher(matchPattern); for (String candidate: candidates) { - for (String pattern: patterns) { - // Empty string matches nothing in Hive's implementation - if (!pattern.isEmpty() && candidate.matches(pattern)) { - filtered.add(candidate); - } - } + if (matcher.matches(candidate)) filtered.add(candidate); } } Collections.sort(filtered, String.CASE_INSENSITIVE_ORDER); diff --git a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java index 81a797e89..0506909f0 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java @@ -37,6 +37,7 @@ import com.cloudera.impala.thrift.TGetAllCatalogObjectsResponse; import com.cloudera.impala.thrift.TTable; import com.cloudera.impala.thrift.TTableName; import com.cloudera.impala.thrift.TUniqueId; +import com.cloudera.impala.util.PatternMatcher; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -185,9 +186,7 @@ public class CatalogServiceCatalog extends Catalog { resp.addToObjects(catalogTbl); } - for (String signature: db.getAllFunctionSignatures(null)) { - Function fn = db.getFunction(signature); - if (fn == null) continue; + for (Function fn: db.getFunctions(null, new PatternMatcher())) { TCatalogObject function = new TCatalogObject(TCatalogObjectType.FUNCTION, fn.getCatalogVersion()); function.setType(TCatalogObjectType.FUNCTION); diff --git a/fe/src/main/java/com/cloudera/impala/catalog/Db.java b/fe/src/main/java/com/cloudera/impala/catalog/Db.java index 9398615fb..7c5751ca0 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/Db.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/Db.java @@ -17,7 +17,6 @@ package com.cloudera.impala.catalog; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import org.apache.log4j.Logger; @@ -25,6 +24,7 @@ import com.cloudera.impala.catalog.Function.CompareMode; import com.cloudera.impala.thrift.TCatalogObjectType; import com.cloudera.impala.thrift.TDatabase; import com.cloudera.impala.thrift.TFunctionType; +import com.cloudera.impala.util.PatternMatcher; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -116,27 +116,6 @@ public class Db implements CatalogObject { return tableCache_.remove(tableName.toLowerCase()); } - /** - * Returns all the function signatures in this DB that match the specified - * function type. If the function type is null, all function signatures are returned. - */ - public List getAllFunctionSignatures(TFunctionType type) { - List names = Lists.newArrayList(); - synchronized (functions_) { - for (List fns: functions_.values()) { - for (Function f: fns) { - if (!f.userVisible()) continue; - if (type == null || - (type == TFunctionType.SCALAR && f instanceof ScalarFunction) || - (type == TFunctionType.AGGREGATE && f instanceof AggregateFunction)) { - names.add(f.signatureString()); - } - } - } - } - return names; - } - /** * Returns the number of functions in this database. */ @@ -272,15 +251,20 @@ public class Db implements CatalogObject { } /** - * Returns all functions that match 'p'. + * Returns all functions that match 'fnPattern'. */ - public List getFunctions(Pattern p) { + public List getFunctions(TFunctionType type, PatternMatcher fnPattern) { List functions = Lists.newArrayList(); synchronized (functions_) { for (Map.Entry> fns: functions_.entrySet()) { - if (p.matcher(fns.getKey()).matches()) { + if (fnPattern.matches(fns.getKey())) { for (Function fn: fns.getValue()) { - if (fn.userVisible()) functions.add(fn); + if (fn.userVisible()) { + if (type == null || + (type == TFunctionType.SCALAR && fn instanceof ScalarFunction) || + (type == TFunctionType.AGGREGATE && fn instanceof AggregateFunction)) + functions.add(fn); + } } } } diff --git a/fe/src/main/java/com/cloudera/impala/service/Frontend.java b/fe/src/main/java/com/cloudera/impala/service/Frontend.java index 93d313c09..d3c66f8cd 100644 --- a/fe/src/main/java/com/cloudera/impala/service/Frontend.java +++ b/fe/src/main/java/com/cloudera/impala/service/Frontend.java @@ -52,6 +52,7 @@ import com.cloudera.impala.catalog.Column; import com.cloudera.impala.catalog.ColumnType; import com.cloudera.impala.catalog.DatabaseNotFoundException; import com.cloudera.impala.catalog.Db; +import com.cloudera.impala.catalog.Function; import com.cloudera.impala.catalog.HBaseTable; import com.cloudera.impala.catalog.HdfsTable; import com.cloudera.impala.catalog.ImpaladCatalog; @@ -93,6 +94,7 @@ import com.cloudera.impala.thrift.TStmtType; import com.cloudera.impala.thrift.TTableName; import com.cloudera.impala.thrift.TUpdateCatalogCacheRequest; import com.cloudera.impala.thrift.TUpdateCatalogCacheResponse; +import com.cloudera.impala.util.PatternMatcher; import com.cloudera.impala.util.TResultRowBuilder; import com.cloudera.impala.util.TSessionStateUtil; import com.google.common.base.Joiner; @@ -177,7 +179,8 @@ public class Frontend { ShowFunctionsStmt stmt = (ShowFunctionsStmt)analysis.getStmt(); ddl.setShow_fns_params(stmt.toThrift()); metadata.setColumns(Arrays.asList( - new TColumn("name", ColumnType.STRING.toThrift()))); + new TColumn("return type", ColumnType.STRING.toThrift()), + new TColumn("signature", ColumnType.STRING.toThrift()))); } else if (analysis.isShowCreateTableStmt()) { ddl.op_type = TCatalogOpType.SHOW_CREATE_TABLE; ddl.setShow_create_table_params(analysis.getShowCreateTableStmt().toThrift()); @@ -431,9 +434,13 @@ public class Frontend { * Returns all function signatures that match the pattern. If pattern is null, * matches all functions. */ - public List getFunctions(TFunctionType type, String dbName, String fnPattern) + public List getFunctions(TFunctionType type, String dbName, String fnPattern) throws DatabaseNotFoundException { - return impaladCatalog_.getFunctionSignatures(type, dbName, fnPattern); + Db db = impaladCatalog_.getDb(dbName); + if (db == null) { + throw new DatabaseNotFoundException("Database '" + dbName + "' not found"); + } + return db.getFunctions(type, PatternMatcher.createHivePatternMatcher(fnPattern)); } /** diff --git a/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java b/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java index 9e7afa94f..019791a28 100644 --- a/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java +++ b/fe/src/main/java/com/cloudera/impala/service/JniFrontend.java @@ -49,6 +49,7 @@ import com.cloudera.impala.authorization.AuthorizationConfig; import com.cloudera.impala.authorization.ImpalaInternalAdminUser; import com.cloudera.impala.authorization.Privilege; import com.cloudera.impala.authorization.User; +import com.cloudera.impala.catalog.Function; import com.cloudera.impala.common.FileSystemUtil; import com.cloudera.impala.common.ImpalaException; import com.cloudera.impala.common.InternalException; @@ -75,6 +76,7 @@ import com.cloudera.impala.thrift.TUpdateCatalogCacheRequest; import com.cloudera.impala.util.GlogAppender; import com.cloudera.impala.util.TSessionStateUtil; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; /** * JNI-callable interface onto a wrapped Frontend instance. The main point is to serialise @@ -258,8 +260,15 @@ public class JniFrontend { JniUtil.deserializeThrift(protocolFactory_, params, thriftGetFunctionsParams); TGetFunctionsResult result = new TGetFunctionsResult(); - result.setFn_signatures( - frontend_.getFunctions(params.type, params.db, params.pattern)); + List signatures = Lists.newArrayList(); + List retTypes = Lists.newArrayList(); + List fns = frontend_.getFunctions(params.type, params.db, params.pattern); + for (Function fn: fns) { + signatures.add(fn.signatureString()); + retTypes.add(fn.getReturnType().toString()); + } + result.setFn_signatures(signatures); + result.setFn_ret_types(retTypes); TSerializer serializer = new TSerializer(protocolFactory_); try { return serializer.serialize(result); diff --git a/fe/src/main/java/com/cloudera/impala/service/MetadataOp.java b/fe/src/main/java/com/cloudera/impala/service/MetadataOp.java index d460cf07a..8abbd17e1 100644 --- a/fe/src/main/java/com/cloudera/impala/service/MetadataOp.java +++ b/fe/src/main/java/com/cloudera/impala/service/MetadataOp.java @@ -18,7 +18,6 @@ import java.sql.DatabaseMetaData; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +39,7 @@ import com.cloudera.impala.thrift.TColumnValue; import com.cloudera.impala.thrift.TResultRow; import com.cloudera.impala.thrift.TResultSet; import com.cloudera.impala.thrift.TResultSetMetadata; +import com.cloudera.impala.util.PatternMatcher; import com.google.common.collect.Lists; /** @@ -251,29 +251,21 @@ public class MetadataOp { return result; } - // Creates the schema, table and column search patterns - String convertedSchemaPattern = convertPattern(schemaName); - String convertedTablePattern = convertPattern(tableName); - String convertedColumnPattern = convertPattern(columnName); - String convertedFunctionPattern = convertPattern(functionName); - Pattern schemaPattern = Pattern.compile(convertedSchemaPattern); - Pattern tablePattern = Pattern.compile(convertedTablePattern); - Pattern columnPattern = Pattern.compile(convertedColumnPattern); - Pattern functionPattern = Pattern.compile(convertedFunctionPattern); + // Creates the schema, table, column and function search patterns + PatternMatcher schemaPattern = PatternMatcher.createJdbcPatternMatcher(schemaName); + PatternMatcher tablePattern = PatternMatcher.createJdbcPatternMatcher(tableName); + PatternMatcher columnPattern = PatternMatcher.createJdbcPatternMatcher(columnName); + PatternMatcher fnPattern = PatternMatcher.createJdbcPatternMatcher(functionName); for (String dbName: catalog.getDbNames(null, user)) { - if (!schemaPattern.matcher(dbName).matches()) { - continue; - } + if (!schemaPattern.matches(dbName)) continue; Db db = catalog.getDb(dbName, user, Privilege.ANY); List tableList = Lists.newArrayList(); List> tablesColumnsList = Lists.newArrayList(); for (String tabName: catalog.getTableNames(db.getName(), "*", user)) { - if (!tablePattern.matcher(tabName).matches()) { - continue; - } + if (!tablePattern.matches(tabName)) continue; tableList.add(tabName); List columns = Lists.newArrayList(); @@ -292,9 +284,7 @@ public class MetadataOp { } else { for (Column column: table.getColumns()) { String colName = column.getName(); - if (!columnPattern.matcher(colName).matches()) { - continue; - } + if (!columnPattern.matches(colName)) continue; columns.add(column); } } @@ -302,7 +292,7 @@ public class MetadataOp { } if (functionName != null) { - List fns = db.getFunctions(functionPattern); + List fns = db.getFunctions(null, fnPattern); result.functions.add(fns); } @@ -622,23 +612,4 @@ public class MetadataOp { return (pattern == null) || pattern.isEmpty() || (pattern.length() == 1 && pattern.equals("%")); } - - /** - * Convert a pattern containing JDBC catalog search wildcard into Java regex patterns. - */ - public static String convertPattern(final String pattern) { - String wildcardPattern = ".*"; - String workPattern = pattern; - if (workPattern == null || pattern.isEmpty()) { - workPattern = "%"; - } - String result = workPattern - .replaceAll("([^\\\\])%", "$1" + wildcardPattern) - .replaceAll("\\\\%", "%") - .replaceAll("^%", wildcardPattern) - .replaceAll("([^\\\\])_", "$1.") - .replaceAll("\\\\_", "_") - .replaceAll("^_", "."); - return result; - } } diff --git a/fe/src/main/java/com/cloudera/impala/util/PatternMatcher.java b/fe/src/main/java/com/cloudera/impala/util/PatternMatcher.java new file mode 100644 index 000000000..18f282bec --- /dev/null +++ b/fe/src/main/java/com/cloudera/impala/util/PatternMatcher.java @@ -0,0 +1,89 @@ +// Copyright 2014 Cloudera Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.cloudera.impala.util; + +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.collect.Lists; + +/** + * Utility class to handle pattern-matching for different types of patterns ( + * e.g. hive SHOW patterns, JDBC patterns). + * It maps those patterns onto the java regex pattern objects. + */ +public class PatternMatcher { + // Patterns to match against. A string is considered to match if it matches + // any of the patterns. + private List patterns_; + + // Returns true if the candidate matches. + public boolean matches(String candidate) { + if (patterns_ == null) return true; + if (patterns_.isEmpty()) return false; + for (Pattern pattern: patterns_) { + if (pattern.matcher(candidate).matches()) return true; + } + return false; + } + + /** + * Creates a pattern matcher for hive patterns. + * The only metacharacters are '*' which matches any string of characters, and '|' + * which denotes choice. + * If hivePattern is null, all strings are considered to match. If it is the + * empty string, no strings match. + */ + public static PatternMatcher createHivePatternMatcher(String hivePattern) { + PatternMatcher result = new PatternMatcher(); + if (hivePattern != null) { + result.patterns_ = Lists.newArrayList(); + // Hive ignores pretty much all metacharacters, so we have to escape them. + final String metaCharacters = "+?.^()]\\/{}"; + final Pattern regex = Pattern.compile("([" + Pattern.quote(metaCharacters) + "])"); + + for (String pattern: Arrays.asList(hivePattern.split("\\|"))) { + Matcher matcher = regex.matcher(pattern); + pattern = matcher.replaceAll("\\\\$1").replace("*", ".*"); + result.patterns_.add(Pattern.compile(pattern)); + } + } + return result; + } + + /** + * Creates a matcher object for JDBC match strings. + */ + public static PatternMatcher createJdbcPatternMatcher(String pattern) { + String wildcardPattern = ".*"; + String workPattern = pattern; + if (workPattern == null || pattern.isEmpty()) { + workPattern = "%"; + } + String result = workPattern + .replaceAll("([^\\\\])%", "$1" + wildcardPattern) + .replaceAll("\\\\%", "%") + .replaceAll("^%", wildcardPattern) + .replaceAll("([^\\\\])_", "$1.") + .replaceAll("\\\\_", "_") + .replaceAll("^_", "."); + PatternMatcher matcher = new PatternMatcher(); + matcher.patterns_ = Lists.newArrayList(); + matcher.patterns_.add(Pattern.compile(result)); + return matcher; + } +} diff --git a/fe/src/test/java/com/cloudera/impala/catalog/CatalogTest.java b/fe/src/test/java/com/cloudera/impala/catalog/CatalogTest.java index cac3cb79e..c37363e1c 100644 --- a/fe/src/test/java/com/cloudera/impala/catalog/CatalogTest.java +++ b/fe/src/test/java/com/cloudera/impala/catalog/CatalogTest.java @@ -22,7 +22,6 @@ import com.cloudera.impala.analysis.HdfsUri; import com.cloudera.impala.analysis.IntLiteral; import com.cloudera.impala.analysis.LiteralExpr; import com.cloudera.impala.catalog.MetaStoreClientPool.MetaStoreClient; -import com.cloudera.impala.thrift.TFunctionType; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -535,10 +534,18 @@ public class CatalogTest { } } + private List getFunctionSignatures(String db) throws DatabaseNotFoundException { + List fns = catalog_.getFunctions(db); + List names = Lists.newArrayList(); + for (Function fn: fns) { + names.add(fn.signatureString()); + } + return names; + } + @Test public void TestUdf() throws CatalogException { - List fnNames = - catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", ""); + List fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 0); ArrayList args1 = Lists.newArrayList(); @@ -548,14 +555,13 @@ public class CatalogTest { catalog_.removeFunction( new Function(new FunctionName("default", "Foo"), args1, ColumnType.INVALID, false)); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); - + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 0); ScalarFunction udf1 = new ScalarFunction(new FunctionName("default", "Foo"), args1, ColumnType.INVALID, new HdfsUri("/Foo"), "Foo.class", null, null); catalog_.addFunction(udf1); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 1); assertTrue(fnNames.contains("foo()")); @@ -563,7 +569,7 @@ public class CatalogTest { ScalarFunction udf2 = new ScalarFunction(new FunctionName("default", "Foo"), args2, ColumnType.INVALID, new HdfsUri("/Foo"), "Foo.class", null, null); catalog_.addFunction(udf2); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 2); assertTrue(fnNames.contains("foo()")); assertTrue(fnNames.contains("foo(INT)")); @@ -572,7 +578,7 @@ public class CatalogTest { ScalarFunction udf3 = new ScalarFunction(new FunctionName("default", "Bar"), args2, ColumnType.INVALID, new HdfsUri("/Foo"), "Foo.class", null, null); catalog_.addFunction(udf3); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 3); assertTrue(fnNames.contains("foo()")); assertTrue(fnNames.contains("foo(INT)")); @@ -581,7 +587,7 @@ public class CatalogTest { // Drop Foo() catalog_.removeFunction(new Function( new FunctionName("default", "Foo"), args1, ColumnType.INVALID, false)); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 2); assertTrue(fnNames.contains("foo(INT)")); assertTrue(fnNames.contains("bar(INT)")); @@ -589,7 +595,7 @@ public class CatalogTest { // Drop it again, no-op catalog_.removeFunction(new Function( new FunctionName("default", "Foo"), args1, ColumnType.INVALID, false)); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 2); assertTrue(fnNames.contains("foo(INT)")); assertTrue(fnNames.contains("bar(INT)")); @@ -597,7 +603,7 @@ public class CatalogTest { // Drop bar(), no-op catalog_.removeFunction(new Function( new FunctionName("default", "Bar"), args1, ColumnType.INVALID, false)); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 2); assertTrue(fnNames.contains("foo(INT)")); assertTrue(fnNames.contains("bar(INT)")); @@ -605,7 +611,7 @@ public class CatalogTest { // Drop bar(tinyint), no-op catalog_.removeFunction(new Function( new FunctionName("default", "Bar"), args3, ColumnType.INVALID, false)); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 2); assertTrue(fnNames.contains("foo(INT)")); assertTrue(fnNames.contains("bar(INT)")); @@ -613,14 +619,14 @@ public class CatalogTest { // Drop bar(int) catalog_.removeFunction(new Function( new FunctionName("default", "Bar"), args2, ColumnType.INVALID, false)); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 1); assertTrue(fnNames.contains("foo(INT)")); // Drop foo(int) catalog_.removeFunction(new Function( new FunctionName("default", "foo"), args2, ColumnType.INVALID, false)); - fnNames = catalog_.getFunctionSignatures(TFunctionType.SCALAR, "default", null); + fnNames = getFunctionSignatures("default"); assertEquals(fnNames.size(), 0); } } diff --git a/testdata/workloads/functional-query/queries/QueryTest/functions-ddl.test b/testdata/workloads/functional-query/queries/QueryTest/functions-ddl.test index f2ee87b93..fb0f296c9 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/functions-ddl.test +++ b/testdata/workloads/functional-query/queries/QueryTest/functions-ddl.test @@ -17,14 +17,14 @@ drop function if exists function_ddl_test.agg_fn(int, string); show functions in function_ddl_test ---- RESULTS ---- TYPES -STRING +STRING, STRING ==== ---- QUERY # Verify all the test functions are removed show aggregate functions in function_ddl_test ---- RESULTS ---- TYPES -STRING +STRING, STRING ==== ---- QUERY # Add them and test function overloading and scoping. @@ -70,30 +70,30 @@ LOCATION '/test-warehouse/libTestUdas.so' UPDATE_FN='TwoArgUpdate' ---- QUERY show functions in default ---- RESULTS -'fn()' +'INT','fn()' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY show functions in function_ddl_test ---- RESULTS -'fn()' -'fn(INT)' -'fn(INT, STRING)' -'fn(STRING, INT)' -'fn2(INT)' -'fn2(INT, STRING)' -'fn_var_arg(INT...)' +'INT','fn_var_arg(INT...)' +'INT','fn2(INT)' +'INT','fn2(INT, STRING)' +'INT','fn()' +'DOUBLE','fn(INT)' +'INT','fn(INT, STRING)' +'INT','fn(STRING, INT)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY show aggregate functions in function_ddl_test ---- RESULTS -'agg_fn(INT)' -'agg_fn(INT, STRING)' +'BIGINT','agg_fn(INT)' +'INT','agg_fn(INT, STRING)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY drop function function_ddl_test.fn2(int, string) @@ -101,14 +101,14 @@ drop function function_ddl_test.fn2(int, string) ---- QUERY show functions ---- RESULTS -'fn()' -'fn(INT)' -'fn(INT, STRING)' -'fn(STRING, INT)' -'fn2(INT)' -'fn_var_arg(INT...)' +'INT','fn_var_arg(INT...)' +'INT','fn2(INT)' +'INT','fn()' +'DOUBLE','fn(INT)' +'INT','fn(INT, STRING)' +'INT','fn(STRING, INT)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY drop function if exists function_ddl_test.fn2(int, string) @@ -116,21 +116,21 @@ drop function if exists function_ddl_test.fn2(int, string) ---- QUERY show functions in function_ddl_test ---- RESULTS -'fn()' -'fn(INT)' -'fn(INT, STRING)' -'fn(STRING, INT)' -'fn2(INT)' -'fn_var_arg(INT...)' +'INT','fn_var_arg(INT...)' +'INT','fn2(INT)' +'INT','fn()' +'DOUBLE','fn(INT)' +'INT','fn(INT, STRING)' +'INT','fn(STRING, INT)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY show functions in default; ---- RESULTS -'fn()' +'INT','fn()' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY drop function default.fn() @@ -139,19 +139,19 @@ drop function default.fn() show functions in default; ---- RESULTS ---- TYPES -STRING +STRING, STRING ==== ---- QUERY show functions in function_ddl_test; ---- RESULTS -'fn()' -'fn(INT)' -'fn(INT, STRING)' -'fn(STRING, INT)' -'fn2(INT)' -'fn_var_arg(INT...)' +'INT','fn_var_arg(INT...)' +'INT','fn2(INT)' +'INT','fn()' +'DOUBLE','fn(INT)' +'INT','fn(INT, STRING)' +'INT','fn(STRING, INT)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY drop function fn() @@ -159,13 +159,13 @@ drop function fn() ---- QUERY show functions; ---- RESULTS -'fn(INT)' -'fn(INT, STRING)' -'fn(STRING, INT)' -'fn2(INT)' -'fn_var_arg(INT...)' +'INT','fn_var_arg(INT...)' +'INT','fn2(INT)' +'DOUBLE','fn(INT)' +'INT','fn(INT, STRING)' +'INT','fn(STRING, INT)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY drop function fn_var_arg(INT...) @@ -173,12 +173,12 @@ drop function fn_var_arg(INT...) ---- QUERY show functions; ---- RESULTS -'fn(INT)' -'fn(INT, STRING)' -'fn(STRING, INT)' -'fn2(INT)' +'INT','fn2(INT)' +'DOUBLE','fn(INT)' +'INT','fn(INT, STRING)' +'INT','fn(STRING, INT)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY drop function agg_fn(int) @@ -186,9 +186,9 @@ drop function agg_fn(int) ---- QUERY show aggregate functions ---- RESULTS -'agg_fn(INT, STRING)' +'INT','agg_fn(INT, STRING)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY # Call invalidate metadata and make sure the functions are still there @@ -199,17 +199,17 @@ invalidate metadata ---- QUERY show functions ---- RESULTS -'fn(INT)' -'fn(INT, STRING)' -'fn(STRING, INT)' -'fn2(INT)' +'INT','fn2(INT)' +'DOUBLE','fn(INT)' +'INT','fn(INT, STRING)' +'INT','fn(STRING, INT)' ---- TYPES -STRING +STRING, STRING ==== ---- QUERY show aggregate functions ---- RESULTS -'agg_fn(INT, STRING)' +'INT','agg_fn(INT, STRING)' ---- TYPES -STRING -==== +STRING, STRING +==== \ No newline at end of file diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index 7cec8bd48..e2b13c4d3 100755 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -123,9 +123,12 @@ class ImpalaTestSuite(BaseTestSuite): self.client.execute("drop view " + full_tbl_name) else: self.client.execute("drop table " + full_tbl_name) - for fn_name in self.client.execute("show functions in " + db_name).data: + for fn_result in self.client.execute("show functions in " + db_name).data: + # First column is the return type, second is the function signature + fn_name = fn_result.split('\t')[1] self.client.execute("drop function %s.%s" % (db_name, fn_name)) - for fn_name in self.client.execute("show aggregate functions in " + db_name).data: + for fn_result in self.client.execute("show aggregate functions in " + db_name).data: + fn_name = fn_result.split('\t')[1] self.client.execute("drop function %s.%s" % (db_name, fn_name)) self.client.execute("drop database " + db_name)