mirror of
https://github.com/apache/impala.git
synced 2025-12-31 06:02:51 -05:00
ef0dac661c
This commit adds a new feature to persist hive/java udfs across catalog restarts. IMPALA-1748 already added this for non-java udfs by storing them in parameters map of the Db object and reading them back at catalog startup. However we follow a different approach for hive udfs by converting them to Hive's function format and adding them as hive functions to the metastore. This makes it possible to share udfs between hive and Impala as the udfs added from one service are accessible to other. This commit takes care of format conversions between hive and impala and user can just add function once in either of the services. Background: Hive and impala treat udfs differently. Hive resolves the evaluate function in the udf class at runtime depending on the data types of the input arguments. So user can add one function by name and can pass any arguments to it as long as there is a compatible evaluate function in the udf class. However Impala takes the input types of the udf as a part of function definition (that maps to only one evaluate function) and loads the function only for those set of input argument types. If we have multiple 'evaluate' methods, we need to add multiple functions one for each of them. This commit adds new variants of CREATE | DROP FUNCTIONS to Impala which lets the user to create and drop hive/java udfs without input argument types or return types. Catalog takes care of loading/dropping the udf signatures corresponding to each "evaluate" method in the udf symbol class. The syntax is as follows, CREATE FUNCTION [IF NOT EXISTS] <function name> <function_opts> DROP FUNCTION [IF EXISTS] <function name> Examples: CREATE FUNCTION IF NOT EXISTS foo location '/path/to/jar' SYMBOL='TestUdf'; CREATE FUNCTION bar location '/path/to/jar' SYMBOL='TestUdf2'; DROP FUNCTION foo; DROP FUNCTION IF EXISTS bar; The older way of creating hive/java udfs with specific signature is still supported, however they are *not* persisted across restarts. So a restart of catalog can wipe them out. Additionally this commit also loads all the compatible java udfs added outside of Impala and they needn't be separately loaded. One thing to note here is that the functions added using the new CREATE FUNCTION can only be dropped using the new DROP FUNCTION syntax (without signature). The same rule applies for the java udfs added using the old CREATE FUNCTION syntax (with signature). Change-Id: If31ed3d5ac4192e3bc2d57610a9a0bbe1f62b42d Reviewed-on: http://gerrit.cloudera.org:8080/2250 Reviewed-by: Bharath Vissapragada <bharathv@cloudera.com> Tested-by: Internal Jenkins
125 lines
5.0 KiB
Plaintext
125 lines
5.0 KiB
Plaintext
====
|
|
---- QUERY
|
|
drop function if exists udf_test.hive_pi();
|
|
drop function if exists udf_test.hive_round(double);
|
|
drop function if exists udf_test.hive_floor(double);
|
|
drop function if exists udf_test.hive_mod(int, int);
|
|
drop function if exists udf_test.hive_bin(bigint);
|
|
drop function if exists udf_test.hive_lower(string);
|
|
|
|
drop function if exists udf_test.identity(boolean);
|
|
drop function if exists udf_test.identity(tinyint);
|
|
drop function if exists udf_test.identity(smallint);
|
|
drop function if exists udf_test.identity(int);
|
|
drop function if exists udf_test.identity(bigint);
|
|
drop function if exists udf_test.identity(float);
|
|
drop function if exists udf_test.identity(double);
|
|
drop function if exists udf_test.identity(string);
|
|
drop function if exists udf_test.identity(string, string);
|
|
drop function if exists udf_test.identity(string, string, string);
|
|
drop function if exists udf_test.identity(timestamp);
|
|
|
|
drop function if exists udf_test.hive_add(int, int);
|
|
drop function if exists udf_test.hive_add(float, float);
|
|
drop function if exists udf_test.hive_add(double, double);
|
|
drop function if exists udf_test.hive_add(smallint, smallint);
|
|
drop function if exists udf_test.hive_add(boolean, boolean);
|
|
|
|
drop function if exists udf_test.throws_exception();
|
|
|
|
drop function if exists java_udfs_test.identity;
|
|
|
|
create function udf_test.hive_pi() returns double
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/hive-exec.jar'
|
|
symbol='org.apache.hadoop.hive.ql.udf.UDFPI';
|
|
|
|
create function udf_test.hive_round(double) returns double
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/hive-exec.jar'
|
|
symbol='org.apache.hadoop.hive.ql.udf.UDFRound';
|
|
|
|
create function udf_test.hive_floor(double) returns bigint
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/hive-exec.jar'
|
|
symbol='org.apache.hadoop.hive.ql.udf.UDFFloor';
|
|
|
|
create function udf_test.hive_mod(int, int) returns int
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/hive-exec.jar'
|
|
symbol='org.apache.hadoop.hive.ql.udf.UDFPosMod';
|
|
|
|
create function udf_test.hive_bin(bigint) returns string
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/hive-exec.jar'
|
|
symbol='org.apache.hadoop.hive.ql.udf.UDFBin';
|
|
|
|
create function udf_test.hive_lower(string) returns string
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/hive-exec.jar'
|
|
symbol='org.apache.hadoop.hive.ql.udf.UDFLower';
|
|
|
|
# Used to test persistent java functions
|
|
create function java_udfs_test.identity
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(boolean) returns boolean
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(tinyint) returns tinyint
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(smallint) returns smallint
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(int) returns int
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(bigint) returns bigint
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(float) returns float
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(double) returns double
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(string) returns string
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(string, string) returns string
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.identity(string, string, string) returns string
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.hive_add(int, int) returns int
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.hive_add(smallint, smallint) returns smallint
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.hive_add(float, float) returns float
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.hive_add(double, double) returns double
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.hive_add(boolean, boolean) returns boolean
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdf';
|
|
|
|
create function udf_test.throws_exception() returns boolean
|
|
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
|
|
symbol='com.cloudera.impala.TestUdfException';
|
|
====
|