mirror of
https://github.com/apache/impala.git
synced 2026-02-01 12:00:22 -05:00
f8e8cd0906
This patch uses JDBC connection string to apply query options to the Impala server by setting the properties in "jdbc.properties" when creating JDBC external DataSource table. jdbc.properties are specified as comma-delimited key=value string, like "MEM_LIMIT=1000000000, ENABLED_RUNTIME_FILTER_TYPES=\"BLOOM,MIN_MAX\"". Fixed Impala to allow value of ENABLED_RUNTIME_FILTER_TYPES to have double quotes in the beginning and ending of string. jdbc.properties can be used for other databases like Postgres and MySQL to set additional properties. The test cases will be added in separate patch. Testing: - Added end-to-end tests for setting query options on Impala JDBC tables. - Passed core tests. Change-Id: I47687b7a93e90cea8ebd5f3fc280c9135bd97992 Reviewed-on: http://gerrit.cloudera.org:8080/20837 Reviewed-by: Abhishek Rawat <arawat@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
197 lines
5.5 KiB
Plaintext
197 lines
5.5 KiB
Plaintext
====
|
|
---- QUERY
|
|
# Create DataSource
|
|
DROP DATA SOURCE IF EXISTS TestJdbcDataSource;
|
|
CREATE DATA SOURCE TestJdbcDataSource
|
|
LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-data-source.jar'
|
|
CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource'
|
|
API_VERSION 'V1';
|
|
---- RESULTS
|
|
'Data source has been created.'
|
|
====
|
|
---- QUERY
|
|
# Show created DataSource
|
|
SHOW DATA SOURCES LIKE 'testjdbcdatasource';
|
|
---- LABELS
|
|
NAME,LOCATION,CLASS NAME,API VERSION
|
|
---- RESULTS
|
|
'testjdbcdatasource',regex:'.*/test-warehouse/data-sources/jdbc-data-source.jar','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
|
|
---- TYPES
|
|
STRING,STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
# Create external JDBC DataSource table
|
|
DROP TABLE IF EXISTS alltypes_jdbc_datasource;
|
|
CREATE TABLE alltypes_jdbc_datasource (
|
|
id INT,
|
|
bool_col BOOLEAN,
|
|
tinyint_col TINYINT,
|
|
smallint_col SMALLINT,
|
|
int_col INT,
|
|
bigint_col BIGINT,
|
|
float_col FLOAT,
|
|
double_col DOUBLE,
|
|
date_string_col STRING,
|
|
string_col STRING,
|
|
timestamp_col TIMESTAMP)
|
|
PRODUCED BY DATA SOURCE TestJdbcDataSource(
|
|
'{"database.type":"POSTGRES",
|
|
"jdbc.url":"jdbc:postgresql://$INTERNAL_LISTEN_HOST:5432/functional",
|
|
"jdbc.properties":"connect_timeout=20, application_name=\"myapp\"",
|
|
"jdbc.driver":"org.postgresql.Driver",
|
|
"driver.url":"$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/postgresql-jdbc.jar",
|
|
"dbcp.username":"hiveuser",
|
|
"dbcp.password":"password",
|
|
"table":"alltypes"}');
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
# Create external JDBC DataSource table
|
|
DROP TABLE IF EXISTS alltypes_jdbc_datasource_2;
|
|
CREATE TABLE alltypes_jdbc_datasource_2 (
|
|
id INT,
|
|
bool_col BOOLEAN,
|
|
tinyint_col TINYINT,
|
|
smallint_col SMALLINT,
|
|
int_col INT,
|
|
bigint_col BIGINT,
|
|
float_col FLOAT,
|
|
double_col DOUBLE,
|
|
date_string_col STRING,
|
|
string_col STRING,
|
|
timestamp_col TIMESTAMP)
|
|
PRODUCED BY DATA SOURCE TestJdbcDataSource(
|
|
'{"database.type":"POSTGRES",
|
|
"jdbc.url":"jdbc:postgresql://$INTERNAL_LISTEN_HOST:5432/functional",
|
|
"jdbc.driver":"org.postgresql.Driver",
|
|
"driver.url":"$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/postgresql-jdbc.jar",
|
|
"dbcp.username":"hiveuser",
|
|
"dbcp.password":"password",
|
|
"table":"AllTypesWithQuote",
|
|
"column.mapping":"id=id, bool_col=Bool_col, tinyint_col=Tinyint_col, smallint_col=Smallint_col, int_col=Int_col, bigint_col=Bigint_col, float_col=Float_col, double_col=Double_col, date_string_col=Date_string_col, string_col=String_col, timestamp=Timestamp"}');
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
# Test the jdbc DataSource
|
|
# count(*) with a predicate evaluated by Impala
|
|
select count(*) from alltypes_jdbc_datasource
|
|
where float_col = 0 and string_col is not NULL
|
|
---- RESULTS
|
|
730
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
# count(*) with no predicates has no materialized slots
|
|
select count(*) from alltypes_jdbc_datasource
|
|
---- RESULTS
|
|
7300
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
# Gets all types including a row with a NULL value. The predicate pushed to
|
|
# the DataSource.
|
|
select *
|
|
from alltypes_jdbc_datasource
|
|
where id > 10 and int_col< 5 limit 5
|
|
---- RESULTS
|
|
11,false,1,1,1,10,1.100000023841858,10.1,'01/02/09','1',2009-01-02 00:11:00.450000000
|
|
12,true,2,2,2,20,2.200000047683716,20.2,'01/02/09','2',2009-01-02 00:12:00.460000000
|
|
13,false,3,3,3,30,3.299999952316284,30.3,'01/02/09','3',2009-01-02 00:13:00.480000000
|
|
14,true,4,4,4,40,4.400000095367432,40.4,'01/02/09','4',2009-01-02 00:14:00.510000000
|
|
20,true,0,0,0,0,0,0,'01/03/09','0',2009-01-03 00:20:00.900000000
|
|
---- TYPES
|
|
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIMESTAMP
|
|
====
|
|
---- QUERY
|
|
# Gets specified columns.
|
|
select id, bool_col, smallint_col, float_col, double_col, date_string_col
|
|
from alltypes_jdbc_datasource
|
|
where id > 10 and int_col< 5 limit 5
|
|
---- RESULTS
|
|
11,false,1,1.100000023841858,10.1,'01/02/09'
|
|
12,true,2,2.200000047683716,20.2,'01/02/09'
|
|
13,false,3,3.299999952316284,30.3,'01/02/09'
|
|
14,true,4,4.400000095367432,40.4,'01/02/09'
|
|
20,true,0,0,0,'01/03/09'
|
|
---- TYPES
|
|
INT, BOOLEAN, SMALLINT, FLOAT, DOUBLE, STRING
|
|
====
|
|
---- QUERY
|
|
# Gets specified columns from external jdbc table with case sensitive column names
|
|
# and table name.
|
|
select id, bool_col, smallint_col, float_col, double_col, date_string_col
|
|
from alltypes_jdbc_datasource_2
|
|
where id > 10 and int_col< 5 limit 5
|
|
---- RESULTS
|
|
11,false,1,1.100000023841858,10.1,'01/02/09'
|
|
12,true,2,2.200000047683716,20.2,'01/02/09'
|
|
13,false,3,3.299999952316284,30.3,'01/02/09'
|
|
14,true,4,4.400000095367432,40.4,'01/02/09'
|
|
20,true,0,0,0,'01/03/09'
|
|
---- TYPES
|
|
INT, BOOLEAN, SMALLINT, FLOAT, DOUBLE, STRING
|
|
====
|
|
---- QUERY
|
|
# Inner join with a non jdbc table
|
|
select a.id, b.int_col
|
|
from alltypes_jdbc_datasource a inner join functional.alltypes b on (a.id = b.id)
|
|
where a.id = 1
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT, INT
|
|
====
|
|
---- QUERY
|
|
# Inner join with another jdbc table
|
|
select a.id, b.int_col
|
|
from alltypes_jdbc_datasource a inner join alltypes_jdbc_datasource_2 b on (a.id = b.id)
|
|
where a.id < 3 group by a.id, b.int_col
|
|
---- RESULTS
|
|
0,0
|
|
1,1
|
|
2,2
|
|
---- TYPES
|
|
INT, INT
|
|
====
|
|
---- QUERY
|
|
# Cross join
|
|
select a.id, b.id
|
|
from alltypes_jdbc_datasource a cross join alltypes_jdbc_datasource b
|
|
where (a.id < 3 and b.id < 3)
|
|
order by a.id, b.id limit 10
|
|
---- RESULTS
|
|
0,0
|
|
0,1
|
|
0,2
|
|
1,0
|
|
1,1
|
|
1,2
|
|
2,0
|
|
2,1
|
|
2,2
|
|
---- TYPES
|
|
INT, INT
|
|
====
|
|
---- QUERY
|
|
# Drop table
|
|
DROP TABLE alltypes_jdbc_datasource;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
# Drop table
|
|
DROP TABLE alltypes_jdbc_datasource_2;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
# Drop DataSource
|
|
DROP DATA SOURCE TestJdbcDataSource;
|
|
---- RESULTS
|
|
'Data source has been dropped.'
|
|
====
|