mirror of
https://github.com/apache/impala.git
synced 2026-01-27 06:10:53 -05:00
c77a457520
This patch makes external data source working in LocalCatalog mode: - Add APIs in CatalogdMetaProvider to fetch DataSource from Catalog server through RPC. - Add getDataSources() and getDataSource() in LocalCatalog. - Add LocalDataSourceTable class for loading DataSource table in LocalCatalog. - Handle request for loading DataSource in CatalogServiceCatalog on Catalog server. - Enable tests which are skipped by SkipIfCatalogV2.data_sources_unsupported(). Remove SkipIfCatalogV2.data_sources_unsupported(). - Add end-to-end tests for LocalCatalog mode. Testing: - Passed core tests Change-Id: I40841c9be9064ac67771c4d3f5acbb3b552a2e55 Reviewed-on: http://gerrit.cloudera.org:8080/20574 Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Reviewed-by: Wenzhe Zhou <wzhou@cloudera.com>
196 lines
5.4 KiB
Plaintext
196 lines
5.4 KiB
Plaintext
====
|
|
---- QUERY
|
|
# Create DataSource
|
|
DROP DATA SOURCE IF EXISTS TestJdbcDataSource;
|
|
CREATE DATA SOURCE TestJdbcDataSource
|
|
LOCATION '$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-data-source.jar'
|
|
CLASS 'org.apache.impala.extdatasource.jdbc.JdbcDataSource'
|
|
API_VERSION 'V1';
|
|
---- RESULTS
|
|
'Data source has been created.'
|
|
====
|
|
---- QUERY
|
|
# Show created DataSource
|
|
SHOW DATA SOURCES LIKE 'testjdbcdatasource';
|
|
---- LABELS
|
|
NAME,LOCATION,CLASS NAME,API VERSION
|
|
---- RESULTS
|
|
'testjdbcdatasource',regex:'.*/test-warehouse/data-sources/jdbc-data-source.jar','org.apache.impala.extdatasource.jdbc.JdbcDataSource','V1'
|
|
---- TYPES
|
|
STRING,STRING,STRING,STRING
|
|
====
|
|
---- QUERY
|
|
# Create external JDBC DataSource table
|
|
DROP TABLE IF EXISTS alltypes_jdbc_datasource;
|
|
CREATE TABLE alltypes_jdbc_datasource (
|
|
id INT,
|
|
bool_col BOOLEAN,
|
|
tinyint_col TINYINT,
|
|
smallint_col SMALLINT,
|
|
int_col INT,
|
|
bigint_col BIGINT,
|
|
float_col FLOAT,
|
|
double_col DOUBLE,
|
|
date_string_col STRING,
|
|
string_col STRING,
|
|
timestamp_col TIMESTAMP)
|
|
PRODUCED BY DATA SOURCE TestJdbcDataSource(
|
|
'{"database.type":"POSTGRES",
|
|
"jdbc.url":"jdbc:postgresql://$INTERNAL_LISTEN_HOST:5432/functional",
|
|
"jdbc.driver":"org.postgresql.Driver",
|
|
"driver.url":"$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/postgresql-jdbc.jar",
|
|
"dbcp.username":"hiveuser",
|
|
"dbcp.password":"password",
|
|
"table":"alltypes"}');
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
# Create external JDBC DataSource table
|
|
DROP TABLE IF EXISTS alltypes_jdbc_datasource_2;
|
|
CREATE TABLE alltypes_jdbc_datasource_2 (
|
|
id INT,
|
|
bool_col BOOLEAN,
|
|
tinyint_col TINYINT,
|
|
smallint_col SMALLINT,
|
|
int_col INT,
|
|
bigint_col BIGINT,
|
|
float_col FLOAT,
|
|
double_col DOUBLE,
|
|
date_string_col STRING,
|
|
string_col STRING,
|
|
timestamp_col TIMESTAMP)
|
|
PRODUCED BY DATA SOURCE TestJdbcDataSource(
|
|
'{"database.type":"POSTGRES",
|
|
"jdbc.url":"jdbc:postgresql://$INTERNAL_LISTEN_HOST:5432/functional",
|
|
"jdbc.driver":"org.postgresql.Driver",
|
|
"driver.url":"$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/postgresql-jdbc.jar",
|
|
"dbcp.username":"hiveuser",
|
|
"dbcp.password":"password",
|
|
"table":"AllTypesWithQuote",
|
|
"column.mapping":"id=id, bool_col=Bool_col, tinyint_col=Tinyint_col, smallint_col=Smallint_col, int_col=Int_col, bigint_col=Bigint_col, float_col=Float_col, double_col=Double_col, date_string_col=Date_string_col, string_col=String_col, timestamp=Timestamp"}');
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
# Test the jdbc DataSource
|
|
# count(*) with a predicate evaluated by Impala
|
|
select count(*) from alltypes_jdbc_datasource
|
|
where float_col = 0 and string_col is not NULL
|
|
---- RESULTS
|
|
730
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
# count(*) with no predicates has no materialized slots
|
|
select count(*) from alltypes_jdbc_datasource
|
|
---- RESULTS
|
|
7300
|
|
---- TYPES
|
|
BIGINT
|
|
====
|
|
---- QUERY
|
|
# Gets all types including a row with a NULL value. The predicate pushed to
|
|
# the DataSource.
|
|
select *
|
|
from alltypes_jdbc_datasource
|
|
where id > 10 and int_col< 5 limit 5
|
|
---- RESULTS
|
|
11,false,1,1,1,10,1.100000023841858,10.1,'01/02/09','1',2009-01-02 00:11:00.450000000
|
|
12,true,2,2,2,20,2.200000047683716,20.2,'01/02/09','2',2009-01-02 00:12:00.460000000
|
|
13,false,3,3,3,30,3.299999952316284,30.3,'01/02/09','3',2009-01-02 00:13:00.480000000
|
|
14,true,4,4,4,40,4.400000095367432,40.4,'01/02/09','4',2009-01-02 00:14:00.510000000
|
|
20,true,0,0,0,0,0,0,'01/03/09','0',2009-01-03 00:20:00.900000000
|
|
---- TYPES
|
|
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIMESTAMP
|
|
====
|
|
---- QUERY
|
|
# Gets specified columns.
|
|
select id, bool_col, smallint_col, float_col, double_col, date_string_col
|
|
from alltypes_jdbc_datasource
|
|
where id > 10 and int_col< 5 limit 5
|
|
---- RESULTS
|
|
11,false,1,1.100000023841858,10.1,'01/02/09'
|
|
12,true,2,2.200000047683716,20.2,'01/02/09'
|
|
13,false,3,3.299999952316284,30.3,'01/02/09'
|
|
14,true,4,4.400000095367432,40.4,'01/02/09'
|
|
20,true,0,0,0,'01/03/09'
|
|
---- TYPES
|
|
INT, BOOLEAN, SMALLINT, FLOAT, DOUBLE, STRING
|
|
====
|
|
---- QUERY
|
|
# Gets specified columns from external jdbc table with case sensitive column names
|
|
# and table name.
|
|
select id, bool_col, smallint_col, float_col, double_col, date_string_col
|
|
from alltypes_jdbc_datasource_2
|
|
where id > 10 and int_col< 5 limit 5
|
|
---- RESULTS
|
|
11,false,1,1.100000023841858,10.1,'01/02/09'
|
|
12,true,2,2.200000047683716,20.2,'01/02/09'
|
|
13,false,3,3.299999952316284,30.3,'01/02/09'
|
|
14,true,4,4.400000095367432,40.4,'01/02/09'
|
|
20,true,0,0,0,'01/03/09'
|
|
---- TYPES
|
|
INT, BOOLEAN, SMALLINT, FLOAT, DOUBLE, STRING
|
|
====
|
|
---- QUERY
|
|
# Inner join with a non jdbc table
|
|
select a.id, b.int_col
|
|
from alltypes_jdbc_datasource a inner join functional.alltypes b on (a.id = b.id)
|
|
where a.id = 1
|
|
---- RESULTS
|
|
1,1
|
|
---- TYPES
|
|
INT, INT
|
|
====
|
|
---- QUERY
|
|
# Inner join with another jdbc table
|
|
select a.id, b.int_col
|
|
from alltypes_jdbc_datasource a inner join alltypes_jdbc_datasource_2 b on (a.id = b.id)
|
|
where a.id < 3 group by a.id, b.int_col
|
|
---- RESULTS
|
|
0,0
|
|
1,1
|
|
2,2
|
|
---- TYPES
|
|
INT, INT
|
|
====
|
|
---- QUERY
|
|
# Cross join
|
|
select a.id, b.id
|
|
from alltypes_jdbc_datasource a cross join alltypes_jdbc_datasource b
|
|
where (a.id < 3 and b.id < 3)
|
|
order by a.id, b.id limit 10
|
|
---- RESULTS
|
|
0,0
|
|
0,1
|
|
0,2
|
|
1,0
|
|
1,1
|
|
1,2
|
|
2,0
|
|
2,1
|
|
2,2
|
|
---- TYPES
|
|
INT, INT
|
|
====
|
|
---- QUERY
|
|
# Drop table
|
|
DROP TABLE alltypes_jdbc_datasource;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
# Drop table
|
|
DROP TABLE alltypes_jdbc_datasource_2;
|
|
---- RESULTS
|
|
'Table has been dropped.'
|
|
====
|
|
---- QUERY
|
|
# Drop DataSource
|
|
DROP DATA SOURCE TestJdbcDataSource;
|
|
---- RESULTS
|
|
'Data source has been dropped.'
|
|
====
|