jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-30 12:02:10 -05:00
Code Issues Projects Releases Wiki Activity
Files
1495b2007d9645053dae71ca4dbf4e3a48558ef4
impala/testdata/workloads/functional-query/queries/QueryTest/insert_permutation.test
Sailesh Mukil ed7f5ebf53 IMPALA-1878: Support INSERT and LOAD DATA on S3 and between filesystems 
Previously Impala disallowed LOAD DATA and INSERT on S3. This patch
functionally enables LOAD DATA and INSERT on S3 without making major
changes for the sake of improving performance over S3. This patch also
enables both INSERT and LOAD DATA between file systems.

S3 does not support the rename operation, so the staged files in S3
are copied instead of renamed, which contributes to the slow
performance on S3.

The FinalizeSuccessfulInsert() function now does not make any
underlying assumptions of the filesystem it is on and works across
all supported filesystems. This is done by adding a full URI field to
the base directory for a partition in the TInsertPartitionStatus.
Also, the HdfsOp class now does not assume a single filesystem and
gets connections to the filesystems based on the URI of the file it
is operating on.

Added a python S3 client called 'boto3' to access S3 from the python
tests. A new class called S3Client is introduced which creates
wrappers around the boto3 functions and have the same function
signatures as PyWebHdfsClient by deriving from a base abstract class
BaseFileSystem so that they can be interchangeably through a
'generic_client'. test_load.py is refactored to use this generic
client. The ImpalaTestSuite setup creates a client according to the
TARGET_FILESYSTEM environment variable and assigns it to the
'generic_client'.

P.S: Currently, the test_load.py runs 4x slower on S3 than on
HDFS. Performance needs to be improved in future patches. INSERT
performance is slower than on HDFS too. This is mainly because of an
extra copy that happens between staging and the final location of a
file. However, larger INSERTs come closer to HDFS permformance than
smaller inserts.

ACLs are not taken care of for S3 in this patch. It is something
that still needs to be discussed before implementing.

Change-Id: I94e15ad67752dce21c9b7c1dced6e114905a942d
Reviewed-on: http://gerrit.cloudera.org:8080/2574
Reviewed-by: Sailesh Mukil <sailesh@cloudera.com>
Tested-by: Internal Jenkins
2016-05-12 14:17:49 -07:00

337 lines
7.0 KiB
Plaintext
Raw Blame History

====
---- QUERY
create database insert_permutation_test location
'$FILESYSTEM_PREFIX/test-warehouse/insert_permutation_test'
---- RESULTS
====
---- QUERY
use insert_permutation_test
---- RESULTS
====
---- QUERY
create table perm_nopart(int_col1 int, string_col string, int_col2 int);
create table perm_part(int_col1 int, string_col string) partitioned by (p1 int, p2 string);
create table parquet_part(int_col1 int, string_col string)
partitioned by (p1 int, p2 string) stored as parquet;
---- RESULTS
====
---- QUERY
# Simple non-permutation
insert into perm_nopart(int_col1, string_col, int_col2) values(1,'str',2)
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 1
====
---- QUERY
select * from perm_nopart
---- RESULTS
1,'str',2
---- TYPES
INT,STRING,INT
====
---- QUERY
# Permute the int columns
insert into perm_nopart(int_col2, string_col, int_col1) values(1,'str',2)
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 1
====
---- QUERY
select * from perm_nopart
---- RESULTS
2,'str',1
---- TYPES
INT,STRING,INT
====
---- QUERY
# Leave out two columns, check they are assigned NULL
insert into perm_nopart(int_col2) values(1)
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 1
====
---- QUERY
select * from perm_nopart
---- RESULTS
NULL,'NULL',1
---- TYPES
INT,STRING,INT
====
---- QUERY
# Permute the partition columns
insert into perm_part(p1, string_col, int_col1, p2) values(10,'str',1, 'hello')
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
1,'str',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Same thing - permute the partition columns, but invert their order relative to Hive
insert into perm_part(p2, string_col, int_col1, p1) values('hello','str',1, 10)
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
1,'str',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check NULL if only partition keys are mentioned
insert into perm_part(p2, p1) values('hello', 10)
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
NULL,'NULL',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check NULL if only partition keys are mentioned, one static
insert into perm_part(p2) PARTITION(p1=10) values('hello')
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
NULL,'NULL',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check dynamic keys mentioned in the PARTITION column are still looked for at the end of
# the select-list
insert into perm_part(int_col1, string_col) PARTITION(p1=10, p2) values(1,'perm_col','part_col')
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=part_col/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
1,'perm_col',10,'part_col'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check behaviour of empty permutation clause with no query statement
insert into perm_part() PARTITION(p1=10, p2='foo')
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=foo/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
NULL,'NULL',10,'foo'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check behaviour of empty permutation clause
insert into perm_part() PARTITION(p1, p2='foo') values(5)
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=5/p2=foo/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
NULL,'NULL',5,'foo'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check behaviour of empty permutation clause with unpartitioned table
insert into perm_nopart()
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 1
====
---- QUERY
select * from perm_nopart
---- RESULTS
NULL,'NULL',NULL
---- TYPES
INT,STRING,INT
====
---- QUERY
# Test inserting NULLs due to an incomplete column mapping into a Parquet table.
# Regression test against IMPALA-671.
insert into parquet_part() partition(p1, p2='foo') values(2)
---- RESULTS
p1=2/p2=foo/: 1
====
---- QUERY
select * from parquet_part
---- RESULTS
NULL,'NULL',2,'foo'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Perform the same set of queries, but with SELECT clauses rather than VALUES
# Simple non-permutation
insert into perm_nopart(int_col1, string_col, int_col2) select 1,'str',2
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 1
====
---- QUERY
select * from perm_nopart
---- RESULTS
1,'str',2
---- TYPES
INT,STRING,INT
====
---- QUERY
# Permute the int columns
insert into perm_nopart(int_col2, string_col, int_col1) select 1,'str',2
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 1
====
---- QUERY
select * from perm_nopart
---- RESULTS
2,'str',1
---- TYPES
INT,STRING,INT
====
---- QUERY
# Leave out two columns, check they are assigned NULL
insert into perm_nopart(int_col2) select 1
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 1
====
---- QUERY
select * from perm_nopart
---- RESULTS
NULL,'NULL',1
---- TYPES
INT,STRING,INT
====
---- QUERY
# Permute the partition columns
insert into perm_part(p1, string_col, int_col1, p2) select 10,'str',1, 'hello'
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
1,'str',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Same thing - permute the partition columns, but invert their order relative to Hive
insert into perm_part(p2, string_col, int_col1, p1) select 'hello','str',1, 10
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
1,'str',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check NULL if only partition keys are mentioned
insert into perm_part(p2, p1) select 'hello', 10
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
NULL,'NULL',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Check NULL if only partition keys are mentioned, one static
insert into perm_part(p2) PARTITION(p1=10) select 'hello'
---- SETUP
RESET insert_permutation_test.perm_part
---- RESULTS
p1=10/p2=hello/: 1
====
---- QUERY
select * from perm_part
---- RESULTS
NULL,'NULL',10,'hello'
---- TYPES
INT,STRING,INT,STRING
====
---- QUERY
# Now some queries that use SELECT FROM
# Simple non-permutation
insert into perm_nopart(int_col1, string_col, int_col2) select 1,'str',2 FROM
functional.alltypes LIMIT 2
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 2
====
---- QUERY
select * from perm_nopart
---- RESULTS
1,'str',2
1,'str',2
---- TYPES
INT,STRING,INT
====
---- QUERY
insert into perm_nopart(int_col1) select id FROM functional.alltypes ORDER BY ID LIMIT 2
---- SETUP
RESET insert_permutation_test.perm_nopart
---- RESULTS
: 2
====
---- QUERY
select * from perm_nopart
---- RESULTS
0,'NULL',NULL
1,'NULL',NULL
---- TYPES
INT,STRING,INT
====
Reference in New Issue View Git Blame Copy Permalink