mirror of
https://github.com/apache/impala.git
synced 2026-01-22 09:01:58 -05:00
27fa27e808
This change is a follow-up to IMPALA-7368 and adds support for DATE type to the avro scanner. Similarly to parquet, avro uses DATE logical type for dates. DATE logical type annotates an INT32 that stores the number of days since the unix epoch, 1 January 1970. This representation introduces an avro interoperability issue between Impala and older versions of Hive: - Before version 3.1, Hive used Julian calendar to represent dates up to 1582-10-05 and Gregorian calendar for dates starting with 1582-10-15. Dates between 1582-10-05 and 1582-10-15 were lost. - Impala uses proleptic Gregorian calendar, extending the Gregorian calendar backward to dates preceding its official introduction in 1582-10-15. This means that pre-1582-10-15 dates written to an avro table by Hive will be read back incorrectly by Impala. Note that Hive 3.1 switched to proleptic Gregorian calendar too, so for Hive 3.1+ this is no longer an issue. Dependency changes: - BE uses avro 1.7.4-p5 from native-toolchain. Change-Id: I7a9d5b93a22cf3a00244037e187f8c145cacc959 Reviewed-on: http://gerrit.cloudera.org:8080/13944 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
124 lines
3.4 KiB
Plaintext
124 lines
3.4 KiB
Plaintext
====
|
|
---- QUERY
|
|
# Create a table partitioned by DATE.
|
|
create table $DATABASE.dtbl (c date) partitioned by (p date);
|
|
---- RESULTS
|
|
'Table has been created.'
|
|
====
|
|
---- QUERY
|
|
# Partition value is an invalid DATE literal
|
|
alter table $DATABASE.dtbl add partition (p='1300-1-');
|
|
---- CATCH
|
|
AnalysisException: Invalid date literal: '1300-1-'
|
|
====
|
|
---- QUERY
|
|
alter table $DATABASE.dtbl add partition (p='1300-1-1');
|
|
---- RESULTS
|
|
'New partition has been added to the table.'
|
|
====
|
|
---- QUERY
|
|
# Date formatted differently referring to the same partition.
|
|
alter table $DATABASE.dtbl add partition (p='1300-01-1');
|
|
---- CATCH
|
|
AnalysisException: Partition spec already exists: (p=DATE '1300-01-01').
|
|
====
|
|
---- QUERY
|
|
# Date partition formatted differently in insert
|
|
insert into $DATABASE.dtbl partition (p='1300-1-01') values ('1300-1-1');
|
|
---- RESULTS
|
|
p=1300-01-01/: 1
|
|
====
|
|
---- QUERY
|
|
insert into $DATABASE.dtbl partition (p='1300-01-1') values ('1300-1-02');
|
|
---- RESULTS
|
|
p=1300-01-01/: 1
|
|
====
|
|
---- QUERY
|
|
insert into $DATABASE.dtbl partition (p=DATE '1300-1-1') values ('1300-1-03');
|
|
---- RESULTS
|
|
p=1300-01-01/: 1
|
|
====
|
|
---- QUERY
|
|
# Insert into a new partition
|
|
insert into $DATABASE.dtbl partition (p=DATE '1400-01-1') values ('1400-1-1');
|
|
---- RESULTS
|
|
p=1400-01-01/: 1
|
|
====
|
|
---- QUERY
|
|
insert into $DATABASE.dtbl partition (p='1400-1-01') values ('1400-1-2');
|
|
---- RESULTS
|
|
p=1400-01-01/: 1
|
|
====
|
|
---- QUERY
|
|
select p, c from $DATABASE.dtbl;
|
|
---- RESULTS
|
|
1300-01-01,1300-01-01
|
|
1300-01-01,1300-01-02
|
|
1300-01-01,1300-01-03
|
|
1400-01-01,1400-01-01
|
|
1400-01-01,1400-01-02
|
|
---- TYPES
|
|
DATE,DATE
|
|
====
|
|
---- QUERY
|
|
# Create a table partitioned by STRING and fill up with date-like values.
|
|
create table $DATABASE.stbl (c string) partitioned by (p string);
|
|
insert into $DATABASE.stbl partition (p='1400-1-1') values ('1400-1-3');
|
|
insert into $DATABASE.stbl partition (p='1400-1-01') values ('1400-01-4');
|
|
insert into $DATABASE.stbl partition (p='1400-01-01') values ('400-01-4');
|
|
insert into $DATABASE.stbl partition (p='1400-01-') values ('1400-01-5');
|
|
insert into $DATABASE.stbl partition (p='1500-01-1') values ('1500-01-1');
|
|
====
|
|
---- QUERY
|
|
# Use dynamic partition specification in insert.
|
|
# When the implicit cast fails, insert fails with error
|
|
insert into $DATABASE.dtbl partition(p) select * from $DATABASE.stbl where p='1400-01-01';
|
|
---- CATCH
|
|
UDF ERROR: String to Date parse failed. Invalid string val: "400-01-4"
|
|
====
|
|
---- QUERY
|
|
insert into $DATABASE.dtbl partition(p) select * from $DATABASE.stbl where p='1400-01-';
|
|
---- CATCH
|
|
UDF ERROR: String to Date parse failed. Invalid string val: "1400-01-"
|
|
====
|
|
---- QUERY
|
|
# Use dynamic partition specification in insert.
|
|
# Test that STRING is implicitly cast to DATE.
|
|
insert into $DATABASE.dtbl partition(p) select * from $DATABASE.stbl
|
|
where p in ('1400-1-1', '1400-1-01', '1500-01-1');
|
|
---- RESULTS
|
|
p=1400-01-01/: 2
|
|
p=1500-01-01/: 1
|
|
====
|
|
---- QUERY
|
|
select p, c from $DATABASE.dtbl;
|
|
---- RESULTS
|
|
1300-01-01,1300-01-01
|
|
1300-01-01,1300-01-02
|
|
1300-01-01,1300-01-03
|
|
1400-01-01,1400-01-01
|
|
1400-01-01,1400-01-02
|
|
1400-01-01,1400-01-03
|
|
1400-01-01,1400-01-04
|
|
1500-01-01,1500-01-01
|
|
---- TYPES
|
|
DATE,DATE
|
|
====
|
|
---- QUERY
|
|
# Test dropping partitions
|
|
alter table $DATABASE.dtbl drop partition(p < '1400-1-1');
|
|
---- RESULTS
|
|
'Dropped 1 partition(s).'
|
|
====
|
|
---- QUERY
|
|
select p, c from $DATABASE.dtbl;
|
|
---- RESULTS
|
|
1400-01-01,1400-01-01
|
|
1400-01-01,1400-01-02
|
|
1400-01-01,1400-01-03
|
|
1400-01-01,1400-01-04
|
|
1500-01-01,1500-01-01
|
|
---- TYPES
|
|
DATE,DATE
|
|
====
|