mirror of
https://github.com/apache/impala.git
synced 2026-01-04 18:00:57 -05:00
bf27a31f98
This change moves (almost) all the functional data loading to the new data loading framework. This removes the need for the create.sql, load.sql, and load-raw-data.sql file. Instead we just have the single schema template file: testdata/datasets/functional/functional_schema_template.sql This template can be used to generate the schema for all file formats and compression variations. It also should help make loading data easier. Now you can run: bin/load-impala-data.sh "query-test" "exhaustive" And get all data needed for running the query tests. This change also includes the initial changes for new dataset/workload directory structure. The new structure looks like: testdata/workload <- Will contain query files and test vectors/dimensions testdata/datasets <- WIll contain the data files and schema templates Note: This is the first part of the change to this directory structure - it's not yet complete. # Please enter the commit message for your changes. Lines starting
21 lines
1.4 KiB
SQL
21 lines
1.4 KiB
SQL
-- Load tables that depend upon data in the hive test-warehouse already existing
|
|
|
|
-- Load a mixed-format table. Hive behaves oddly when mixing formats,
|
|
-- but the following incantation ensures that the result is a
|
|
-- three-partition table. First is text format, second is sequence
|
|
-- file, third is RC file. Must be called after test-warehouse is
|
|
-- successfully populated
|
|
INSERT OVERWRITE TABLE alltypesmixedformat PARTITION (year=2009, month=1)
|
|
SELECT id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
|
|
FROM alltypes WHERE year=2009 and month=1;
|
|
|
|
ALTER TABLE alltypesmixedformat SET FILEFORMAT SEQUENCEFILE;
|
|
LOAD DATA INPATH '/tmp/alltypes_seq/year=2009/month=2/' OVERWRITE INTO TABLE alltypesmixedformat PARTITION (year=2009, month=2);
|
|
ALTER TABLE alltypesmixedformat SET FILEFORMAT RCFILE;
|
|
LOAD DATA INPATH '/tmp/alltypes_rc/year=2009/month=3/' OVERWRITE INTO TABLE alltypesmixedformat PARTITION (year=2009, month=3);
|
|
|
|
ALTER TABLE alltypesmixedformat PARTITION (year=2009, month=1) SET FILEFORMAT TEXTFILE;
|
|
ALTER TABLE alltypesmixedformat PARTITION (year=2009, month=1) SET SERDEPROPERTIES('field.delim'=',', 'escape.delim'='\\');
|
|
ALTER TABLE alltypesmixedformat PARTITION(year=2009, month=2) SET SERDEPROPERTIES('field.delim'='\001');
|
|
ALTER TABLE alltypesmixedformat PARTITION (year=2009, month=2) SET FILEFORMAT SEQUENCEFILE;
|