Files
impala/testdata/datasets/functional/schema_constraints.csv
Juan Yu c9b33ddf63 IMPALA-1886/IMPALA-2154: Add support for multi-stream bz2/gzip compressed files.
Fix a bug in which Impala only reads the first stream
of a multi-stream bz2/gzip file.
Changes the bz2 decoder to read the file in a streaming
fashion rather than reading the entire file into memory
before it can be decompressed.

Change-Id: Icbe617d03a69953f0bf3aa0f7c30d34bc612f9f8
(cherry picked from commit b6d0b4e059329633dc50f1f73ebe35b7ac317a8e)
Reviewed-on: http://gerrit.cloudera.org:8080/2219
Reviewed-by: Juan Yu <jyu@cloudera.com>
Tested-by: Internal Jenkins
2016-02-28 21:31:37 -08:00

10 KiB

1# Table level constraints:
2# Allows for defining constraints on which file formats to generate for an individual
3# table. The table name should match the base table name defined in the schema template
4# file.
5table_name:stringids, constraint:restrict_to, table_format:hbase/none/none
6table_name:hbasecolumnfamilies, constraint:restrict_to, table_format:hbase/none/none
7table_name:insertalltypesagg, constraint:restrict_to, table_format:hbase/none/none
8table_name:alltypessmallbinary, constraint:restrict_to, table_format:hbase/none/none
9table_name:insertalltypesaggbinary, constraint:restrict_to, table_format:hbase/none/none
10table_name:hbasealltypeserror, constraint:restrict_to, table_format:hbase/none/none
11table_name:hbasealltypeserrornonulls, constraint:restrict_to, table_format:hbase/none/none
12table_name:alltypesinsert, constraint:restrict_to, table_format:text/none/none
13table_name:alltypesnopart_insert, constraint:restrict_to, table_format:text/none/none
14table_name:insert_overwrite_nopart, constraint:restrict_to, table_format:text/none/none
15table_name:insert_overwrite_partitioned, constraint:restrict_to, table_format:text/none/none
16table_name:insert_string_partitioned, constraint:restrict_to, table_format:text/none/none
17table_name:alltypesinsert, constraint:restrict_to, table_format:parquet/none/none
18table_name:alltypesnopart_insert, constraint:restrict_to, table_format:parquet/none/none
19table_name:alltypesinsert, constraint:restrict_to, table_format:text/none/none
20table_name:alltypesnopart_insert, constraint:restrict_to, table_format:text/none/none
21table_name:insert_overwrite_nopart, constraint:restrict_to, table_format:text/none/none
22table_name:insert_overwrite_partitioned, constraint:restrict_to, table_format:text/none/none
23table_name:insert_string_partitioned, constraint:restrict_to, table_format:text/none/none
24table_name:alltypesinsert, constraint:restrict_to, table_format:parquet/none/none
25table_name:alltypesnopart_insert, constraint:restrict_to, table_format:parquet/none/none
26table_name:insert_overwrite_nopart, constraint:restrict_to, table_format:parquet/none/none
27table_name:insert_overwrite_partitioned, constraint:restrict_to, table_format:parquet/none/none
28table_name:insert_string_partitioned, constraint:restrict_to, table_format:parquet/none/none
29table_name:old_rcfile_table, constraint:restrict_to, table_format:rc/none/none
30table_name:bad_text_lzo, constraint:restrict_to, table_format:text/lzo/block
31table_name:bad_text_gzip, constraint:restrict_to, table_format:text/gzip/block
32table_name:bad_seq_snap, constraint:restrict_to, table_format:seq/snap/block
33table_name:bad_parquet, constraint:restrict_to, table_format:parquet/none/none
34table_name:bad_magic_number, constraint:restrict_to, table_format:parquet/none/none
35table_name:bad_metadata_len, constraint:restrict_to, table_format:parquet/none/none
36table_name:bad_dict_page_offset, constraint:restrict_to, table_format:parquet/none/none
37table_name:bad_compressed_size, constraint:restrict_to, table_format:parquet/none/none
38table_name:alltypesagg_hive_13_1, constraint:restrict_to, table_format:parquet/none/none
39table_name:kite_required_fields, constraint:restrict_to, table_format:parquet/none/none
40table_name:bad_column_metadata, constraint:restrict_to, table_format:parquet/none/none
41table_name:lineitem_multiblock, constraint:restrict_to, table_format:parquet/none/none
42# TODO: Support Avro. Data loading currently fails for Avro because complex types
43# cannot be converted to the corresponding Avro types yet.
44table_name:allcomplextypes, constraint:restrict_to, table_format:text/none/none
45table_name:allcomplextypes, constraint:restrict_to, table_format:parquet/none/none
46table_name:allcomplextypes, constraint:restrict_to, table_format:hbase/none/none
47table_name:functional, constraint:restrict_to, table_format:text/none/none
48table_name:complextypes_fileformat, constraint:restrict_to, table_format:text/none/none
49table_name:complextypes_fileformat, constraint:restrict_to, table_format:parquet/none/none
50table_name:complextypes_fileformat, constraint:restrict_to, table_format:avro/snap/block
51table_name:complextypes_fileformat, constraint:restrict_to, table_format:rc/snap/block
52table_name:complextypes_fileformat, constraint:restrict_to, table_format:seq/snap/block
53table_name:complextypes_multifileformat, constraint:restrict_to, table_format:text/none/none
54# TODO: Avro
55table_name:complextypestbl, constraint:restrict_to, table_format:parquet/none/none
56table_name:alltypeserror, constraint:exclude, table_format:parquet/none/none
57table_name:alltypeserrornonulls, constraint:exclude, table_format:parquet/none/none
58table_name:unsupported_types, constraint:exclude, table_format:parquet/none/none
59table_name:escapechartesttable, constraint:exclude, table_format:parquet/none/none
60table_name:TblWithRaggedColumns, constraint:exclude, table_format:parquet/none/none
61# the text_ tables are for testing test delimiters and escape chars in text files
62table_name:text_comma_backslash_newline, constraint:restrict_to, table_format:text/none/none
63table_name:text_dollar_hash_pipe, constraint:restrict_to, table_format:text/none/none
64table_name:text_thorn_ecirc_newline, constraint:restrict_to, table_format:text/none/none
65table_name:bad_serde, constraint:restrict_to, table_format:text/none/none
66table_name:rcfile_lazy_binary_serde, constraint:restrict_to, table_format:rc/none/none
67table_name:unsupported_partition_types, constraint:restrict_to, table_format:text/none/none
68table_name:nullformat_custom, constraint:exclude, table_format:parquet/none/none
69table_name:alltypes_view, constraint:restrict_to, table_format:text/none/none
70table_name:allcomplextypes_view, constraint:restrict_to, table_format:text/none/none
71table_name:alltypes_view, constraint:restrict_to, table_format:seq/snap/block
72table_name:alltypes_hive_view, constraint:restrict_to, table_format:text/none/none
73table_name:alltypes_view_sub, constraint:restrict_to, table_format:text/none/none
74table_name:alltypes_view_sub, constraint:restrict_to, table_format:seq/snap/block
75table_name:alltypes_parens, constraint:restrict_to, table_format:text/none/none
76table_name:complex_view, constraint:restrict_to, table_format:text/none/none
77table_name:complex_view, constraint:restrict_to, table_format:seq/snap/block
78table_name:view_view, constraint:restrict_to, table_format:text/none/none
79table_name:view_view, constraint:restrict_to, table_format:seq/snap/block
80# liketbl and tblwithraggedcolumns all have
81# NULLs in primary key columns. hbase does not support
82# writing NULLs to primary key columns.
83table_name:liketbl, constraint:exclude, table_format:hbase/none/none
84table_name:tblwithraggedcolumns, constraint:exclude, table_format:hbase/none/none
85# Tables with only one column are not supported in hbase.
86table_name:greptiny, constraint:exclude, table_format:hbase/none/none
87table_name:tinyinttable, constraint:exclude, table_format:hbase/none/none
88# overflow has a bigint that's too big. hbase may lose precision, hence this
89# table cannot be loaded.
90table_name:overflow, constraint:exclude, table_format:hbase/none/none
91# widerow has a single column with a single row containing a 10MB string. hbase doesn't
92# seem to like this.
93table_name:widerow, constraint:exclude, table_format:hbase/none/none
94# nullformat_custom is used in null-insert tests, which user insert overwrite,
95# which is not supported in hbase. The schema is also specified in HIVE_CREATE
96# with no corresponding LOAD statement.
97table_name:nullformat_custom, constraint:exclude, table_format:hbase/none/none
98table_name:unsupported_types, constraint:exclude, table_format:hbase/none/none
99# On CDH4, decimal can only be tested on formats Impala can write to (text and parquet)
100# TODO: add Avro once Hive or Impala can write Avro decimals
101table_name:decimal_tbl, constraint:restrict_to, table_format:text/none/none
102table_name:decimal_tiny, constraint:restrict_to, table_format:text/none/none
103table_name:decimal_tbl, constraint:restrict_to, table_format:parquet/none/none
104table_name:decimal_tiny, constraint:restrict_to, table_format:parquet/none/none
105table_name:avro_decimal_tbl, constraint:restrict_to, table_format:avro/snap/block
106# TODO first set of tests are for text/none/none
107table_name:chars_tiny, constraint:restrict_to, table_format:text/none/none
108# invalid_decimal_part_tbl[1,2,3] tables are used for testing invalid decimal
109# partition key values (see IMPALA-1040)
110table_name:invalid_decimal_part_tbl1, constraint:restrict_to, table_format:text/none/none
111table_name:invalid_decimal_part_tbl2, constraint:restrict_to, table_format:text/none/none
112table_name:invalid_decimal_part_tbl3, constraint:restrict_to, table_format:text/none/none
113table_name:avro_decimal_tbl, constraint:restrict_to, table_format:avro/snap/block
114# testescape tables are used for testing text scanner delimiter handling
115table_name:table_no_newline, constraint:restrict_to, table_format:text/none/none
116table_name:table_no_newline_part, constraint:restrict_to, table_format:text/none/none
117table_name:testescape_16_lf, constraint:restrict_to, table_format:text/none/none
118table_name:testescape_16_crlf, constraint:restrict_to, table_format:text/none/none
119table_name:testescape_17_lf, constraint:restrict_to, table_format:text/none/none
120table_name:testescape_17_crlf, constraint:restrict_to, table_format:text/none/none
121table_name:testescape_32_lf, constraint:restrict_to, table_format:text/none/none
122table_name:testescape_32_crlf, constraint:restrict_to, table_format:text/none/none
123# alltimezones is used to verify that impala properly deals with timezones
124table_name:alltimezones, constraint:restrict_to, table_format:text/none/none
125# Avro schema is inferred from the column definitions (IMPALA-1136)
126table_name:no_avro_schema, constraint:restrict_to, table_format:avro/snap/block
127table_name:avro_unicode_nulls, constraint:restrict_to, table_format:avro/snap/block
128# test single and multi stream bz2 files
129table_name:bzip2_tbl, constraint:restrict_to, table_format:text/bzip/block
130table_name:large_bzip2_tbl, constraint:restrict_to, table_format:text/bzip/block
131table_name:multistream_bzip2_tbl, constraint:restrict_to, table_format:text/bzip/block
132table_name:large_multistream_bzip2_tbl, constraint:restrict_to, table_format:text/bzip/block