mirror of
https://github.com/apache/impala.git
synced 2026-01-04 18:00:57 -05:00
9147cd7518
We were previously wasting memory by always reading into 8MB IO buffers, even when the data read was much less than 8MB. With this patch, the IO manager picks a buffer size closer to the actual amount being read (we don't use the exact size so we can continue to recycle buffers). The minimum IO buffer size is determined via the --min_buffer_size flag, and the max IO buffer size via the --read_size flag. This technique also helps with IMPALA-652, since short columns will not use as much memory as before (we will not use considerably more memory than the size of the table). This patch also changes StringBuffer to use a doubling strategy so it doesn't end up allocating many large unused buffers, and has the scanner context use the requested length as the sync read size if it's larger than the size produced by read_past_size_cb(). These changes help prevent the boundary buffer in the scanner context from allocating excess memory. Change-Id: I0efb3b023ddfddb08bca22d5cb5f9511fb4d6c50 Reviewed-on: http://gerrit.ent.cloudera.com:8080/938 Reviewed-by: Skye Wanderman-Milne <skye@cloudera.com> Tested-by: jenkins
5.5 KiB
5.5 KiB
| 1 | # Table level constraints: |
|---|---|
| 2 | # Allows for defining constraints on which file formats to generate for an individual |
| 3 | # table. The table name should match the base table name defined in the schema template |
| 4 | # file. |
| 5 | table_name:stringids, constraint:restrict_to, table_format:hbase/none/none |
| 6 | table_name:insertalltypesagg, constraint:restrict_to, table_format:hbase/none/none |
| 7 | table_name:alltypessmallbinary, constraint:restrict_to, table_format:hbase/none/none |
| 8 | table_name:insertalltypesaggbinary, constraint:restrict_to, table_format:hbase/none/none |
| 9 | table_name:hbasealltypeserror, constraint:restrict_to, table_format:hbase/none/none |
| 10 | table_name:hbasealltypeserrornonulls, constraint:restrict_to, table_format:hbase/none/none |
| 11 | table_name:alltypesinsert, constraint:restrict_to, table_format:text/none/none |
| 12 | table_name:alltypesnopart_insert, constraint:restrict_to, table_format:text/none/none |
| 13 | table_name:insert_overwrite_nopart, constraint:restrict_to, table_format:text/none/none |
| 14 | table_name:insert_overwrite_partitioned, constraint:restrict_to, table_format:text/none/none |
| 15 | table_name:insert_string_partitioned, constraint:restrict_to, table_format:text/none/none |
| 16 | table_name:alltypesinsert, constraint:restrict_to, table_format:parquet/none/none |
| 17 | table_name:alltypesnopart_insert, constraint:restrict_to, table_format:parquet/none/none |
| 18 | table_name:alltypesinsert, constraint:restrict_to, table_format:text/none/none |
| 19 | table_name:alltypesnopart_insert, constraint:restrict_to, table_format:text/none/none |
| 20 | table_name:insert_overwrite_nopart, constraint:restrict_to, table_format:text/none/none |
| 21 | table_name:insert_overwrite_partitioned, constraint:restrict_to, table_format:text/none/none |
| 22 | table_name:insert_string_partitioned, constraint:restrict_to, table_format:text/none/none |
| 23 | table_name:alltypesinsert, constraint:restrict_to, table_format:parquet/none/none |
| 24 | table_name:alltypesnopart_insert, constraint:restrict_to, table_format:parquet/none/none |
| 25 | table_name:insert_overwrite_nopart, constraint:restrict_to, table_format:parquet/none/none |
| 26 | table_name:insert_overwrite_partitioned, constraint:restrict_to, table_format:parquet/none/none |
| 27 | table_name:insert_string_partitioned, constraint:restrict_to, table_format:parquet/none/none |
| 28 | table_name:old_rcfile_table, constraint:restrict_to, table_format:rc/none/none |
| 29 | table_name:bad_text_lzo, constraint:restrict_to, table_format:text/lzo/block |
| 30 | table_name:bad_seq_snap, constraint:restrict_to, table_format:seq/snap/block |
| 31 | table_name:map_table, constraint:restrict_to, table_format:text/none/none |
| 32 | table_name:map_table_hbase, constraint:restrict_to, table_format:hbase/none/none |
| 33 | table_name:array_table, constraint:restrict_to, table_format:text/none/none |
| 34 | table_name:alltypeserror, constraint:exclude, table_format:parquet/none/none |
| 35 | table_name:alltypeserrornonulls, constraint:exclude, table_format:parquet/none/none |
| 36 | table_name:unsupported_types, constraint:exclude, table_format:parquet/none/none |
| 37 | table_name:escapechartesttable, constraint:exclude, table_format:parquet/none/none |
| 38 | table_name:nulltable, constraint:exclude, table_format:parquet/none/none |
| 39 | table_name:nullescapedtable, constraint:exclude, table_format:parquet/none/none |
| 40 | table_name:TblWithRaggedColumns, constraint:exclude, table_format:parquet/none/none |
| 41 | # the text_ tables are for testing test delimiters and escape chars in text files |
| 42 | table_name:text_comma_backslash_newline, constraint:restrict_to, table_format:text/none/none |
| 43 | table_name:text_dollar_hash_pipe, constraint:restrict_to, table_format:text/none/none |
| 44 | table_name:bad_serde, constraint:restrict_to, table_format:text/none/none |
| 45 | table_name:unsupported_partition_types, constraint:restrict_to, table_format:text/none/none |
| 46 | table_name:nullformat_custom, constraint:exclude, table_format:parquet/none/none |
| 47 | table_name:alltypes_view, constraint:restrict_to, table_format:text/none/none |
| 48 | table_name:alltypes_view, constraint:restrict_to, table_format:seq/snap/block |
| 49 | table_name:alltypes_view_sub, constraint:restrict_to, table_format:text/none/none |
| 50 | table_name:alltypes_view_sub, constraint:restrict_to, table_format:seq/snap/block |
| 51 | table_name:alltypes_parens, constraint:restrict_to, table_format:text/none/none |
| 52 | table_name:complex_view, constraint:restrict_to, table_format:text/none/none |
| 53 | table_name:complex_view, constraint:restrict_to, table_format:seq/snap/block |
| 54 | table_name:view_view, constraint:restrict_to, table_format:text/none/none |
| 55 | table_name:view_view, constraint:restrict_to, table_format:seq/snap/block |
| 56 | # liketbl and tblwithraggedcolumns all have |
| 57 | # NULLs in primary key columns. hbase does not support |
| 58 | # writing NULLs to primary key columns. |
| 59 | table_name:liketbl, constraint:exclude, table_format:hbase/none/none |
| 60 | table_name:tblwithraggedcolumns, constraint:exclude, table_format:hbase/none/none |
| 61 | # Tables with only one column are not supported in hbase. |
| 62 | table_name:greptiny, constraint:exclude, table_format:hbase/none/none |
| 63 | table_name:tinyinttable, constraint:exclude, table_format:hbase/none/none |
| 64 | # overflow has a bigint that's too big. hbase may lose precision, hence this |
| 65 | # table cannot be loaded. |
| 66 | table_name:overflow, constraint:exclude, table_format:hbase/none/none |
| 67 | # widerow has a single column with a single row containing a 10MB string. hbase doesn't |
| 68 | # seem to like this. |
| 69 | table_name:widerow, constraint:exclude, table_format:hbase/none/none |
| 70 | # nullformat_custom is used in null-insert tests, which user insert overwrite, |
| 71 | # which is not supported in hbase. The schema is also specified in HIVE_CREATE |
| 72 | # with no corresponding LOAD statement. |
| 73 | table_name:nullformat_custom, constraint:exclude, table_format:hbase/none/none |
| 74 | table_name:unsupported_types, constraint:exclude, table_format:hbase/none/none |