mirror of
https://github.com/apache/impala.git
synced 2026-01-05 03:01:02 -05:00
452ebee59d
The value of PARQUET_FILE_SIZE overflows when RoundUp() is called because this function
returns an int32. Even with this change, this value will still overflow when calling the
HDFS API since it is passed to hdfsOpenFile() as blocksize, which is an int32 parameter
(see HDFS-8949).
Changes:
- Return an error if PARQUET_FILE_SIZE is set to a value greater than or equal to 2GB.
- If PARQUET_FILE_SIZE is set in an Impala session to a value greater than or equal to
2GB, then every query will fail with an error message.
- If PARQUET_FILE_SIZE is changed to a value greater than or equal to 2GB as an impalad
argument, impalad will not start and log an error.
- Ceil(), RoundUp(), RoundDown() return int64.
Change-Id: Ie4f2551b72954e2a57db5594e4789e3f7434d578
Reviewed-on: http://gerrit.cloudera.org:8080/678
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Reviewed-by: Vlad Berindei <vlad.berindei@cloudera.com>
Tested-by: Internal Jenkins
176 lines
3.5 KiB
Plaintext
176 lines
3.5 KiB
Plaintext
====
|
|
---- QUERY
|
|
set
|
|
---- RESULTS: VERIFY_IS_SUBSET
|
|
'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0'
|
|
'ABORT_ON_ERROR','0'
|
|
'ALLOW_UNSUPPORTED_FORMATS','0'
|
|
'BATCH_SIZE','0'
|
|
'DEBUG_ACTION',''
|
|
'DEFAULT_ORDER_BY_LIMIT','-1'
|
|
'DISABLE_CACHED_READS','0'
|
|
'DISABLE_CODEGEN','0'
|
|
'DISABLE_OUTERMOST_TOPN','0'
|
|
'EXPLAIN_LEVEL','1'
|
|
'HBASE_CACHE_BLOCKS','0'
|
|
'HBASE_CACHING','0'
|
|
'MAX_ERRORS','0'
|
|
'MAX_IO_BUFFERS','0'
|
|
'MAX_SCAN_RANGE_LENGTH','0'
|
|
'MEM_LIMIT','0'
|
|
'NUM_NODES','0'
|
|
'NUM_SCANNER_THREADS','0'
|
|
'COMPRESSION_CODEC','NONE'
|
|
'PARQUET_FILE_SIZE','0'
|
|
'REQUEST_POOL',''
|
|
'RESERVATION_REQUEST_TIMEOUT','0'
|
|
'RM_INITIAL_MEM','0'
|
|
'SYNC_DDL','0'
|
|
'V_CPU_CORES','0'
|
|
---- TYPES
|
|
STRING, STRING
|
|
====
|
|
---- QUERY
|
|
set explain_level=3
|
|
====
|
|
---- QUERY
|
|
set
|
|
---- RESULTS: VERIFY_IS_SUBSET
|
|
'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0'
|
|
'ABORT_ON_ERROR','0'
|
|
'ALLOW_UNSUPPORTED_FORMATS','0'
|
|
'BATCH_SIZE','0'
|
|
'DEBUG_ACTION',''
|
|
'DEFAULT_ORDER_BY_LIMIT','-1'
|
|
'DISABLE_CACHED_READS','0'
|
|
'DISABLE_CODEGEN','0'
|
|
'DISABLE_OUTERMOST_TOPN','0'
|
|
'EXPLAIN_LEVEL','3'
|
|
'HBASE_CACHE_BLOCKS','0'
|
|
'HBASE_CACHING','0'
|
|
'MAX_ERRORS','0'
|
|
'MAX_IO_BUFFERS','0'
|
|
'MAX_SCAN_RANGE_LENGTH','0'
|
|
'MEM_LIMIT','0'
|
|
'NUM_NODES','0'
|
|
'NUM_SCANNER_THREADS','0'
|
|
'COMPRESSION_CODEC','NONE'
|
|
'PARQUET_FILE_SIZE','0'
|
|
'REQUEST_POOL',''
|
|
'RESERVATION_REQUEST_TIMEOUT','0'
|
|
'RM_INITIAL_MEM','0'
|
|
'SYNC_DDL','0'
|
|
'V_CPU_CORES','0'
|
|
---- TYPES
|
|
STRING, STRING
|
|
====
|
|
---- QUERY
|
|
set explain_level='0'
|
|
====
|
|
---- QUERY
|
|
set
|
|
---- RESULTS: VERIFY_IS_SUBSET
|
|
'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0'
|
|
'ABORT_ON_ERROR','0'
|
|
'ALLOW_UNSUPPORTED_FORMATS','0'
|
|
'BATCH_SIZE','0'
|
|
'DEBUG_ACTION',''
|
|
'DEFAULT_ORDER_BY_LIMIT','-1'
|
|
'DISABLE_CACHED_READS','0'
|
|
'DISABLE_CODEGEN','0'
|
|
'DISABLE_OUTERMOST_TOPN','0'
|
|
'EXPLAIN_LEVEL','0'
|
|
'HBASE_CACHE_BLOCKS','0'
|
|
'HBASE_CACHING','0'
|
|
'MAX_ERRORS','0'
|
|
'MAX_IO_BUFFERS','0'
|
|
'MAX_SCAN_RANGE_LENGTH','0'
|
|
'MEM_LIMIT','0'
|
|
'NUM_NODES','0'
|
|
'NUM_SCANNER_THREADS','0'
|
|
'COMPRESSION_CODEC','NONE'
|
|
'PARQUET_FILE_SIZE','0'
|
|
'REQUEST_POOL',''
|
|
'RESERVATION_REQUEST_TIMEOUT','0'
|
|
'RM_INITIAL_MEM','0'
|
|
'SYNC_DDL','0'
|
|
'V_CPU_CORES','0'
|
|
---- TYPES
|
|
STRING, STRING
|
|
====
|
|
---- QUERY
|
|
# IMPALA-1906: Test that SET changes PARQUET_FILE_SIZE only if it's less than 2GB.
|
|
set parquet_file_size='1.5g'
|
|
====
|
|
---- QUERY
|
|
set
|
|
---- RESULTS: VERIFY_IS_SUBSET
|
|
'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0'
|
|
'ABORT_ON_ERROR','0'
|
|
'ALLOW_UNSUPPORTED_FORMATS','0'
|
|
'BATCH_SIZE','0'
|
|
'DEBUG_ACTION',''
|
|
'DEFAULT_ORDER_BY_LIMIT','-1'
|
|
'DISABLE_CACHED_READS','0'
|
|
'DISABLE_CODEGEN','0'
|
|
'DISABLE_OUTERMOST_TOPN','0'
|
|
'EXPLAIN_LEVEL','0'
|
|
'HBASE_CACHE_BLOCKS','0'
|
|
'HBASE_CACHING','0'
|
|
'MAX_ERRORS','0'
|
|
'MAX_IO_BUFFERS','0'
|
|
'MAX_SCAN_RANGE_LENGTH','0'
|
|
'MEM_LIMIT','0'
|
|
'NUM_NODES','0'
|
|
'NUM_SCANNER_THREADS','0'
|
|
'COMPRESSION_CODEC','NONE'
|
|
'PARQUET_FILE_SIZE','1610612736'
|
|
'REQUEST_POOL',''
|
|
'RESERVATION_REQUEST_TIMEOUT','0'
|
|
'RM_INITIAL_MEM','0'
|
|
'SYNC_DDL','0'
|
|
'V_CPU_CORES','0'
|
|
---- TYPES
|
|
STRING, STRING
|
|
====
|
|
---- QUERY
|
|
set parquet_file_size='2g'
|
|
---- CATCH
|
|
The PARQUET_FILE_SIZE query option must be less than 2GB.
|
|
====
|
|
---- QUERY
|
|
set foo=bar
|
|
---- CATCH
|
|
Ignoring invalid configuration option: foo
|
|
====
|
|
---- QUERY
|
|
set parquet_compression_codec=bar
|
|
---- CATCH
|
|
Invalid compression codec: bar
|
|
====
|
|
---- QUERY
|
|
# Test that SET actually does change the mem_limit.
|
|
# First, show mem_limit is not hit.
|
|
select 1
|
|
---- RESULTS
|
|
1
|
|
====
|
|
---- QUERY
|
|
# Set mem_limit really small so that queries will fail.
|
|
set mem_limit=1
|
|
====
|
|
---- QUERY
|
|
select 1
|
|
---- CATCH
|
|
Memory limit exceeded
|
|
====
|
|
---- QUERY
|
|
# Set mem_limit back to unlimited and query should succeed again.
|
|
set mem_limit=0
|
|
====
|
|
---- QUERY
|
|
select 1
|
|
---- RESULTS
|
|
1
|
|
====
|