diff --git a/be/src/exec/parquet-common.h b/be/src/exec/parquet-common.h index 9b1b58f93..4fbe0084d 100644 --- a/be/src/exec/parquet-common.h +++ b/be/src/exec/parquet-common.h @@ -270,88 +270,46 @@ inline int ParquetPlainEncoder::Decode( // that the value in the in-memory format has leading zeros or negative 1's. // For example, precision 2 fits in 1 byte. All decimals stored as Decimal4Value // will have 3 bytes of leading zeros, we will only store the interesting byte. -template -inline int ParquetPlainEncoder::EncodeToFixedLenByteArray( - uint8_t* buffer, int fixed_len_size, const T& v) { - DCHECK_GT(fixed_len_size, 0); - DCHECK_LE(fixed_len_size, sizeof(T)); - const int8_t* skipped_bytes_start = NULL; -#if __BYTE_ORDER == __LITTLE_ENDIAN - BitUtil::ByteSwap(buffer, &v, fixed_len_size); - skipped_bytes_start = reinterpret_cast(&v) + fixed_len_size; -#else - memcpy(buffer, &v + sizeof(T) - fixed_len_size, fixed_len_size); - skipped_bytes_start = reinterpret_cast(&v); -#endif - -#ifndef NDEBUG - // On debug, verify that the skipped bytes are what we expect. - for (int i = 0; i < sizeof(T) - fixed_len_size; ++i) { - DCHECK_EQ(skipped_bytes_start[i], v.value() < 0 ? -1 : 0); - } -#endif - return fixed_len_size; -} - -template -inline int ParquetPlainEncoder::DecodeFromFixedLenByteArray( - uint8_t* buffer, int fixed_len_size, T* v) { - DCHECK_GT(fixed_len_size, 0); - DCHECK_LE(fixed_len_size, sizeof(T)); - *v = 0; - // We need to sign extend val. For example, if the original value was - // -1, the original bytes were -1,-1,-1,-1. If we only wrote out 1 byte, after - // the encode step above, val would contain (-1, 0, 0, 0). We need to sign - // extend the remaining 3 bytes to get the original value. - // We do this by filling in the most significant bytes and (arithmetic) bit - // shifting down. - int bytes_to_fill = sizeof(T) - fixed_len_size; -#if __BYTE_ORDER == __LITTLE_ENDIAN - BitUtil::ByteSwap(reinterpret_cast(v) + bytes_to_fill, buffer, fixed_len_size); -#else - memcpy(v, buffer, fixed_len_size); -#endif - *v >>= (bytes_to_fill * 8); - return fixed_len_size; -} - template<> inline int ParquetPlainEncoder::Encode( uint8_t* buffer, int fixed_len_size, const Decimal4Value& v) { - return EncodeToFixedLenByteArray(buffer, fixed_len_size, v); + DecimalUtil::EncodeToFixedLenByteArray(buffer, fixed_len_size, v); + return fixed_len_size; } template<> inline int ParquetPlainEncoder::Encode( uint8_t* buffer, int fixed_len_size, const Decimal8Value& v) { - return EncodeToFixedLenByteArray(buffer, fixed_len_size, v); + DecimalUtil::EncodeToFixedLenByteArray(buffer, fixed_len_size, v); + return fixed_len_size; } template<> inline int ParquetPlainEncoder::Encode( uint8_t* buffer, int fixed_len_size, const Decimal16Value& v) { - return EncodeToFixedLenByteArray(buffer, fixed_len_size, v); + DecimalUtil::EncodeToFixedLenByteArray(buffer, fixed_len_size, v); + return fixed_len_size; } template<> inline int ParquetPlainEncoder::Decode( uint8_t* buffer, int fixed_len_size, Decimal4Value* v) { - return DecodeFromFixedLenByteArray( - buffer, fixed_len_size, reinterpret_cast(v)); + DecimalUtil::DecodeFromFixedLenByteArray(buffer, fixed_len_size, v); + return fixed_len_size; } template<> inline int ParquetPlainEncoder::Decode( uint8_t* buffer, int fixed_len_size, Decimal8Value* v) { - return DecodeFromFixedLenByteArray( - buffer, fixed_len_size, reinterpret_cast(v)); + DecimalUtil::DecodeFromFixedLenByteArray(buffer, fixed_len_size, v); + return fixed_len_size; } template<> inline int ParquetPlainEncoder::Decode( uint8_t* buffer, int fixed_len_size, Decimal16Value* v) { - return DecodeFromFixedLenByteArray( - buffer, fixed_len_size, reinterpret_cast(v)); + DecimalUtil::DecodeFromFixedLenByteArray(buffer, fixed_len_size, v); + return fixed_len_size; } } diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt index 8633fd0c9..ff93ba06f 100644 --- a/be/src/exprs/CMakeLists.txt +++ b/be/src/exprs/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(Exprs conditional-functions.cc date-literal.cc decimal-functions.cc + decimal-literal.cc decimal-operators.cc expr.cc float-literal.cc diff --git a/be/src/exprs/decimal-literal.cc b/be/src/exprs/decimal-literal.cc new file mode 100644 index 000000000..64e64780f --- /dev/null +++ b/be/src/exprs/decimal-literal.cc @@ -0,0 +1,103 @@ +// Copyright 2012 Cloudera Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exprs/decimal-literal.h" + +#include + +#include "gen-cpp/Exprs_types.h" +#include "util/decimal-util.h" + +using namespace llvm; +using namespace std; + +namespace impala { + +DecimalLiteral::DecimalLiteral(const TExprNode& node) + : Expr(node) { + DCHECK(type_.type == TYPE_DECIMAL) << type_; + const uint8_t* buffer = + reinterpret_cast(&node.decimal_literal.value[0]); + int len = node.decimal_literal.value.size(); + DCHECK_GT(len, 0); + + switch (type().GetByteSize()) { + case 4: + DCHECK_LE(len, 4); + DecimalUtil::DecodeFromFixedLenByteArray(buffer, len, &result_.decimal4_val); + break; + case 8: + DCHECK_LE(len, 8); + DecimalUtil::DecodeFromFixedLenByteArray(buffer, len, &result_.decimal8_val); + break; + case 16: + DCHECK_LE(len, 16); + DecimalUtil::DecodeFromFixedLenByteArray(buffer, len, &result_.decimal16_val); + break; + default: + DCHECK(false) << "DecimalLiteral ctor: bad type: " << type_.DebugString(); + } +} + +void* DecimalLiteral::ReturnDecimal4Value(Expr* e, TupleRow* row) { + return &reinterpret_cast(e)->result_.decimal4_val; +} + +void* DecimalLiteral::ReturnDecimal8Value(Expr* e, TupleRow* row) { + return &reinterpret_cast(e)->result_.decimal8_val; +} + +void* DecimalLiteral::ReturnDecimal16Value(Expr* e, TupleRow* row) { + return &reinterpret_cast(e)->result_.decimal16_val; +} + +Status DecimalLiteral::Prepare(RuntimeState* state, const RowDescriptor& row_desc) { + DCHECK_EQ(children_.size(), 0); + switch (type().GetByteSize()) { + case 4: + compute_fn_ = ReturnDecimal4Value; + break; + case 8: + compute_fn_ = ReturnDecimal8Value; + break; + case 16: + compute_fn_ = ReturnDecimal16Value; + break; + default: + DCHECK(false) << "DecimalLiteral::DebugString(): bad type: " << type_.DebugString(); + } + return Status::OK; +} + +string DecimalLiteral::DebugString() const { + stringstream out; + out << "DecimalLiteral(type=" << type_ << " value="; + switch (type().GetByteSize()) { + case 4: + out << result_.decimal4_val.ToString(type()); + break; + case 8: + out << result_.decimal8_val.ToString(type()); + break; + case 16: + out << result_.decimal16_val.ToString(type()); + break; + default: + DCHECK(false) << "DecimalLiteral::DebugString(): bad type: " << type_.DebugString(); + } + out << ")"; + return out.str(); +} + +} diff --git a/be/src/exprs/decimal-literal.h b/be/src/exprs/decimal-literal.h new file mode 100644 index 000000000..7f6edecf3 --- /dev/null +++ b/be/src/exprs/decimal-literal.h @@ -0,0 +1,42 @@ +// Copyright 2012 Cloudera Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#ifndef IMPALA_EXPRS_DECIMAL_LITERAL_H +#define IMPALA_EXPRS_DECIMAL_LITERAL_H + +#include +#include "exprs/expr.h" + +namespace impala { + +class TExprNode; + +class DecimalLiteral: public Expr { + protected: + friend class Expr; + DecimalLiteral(const TExprNode& node); + + virtual Status Prepare(RuntimeState* state, const RowDescriptor& row_desc); + virtual std::string DebugString() const; + + private: + static void* ReturnDecimal4Value(Expr* e, TupleRow* row); + static void* ReturnDecimal8Value(Expr* e, TupleRow* row); + static void* ReturnDecimal16Value(Expr* e, TupleRow* row); +}; + +} + +#endif diff --git a/be/src/exprs/decimal-operators.cc b/be/src/exprs/decimal-operators.cc index 2cebe1ed0..20c177861 100644 --- a/be/src/exprs/decimal-operators.cc +++ b/be/src/exprs/decimal-operators.cc @@ -122,6 +122,7 @@ inline void* DecimalOperators::SetDecimalVal(Expr* e, const ColumnType& val_type return &e->result_.decimal16_val; } default: + DCHECK(false); return NULL; } } @@ -145,6 +146,7 @@ inline void* DecimalOperators::SetDecimalVal(Expr* e, const ColumnType& val_type e->result_.decimal16_val = val.ScaleTo(val_type, e->type()); return &e->result_.decimal16_val; default: + DCHECK(false); return NULL; } } @@ -447,6 +449,53 @@ void* DecimalOperators::Cast_decimal_StringValue(Expr* e, TupleRow* row) { return &e->result_.string_val; } +void* DecimalOperators::Cast_decimal_TimestampValue(Expr* e, TupleRow* row) { + DCHECK_EQ(e->GetNumChildren(), 1); + Expr* c = e->GetChild(0); + void* v = c->GetValue(row); + if (v == NULL) return NULL; + DCHECK_EQ(c->type().type, TYPE_DECIMAL); + switch (c->type().GetByteSize()) { + case 4: + e->result_.timestamp_val = + reinterpret_cast(v)->ToDouble(c->type()); + break; + case 8: + e->result_.timestamp_val = + reinterpret_cast(v)->ToDouble(c->type()); + break; + case 16: + e->result_.timestamp_val = + reinterpret_cast(v)->ToDouble(c->type()); + break; + default: + return NULL; + } + return &e->result_.timestamp_val; +} + +void* DecimalOperators::Cast_decimal_bool(Expr* e, TupleRow* row) { + DCHECK_EQ(e->GetNumChildren(), 1); + Expr* c = e->GetChild(0); + void* v = c->GetValue(row); + if (v == NULL) return NULL; + DCHECK_EQ(c->type().type, TYPE_DECIMAL); + switch (c->type().GetByteSize()) { + case 4: + e->result_.bool_val = reinterpret_cast(v)->value() != 0; + break; + case 8: + e->result_.bool_val = reinterpret_cast(v)->value() != 0; + break; + case 16: + e->result_.bool_val = reinterpret_cast(v)->value() != 0; + break; + default: + return NULL; + } + return &e->result_.bool_val; +} + #define DECIMAL_ARITHMETIC_OP(FN_NAME, OP_FN) \ void* DecimalOperators::FN_NAME(Expr* e, TupleRow* row) {\ DCHECK_EQ(e->GetNumChildren(), 2);\ diff --git a/be/src/exprs/decimal-operators.h b/be/src/exprs/decimal-operators.h index 1a680846c..5bb1c7f25 100644 --- a/be/src/exprs/decimal-operators.h +++ b/be/src/exprs/decimal-operators.h @@ -39,6 +39,7 @@ class DecimalOperators { static void* Cast_double_decimal(Expr* e, TupleRow* row); static void* Cast_StringValue_decimal(Expr* e, TupleRow* row); + static void* Cast_decimal_bool(Expr* e, TupleRow* row); static void* Cast_decimal_char(Expr* e, TupleRow* row); static void* Cast_decimal_short(Expr* e, TupleRow* row); static void* Cast_decimal_int(Expr* e, TupleRow* row); @@ -46,6 +47,7 @@ class DecimalOperators { static void* Cast_decimal_float(Expr* e, TupleRow* row); static void* Cast_decimal_double(Expr* e, TupleRow* row); static void* Cast_decimal_StringValue(Expr* e, TupleRow* row); + static void* Cast_decimal_TimestampValue(Expr* e, TupleRow* row); static void* Add_decimal_decimal(Expr* e, TupleRow* row); static void* Subtract_decimal_decimal(Expr* e, TupleRow* row); diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc index 63d5fa815..c30b47026 100644 --- a/be/src/exprs/expr-test.cc +++ b/be/src/exprs/expr-test.cc @@ -61,6 +61,38 @@ namespace impala { ImpaladQueryExecutor* executor_; bool disable_codegen_; +template +string LiteralToString(VAL_TYPE val) { + return lexical_cast(val); +} + +template<> +string LiteralToString(float val) { + stringstream ss; + ss << "cast(" + << lexical_cast(val) + << " as float)"; + return ss.str(); +} + +template<> +string LiteralToString(double val) { + stringstream ss; + ss << "cast(" + << lexical_cast(val) + << " as float)"; + return ss.str(); +} + +template<> +string LiteralToString(double val) { + stringstream ss; + ss << "cast(" + << lexical_cast(val) + << " as double)"; + return ss.str(); +} + class ExprTest : public testing::Test { protected: // Maps from enum value of primitive integer type to @@ -118,12 +150,16 @@ class ExprTest : public testing::Test { lexical_cast(min_int_values_[TYPE_INT]); default_type_strs_[TYPE_BIGINT] = lexical_cast(min_int_values_[TYPE_BIGINT]); - // Don't use lexical case here because it results + // Don't use lexical cast here because it results // in a string 1.1000000000000001 that messes up the tests. - default_type_strs_[TYPE_FLOAT] = - lexical_cast(min_float_values_[TYPE_FLOAT]); - default_type_strs_[TYPE_DOUBLE] = - lexical_cast(min_float_values_[TYPE_DOUBLE]); + stringstream ss; + ss << "cast(" + << lexical_cast(min_float_values_[TYPE_FLOAT]) << " as float)"; + default_type_strs_[TYPE_FLOAT] = ss.str(); + ss.str(""); + ss << "cast(" + << lexical_cast(min_float_values_[TYPE_FLOAT]) << " as double)"; + default_type_strs_[TYPE_DOUBLE] = ss.str(); default_type_strs_[TYPE_BOOLEAN] = default_bool_str_; default_type_strs_[TYPE_STRING] = default_string_str_; default_type_strs_[TYPE_TIMESTAMP] = default_timestamp_str_; @@ -524,8 +560,8 @@ class ExprTest : public testing::Test { void TestFixedResultTypeOps(LeftOp a, RightOp b, const ColumnType& expected_type) { Result cast_a = static_cast(a); Result cast_b = static_cast(b); - string a_str = lexical_cast(cast_a); - string b_str = lexical_cast(cast_b); + string a_str = LiteralToString(cast_a); + string b_str = LiteralToString(cast_b); TestValue(a_str + " + " + b_str, expected_type, static_cast(cast_a + cast_b)); TestValue(a_str + " - " + b_str, expected_type, @@ -561,7 +597,7 @@ class ExprTest : public testing::Test { template void TestNullOperandFixedResultTypeOps(NonNullOp op, const ColumnType& expected_type) { CastType cast_op = static_cast(op); - string op_str = lexical_cast(cast_op); + string op_str = LiteralToString(cast_op); // NULL as right operand. TestIsNull(op_str + " + NULL", expected_type); TestIsNull(op_str + " - NULL", expected_type); @@ -1970,8 +2006,8 @@ TEST_F(ExprTest, MathConversionFunctions) { TEST_F(ExprTest, MathFunctions) { TestValue("pi()", TYPE_DOUBLE, M_PI); TestValue("e()", TYPE_DOUBLE, M_E); - TestValue("abs(-1.0)", TYPE_DOUBLE, 1.0); - TestValue("abs(1.0)", TYPE_DOUBLE, 1.0); + TestValue("abs(cast(-1.0 as double))", TYPE_DOUBLE, 1.0); + TestValue("abs(cast(1.0 as double))", TYPE_DOUBLE, 1.0); TestValue("sign(0.0)", TYPE_FLOAT, 0.0f); TestValue("sign(-0.0)", TYPE_FLOAT, 0.0f); TestValue("sign(+0.0)", TYPE_FLOAT, 0.0f); @@ -2259,41 +2295,41 @@ TEST_F(ExprTest, MathFunctions) { } TEST_F(ExprTest, MathRoundingFunctions) { - TestValue("ceil(0.1)", TYPE_BIGINT, 1); - TestValue("ceil(-10.05)", TYPE_BIGINT, -10); - TestValue("ceiling(0.1)", TYPE_BIGINT, 1); - TestValue("ceiling(-10.05)", TYPE_BIGINT, -10); - TestValue("floor(0.1)", TYPE_BIGINT, 0); - TestValue("floor(-10.007)", TYPE_BIGINT, -11); + TestValue("ceil(cast(0.1 as double))", TYPE_BIGINT, 1); + TestValue("ceil(cast(-10.05 as double))", TYPE_BIGINT, -10); + TestValue("ceiling(cast(0.1 as double))", TYPE_BIGINT, 1); + TestValue("ceiling(cast(-10.05 as double))", TYPE_BIGINT, -10); + TestValue("floor(cast(0.1 as double))", TYPE_BIGINT, 0); + TestValue("floor(cast(-10.007 as double))", TYPE_BIGINT, -11); - TestValue("round(1.499999)", TYPE_BIGINT, 1); - TestValue("round(1.5)", TYPE_BIGINT, 2); - TestValue("round(1.500001)", TYPE_BIGINT, 2); - TestValue("round(-1.499999)", TYPE_BIGINT, -1); - TestValue("round(-1.5)", TYPE_BIGINT, -2); - TestValue("round(-1.500001)", TYPE_BIGINT, -2); + TestValue("round(cast(1.499999 as double))", TYPE_BIGINT, 1); + TestValue("round(cast(1.5 as double))", TYPE_BIGINT, 2); + TestValue("round(cast(1.500001 as double))", TYPE_BIGINT, 2); + TestValue("round(cast(-1.499999 as double))", TYPE_BIGINT, -1); + TestValue("round(cast(-1.5 as double))", TYPE_BIGINT, -2); + TestValue("round(cast(-1.500001 as double))", TYPE_BIGINT, -2); - TestValue("round(3.14159265, 0)", TYPE_DOUBLE, 3.0); - TestValue("round(3.14159265, 1)", TYPE_DOUBLE, 3.1); - TestValue("round(3.14159265, 2)", TYPE_DOUBLE, 3.14); - TestValue("round(3.14159265, 3)", TYPE_DOUBLE, 3.142); - TestValue("round(3.14159265, 4)", TYPE_DOUBLE, 3.1416); - TestValue("round(3.14159265, 5)", TYPE_DOUBLE, 3.14159); - TestValue("round(-3.14159265, 0)", TYPE_DOUBLE, -3.0); - TestValue("round(-3.14159265, 1)", TYPE_DOUBLE, -3.1); - TestValue("round(-3.14159265, 2)", TYPE_DOUBLE, -3.14); - TestValue("round(-3.14159265, 3)", TYPE_DOUBLE, -3.142); - TestValue("round(-3.14159265, 4)", TYPE_DOUBLE, -3.1416); - TestValue("round(-3.14159265, 5)", TYPE_DOUBLE, -3.14159); + TestValue("round(cast(3.14159265 as double), 0)", TYPE_DOUBLE, 3.0); + TestValue("round(cast(3.14159265 as double), 1)", TYPE_DOUBLE, 3.1); + TestValue("round(cast(3.14159265 as double), 2)", TYPE_DOUBLE, 3.14); + TestValue("round(cast(3.14159265 as double), 3)", TYPE_DOUBLE, 3.142); + TestValue("round(cast(3.14159265 as double), 4)", TYPE_DOUBLE, 3.1416); + TestValue("round(cast(3.14159265 as double), 5)", TYPE_DOUBLE, 3.14159); + TestValue("round(cast(-3.14159265 as double), 0)", TYPE_DOUBLE, -3.0); + TestValue("round(cast(-3.14159265 as double), 1)", TYPE_DOUBLE, -3.1); + TestValue("round(cast(-3.14159265 as double), 2)", TYPE_DOUBLE, -3.14); + TestValue("round(cast(-3.14159265 as double), 3)", TYPE_DOUBLE, -3.142); + TestValue("round(cast(-3.14159265 as double), 4)", TYPE_DOUBLE, -3.1416); + TestValue("round(cast(-3.14159265 as double), 5)", TYPE_DOUBLE, -3.14159); // NULL arguments. - TestIsNull("ceil(NULL)", TYPE_BIGINT); - TestIsNull("ceiling(NULL)", TYPE_BIGINT); - TestIsNull("floor(NULL)", TYPE_BIGINT); - TestIsNull("round(NULL)", TYPE_BIGINT); - TestIsNull("round(NULL, 1)", TYPE_DOUBLE); - TestIsNull("round(3.14159265, NULL)", TYPE_DOUBLE); - TestIsNull("round(NULL, NULL)", TYPE_DOUBLE); + TestIsNull("ceil(cast(NULL as double))", TYPE_BIGINT); + TestIsNull("ceiling(cast(NULL as double))", TYPE_BIGINT); + TestIsNull("floor(cast(NULL as double))", TYPE_BIGINT); + TestIsNull("round(cast(NULL as double))", TYPE_BIGINT); + TestIsNull("round(cast(NULL as double), 1)", TYPE_DOUBLE); + TestIsNull("round(cast(3.14159265 as double), NULL)", TYPE_DOUBLE); + TestIsNull("round(cast(NULL as double), NULL)", TYPE_DOUBLE); } TEST_F(ExprTest, UnaryOperators) { @@ -2303,9 +2339,9 @@ TEST_F(ExprTest, UnaryOperators) { TestValue("+-1", TYPE_TINYINT, -1); TestValue("++1", TYPE_TINYINT, 1); - TestValue("+1.f", TYPE_FLOAT, 1.0f); - TestValue("+1.0", TYPE_FLOAT, 1.0f); - TestValue("-1.0", TYPE_FLOAT, -1.0f); + TestValue("+cast(1. as float)", TYPE_FLOAT, 1.0f); + TestValue("+cast(1.0 as float)", TYPE_FLOAT, 1.0f); + TestValue("-cast(1.0 as float)", TYPE_DOUBLE, -1.0); TestValue("1 - - - 1", TYPE_SMALLINT, 0); } @@ -2906,8 +2942,8 @@ TEST_F(ExprTest, ConditionalFunctions) { TestValue("if(FALSE, FALSE, TRUE)", TYPE_BOOLEAN, true); TestValue("if(TRUE, 10, 20)", TYPE_TINYINT, 10); TestValue("if(FALSE, 10, 20)", TYPE_TINYINT, 20); - TestValue("if(TRUE, 5.5, 8.8)", TYPE_DOUBLE, 5.5); - TestValue("if(FALSE, 5.5, 8.8)", TYPE_DOUBLE, 8.8); + TestValue("if(TRUE, cast(5.5 as double), cast(8.8 as double))", TYPE_DOUBLE, 5.5); + TestValue("if(FALSE, cast(5.5 as double), cast(8.8 as double))", TYPE_DOUBLE, 8.8); TestStringValue("if(TRUE, 'abc', 'defgh')", "abc"); TestStringValue("if(FALSE, 'abc', 'defgh')", "defgh"); TimestampValue then_val(1293872461); @@ -2927,10 +2963,10 @@ TEST_F(ExprTest, ConditionalFunctions) { TestValue("nullif(10, NULL)", TYPE_TINYINT, 10); TestIsNull("nullif(10, 10)", TYPE_TINYINT); TestValue("nullif(10, 20)", TYPE_TINYINT, 10); - TestIsNull("nullif(10.10, 10.10)", TYPE_DOUBLE); - TestValue("nullif(10.10, 20.20)", TYPE_DOUBLE, 10.10); - TestIsNull("nullif(NULL, 10.10)", TYPE_DOUBLE); - TestValue("nullif(10.10, NULL)", TYPE_DOUBLE, 10.10); + TestIsNull("nullif(cast(10.10 as double), cast(10.10 as double))", TYPE_DOUBLE); + TestValue("nullif(cast(10.10 as double), cast(20.20 as double))", TYPE_DOUBLE, 10.10); + TestIsNull("nullif(cast(NULL as double), 10.10)", TYPE_DOUBLE); + TestValue("nullif(cast(10.10 as double), NULL)", TYPE_DOUBLE, 10.10); TestIsNull("nullif('abc', 'abc')", TYPE_STRING); TestStringValue("nullif('abc', 'def')", "abc"); TestIsNull("nullif(NULL, 'abc')", TYPE_STRING); @@ -3269,11 +3305,13 @@ TEST_F(ExprTest, DecimalFunctions) { TestValue("precision(1)", TYPE_INT, 3); TestValue("precision(cast(1 as smallint))", TYPE_INT, 5); - TestValue("precision(cast(123 as bigint))", TYPE_INT, 20); + TestValue("precision(cast(123 as bigint))", TYPE_INT, 19); TestValue("precision(cast(123 as float))", TYPE_INT, 38); TestValue("scale(cast(123 as float))", TYPE_INT, 9); - TestValue("precision(123.45)", TYPE_INT, 38); - TestValue("scale(123.45)", TYPE_INT, 17); + TestValue("precision(cast(123.45 as double))", TYPE_INT, 38); + TestValue("scale(cast(123.45 as double))", TYPE_INT, 17); + TestValue("precision(123.45)", TYPE_INT, 5); + TestValue("scale(123.45)", TYPE_INT, 2); TestValue("precision(1 + 1)", TYPE_INT, 5); TestValue("scale(1 + 1)", TYPE_INT, 0); TestValue("precision(1 + 1)", TYPE_INT, 5); diff --git a/be/src/exprs/expr.cc b/be/src/exprs/expr.cc index 9df3a387e..773c1b632 100644 --- a/be/src/exprs/expr.cc +++ b/be/src/exprs/expr.cc @@ -32,6 +32,7 @@ #include "exprs/conditional-functions.h" #include "exprs/date-literal.h" #include "exprs/decimal-functions.h" +#include "exprs/decimal-literal.h" #include "exprs/decimal-operators.h" #include "exprs/float-literal.h" #include "exprs/function-call.h" @@ -295,6 +296,12 @@ Status Expr::CreateExpr(ObjectPool* pool, const TExprNode& texpr_node, Expr** ex } *expr = pool->Add(new DateLiteral(texpr_node)); return Status::OK; + case TExprNodeType::DECIMAL_LITERAL: + if (!texpr_node.__isset.decimal_literal) { + return Status("Decimal literal not set in thrift node"); + } + *expr = pool->Add(new DecimalLiteral(texpr_node)); + return Status::OK; case TExprNodeType::FLOAT_LITERAL: if (!texpr_node.__isset.float_literal) { return Status("Float literal not set in thrift node"); @@ -459,6 +466,25 @@ void Expr::GetValue(TupleRow* row, bool as_ascii, TColumnValue* col_val) { case TYPE_DOUBLE: col_val->__set_double_val(*reinterpret_cast(value)); break; + case TYPE_DECIMAL: + switch (type_.GetByteSize()) { + case 4: + col_val->string_val = + reinterpret_cast(value)->ToString(type_); + break; + case 8: + col_val->string_val = + reinterpret_cast(value)->ToString(type_); + break; + case 16: + col_val->string_val = + reinterpret_cast(value)->ToString(type_); + break; + default: + DCHECK(false) << "Bad Type: " << type_; + } + col_val->__isset.string_val = true; + break; case TYPE_STRING: string_val = reinterpret_cast(value); tmp.assign(static_cast(string_val->ptr), string_val->len); diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h index c85135d99..246b57008 100644 --- a/be/src/exprs/expr.h +++ b/be/src/exprs/expr.h @@ -434,6 +434,7 @@ class Expr { friend class CastExpr; friend class ComputeFunctions; friend class DecimalFunctions; + friend class DecimalLliteral; friend class DecimalOperators; friend class MathFunctions; friend class StringFunctions; diff --git a/be/src/util/decimal-util.h b/be/src/util/decimal-util.h index c566064ba..888ab654e 100644 --- a/be/src/util/decimal-util.h +++ b/be/src/util/decimal-util.h @@ -22,6 +22,7 @@ #include "runtime/types.h" #include "runtime/multi-precision.h" +#include "util/bit-util.h" namespace impala { @@ -55,6 +56,50 @@ class DecimalUtil { } return result; } + + // Write decimals as big endian (byte comparable) in fixed_len_size bytes. + template + static inline void EncodeToFixedLenByteArray( + uint8_t* buffer, int fixed_len_size, const T& v) { + DCHECK_GT(fixed_len_size, 0); + DCHECK_LE(fixed_len_size, sizeof(T)); + const int8_t* skipped_bytes_start = NULL; +#if __BYTE_ORDER == __LITTLE_ENDIAN + BitUtil::ByteSwap(buffer, &v, fixed_len_size); + skipped_bytes_start = reinterpret_cast(&v) + fixed_len_size; +#else + memcpy(buffer, &v + sizeof(T) - fixed_len_size, fixed_len_size); + skipped_bytes_start = reinterpret_cast(&v); +#endif + +#ifndef NDEBUG + // On debug, verify that the skipped bytes are what we expect. + for (int i = 0; i < sizeof(T) - fixed_len_size; ++i) { + DCHECK_EQ(skipped_bytes_start[i], v.value() < 0 ? -1 : 0); + } +#endif + } + + template + static inline void DecodeFromFixedLenByteArray( + const uint8_t* buffer, int fixed_len_size, T* v) { + DCHECK_GT(fixed_len_size, 0); + DCHECK_LE(fixed_len_size, sizeof(T)); + *v = 0; + // We need to sign extend val. For example, if the original value was + // -1, the original bytes were -1,-1,-1,-1. If we only wrote out 1 byte, after + // the encode step above, val would contain (-1, 0, 0, 0). We need to sign + // extend the remaining 3 bytes to get the original value. + // We do this by filling in the most significant bytes and (arithmetic) bit + // shifting down. + int bytes_to_fill = sizeof(T) - fixed_len_size; +#if __BYTE_ORDER == __LITTLE_ENDIAN + BitUtil::ByteSwap(reinterpret_cast(v) + bytes_to_fill, buffer, fixed_len_size); +#else + memcpy(v, buffer, fixed_len_size); +#endif + v->value() >>= (bytes_to_fill * 8); + } }; } diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py index 90f99c4cf..d2b9409b9 100755 --- a/common/function-registry/impala_functions.py +++ b/common/function-registry/impala_functions.py @@ -471,9 +471,19 @@ functions = [ [['ceil', 'ceiling'], 'DECIMAL', ['DECIMAL'], symbol('DecimalFunctions', 'Ceil')], [['floor'], 'DECIMAL', ['DECIMAL'], symbol('DecimalFunctions', 'Floor')], [['round'], 'DECIMAL', ['DECIMAL'], symbol('DecimalFunctions', 'Round')], + [['round'], 'DECIMAL', ['DECIMAL', 'TINYINT'], symbol('DecimalFunctions', 'RoundTo')], + [['round'], 'DECIMAL', ['DECIMAL', 'SMALLINT'], symbol('DecimalFunctions', 'RoundTo')], [['round'], 'DECIMAL', ['DECIMAL', 'INT'], symbol('DecimalFunctions', 'RoundTo')], + [['round'], 'DECIMAL', ['DECIMAL', 'BIGINT'], symbol('DecimalFunctions', 'RoundTo')], [['truncate'], 'DECIMAL', ['DECIMAL'], symbol('DecimalFunctions', 'Truncate')], - [['truncate'], 'DECIMAL', ['DECIMAL', 'INT'], symbol('DecimalFunctions', 'TruncateTo')], + [['truncate'], 'DECIMAL', ['DECIMAL', 'TINYINT'], + symbol('DecimalFunctions', 'TruncateTo')], + [['truncate'], 'DECIMAL', ['DECIMAL', 'SMALLINT'], + symbol('DecimalFunctions', 'TruncateTo')], + [['truncate'], 'DECIMAL', ['DECIMAL', 'INT'], + symbol('DecimalFunctions', 'TruncateTo')], + [['truncate'], 'DECIMAL', ['DECIMAL', 'BIGINT'], + symbol('DecimalFunctions', 'TruncateTo')], ] # These functions are implemented against the UDF interface. diff --git a/common/thrift/Exprs.thrift b/common/thrift/Exprs.thrift index 1c9526b9d..0148089ef 100644 --- a/common/thrift/Exprs.thrift +++ b/common/thrift/Exprs.thrift @@ -37,9 +37,9 @@ enum TExprNodeType { TUPLE_IS_NULL_PRED, FUNCTION_CALL, AGGREGATE_EXPR, - // TODO: old style compute functions. this will be deprecated COMPUTE_FUNCTION_CALL, + DECIMAL_LITERAL, } struct TBoolLiteral { @@ -55,6 +55,12 @@ struct TDateLiteral { 1: required Types.TTimestamp value } +struct TDecimalLiteral { + // Value of the unscaled decimal in two's complement big endian + // i.e. BigInteger.getBytes() + 1: required binary value +} + struct TFloatLiteral { 1: required double value } @@ -115,6 +121,7 @@ struct TExprNode { 15: optional TSlotRef slot_ref 16: optional TStringLiteral string_literal 17: optional TTupleIsNullPredicate tuple_is_null_pred + 18: optional TDecimalLiteral decimal_literal } // A flattened representation of a tree of Expr nodes, obtained by depth-first diff --git a/fe/src/main/cup/sql-parser.y b/fe/src/main/cup/sql-parser.y index 171443bfe..b033fbb1e 100644 --- a/fe/src/main/cup/sql-parser.y +++ b/fe/src/main/cup/sql-parser.y @@ -22,6 +22,7 @@ import com.cloudera.impala.analysis.UnionStmt.Qualifier; import com.cloudera.impala.thrift.TDescribeTableOutputStyle; import com.cloudera.impala.thrift.THdfsFileFormat; import com.cloudera.impala.thrift.TTablePropertyType; +import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; @@ -228,7 +229,7 @@ terminal EQUAL, NOT, LESSTHAN, GREATERTHAN; terminal String IDENT; terminal String NUMERIC_OVERFLOW; terminal BigInteger INTEGER_LITERAL; -terminal Double FLOATINGPOINT_LITERAL; +terminal BigDecimal DECIMAL_LITERAL; terminal String STRING_LITERAL; terminal String UNMATCHED_STRING_LITERAL; @@ -1698,7 +1699,7 @@ sign_chain_expr ::= // integer literals require analysis to set their type, so the instance check below // is not equivalent to e.getType().isNumericType() if (e.isLiteral() && - (e instanceof IntLiteral || e instanceof FloatLiteral)) { + (e instanceof IntLiteral || e instanceof DecimalLiteral)) { ((LiteralExpr)e).swapSign(); RESULT = e; } else { @@ -1811,8 +1812,8 @@ timestamp_arithmetic_expr ::= literal ::= INTEGER_LITERAL:l {: RESULT = new IntLiteral(l); :} - | FLOATINGPOINT_LITERAL:l - {: RESULT = new FloatLiteral(l); :} + | DECIMAL_LITERAL:l + {: RESULT = new DecimalLiteral(l); :} | STRING_LITERAL:l {: RESULT = new StringLiteral(l); :} | KW_TRUE diff --git a/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java index 33a6e48f1..6f1605e21 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java @@ -155,27 +155,27 @@ public class ArithmeticExpr extends Expr { } } + ColumnType t0 = getChild(0).getType(); // bitnot is the only unary op, deal with it here if (op_ == Operator.BITNOT) { - type_ = getChild(0).getType(); - if (type_.isNull()) { - // Special case ~NULL to resolve to TYPE_INT. - type_ = ColumnType.INT; - castChild(0, type_); - } - fn_ = getBuiltinFunction(analyzer, op_.getName(), collectChildReturnTypes(), - CompareMode.IS_SUPERTYPE_OF); - if (fn_ == null) { - throw new AnalysisException("Bitwise operations only allowed on fixed-point " + + // Special case ~NULL to resolve to TYPE_INT. + if (!t0.isNull() && !t0.isIntegerType()) { + throw new AnalysisException("Bitwise operations only allowed on integer " + "types: " + toSql()); } - Preconditions.checkState(type_.equals(fn_.getReturnType()) || type_.isNull()); + if (t0.isNull()) castChild(0, ColumnType.INT); + fn_ = getBuiltinFunction(analyzer, op_.getName(), collectChildReturnTypes(), + CompareMode.IS_SUPERTYPE_OF); + Preconditions.checkNotNull(fn_); + castForFunctionCall(); + type_ = fn_.getReturnType(); return; } Preconditions.checkState(children_.size() == 2); // only bitnot is unary - ColumnType t1 = getChild(0).getType(); - ColumnType t2 = getChild(1).getType(); + convertNumericLiteralsFromDecimal(analyzer); + t0 = getChild(0).getType(); + ColumnType t1 = getChild(1).getType(); String fnName = op_.getName(); switch (op_) { @@ -184,19 +184,19 @@ public class ArithmeticExpr extends Expr { case DIVIDE: case MULTIPLY: case MOD: - type_ = TypesUtil.getArithmeticResultType(t1, t2, op_); + type_ = TypesUtil.getArithmeticResultType(t0, t1, op_); break; case INT_DIVIDE: case BITAND: case BITOR: case BITXOR: - if ((!t1.isNull() & !t1.isIntegerType()) || - (!t2.isNull() && !t2.isIntegerType())) { + if ((!t0.isNull() & !t0.isIntegerType()) || + (!t1.isNull() && !t1.isIntegerType())) { throw new AnalysisException("Invalid non-integer argument to operation '" + op_.toString() + "': " + this.toSql()); } - type_ = ColumnType.getAssignmentCompatibleType(t1, t2); + type_ = ColumnType.getAssignmentCompatibleType(t0, t1); // the result is always an integer or null Preconditions.checkState(type_.isIntegerType() || type_.isNull()); break; @@ -211,7 +211,7 @@ public class ArithmeticExpr extends Expr { // Use MATH_MOD function operator for floating-point modulo. // TODO remove this when we have operators implemented using the UDF interface // and we can resolve this just using function overloading. - if ((t1.isFloatingPointType() || t2.isFloatingPointType()) && + if ((t0.isFloatingPointType() || t1.isFloatingPointType()) && op_ == ArithmeticExpr.Operator.MOD) { fnName = "fmod"; } @@ -231,7 +231,7 @@ public class ArithmeticExpr extends Expr { CompareMode.IS_SUPERTYPE_OF); if (fn_ == null) { Preconditions.checkState(false, String.format("No match " + - "for '%s' with operand types %s and %s", toSql(), t1, t2)); + "for '%s' with operand types %s and %s", toSql(), t0, t1)); } } } diff --git a/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java index 4835aa9a4..69fd2b788 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java @@ -125,6 +125,7 @@ public class BinaryPredicate extends Predicate { if (isAnalyzed_) return; super.analyze(analyzer); + convertNumericLiteralsFromDecimal(analyzer); fn_ = getBuiltinFunction(analyzer, op_.getName(), collectChildReturnTypes(), CompareMode.IS_SUPERTYPE_OF); if (fn_ == null) { diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java index d3aed36b8..d5906416a 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java @@ -79,9 +79,7 @@ public class CastExpr extends Expr { // For some reason we don't allow string->bool. // TODO: revisit if (t1.isStringType() && t2.isBoolean()) continue; - - // Disable casting from decimal to boolean and to timestamp - if (t1.isDecimal() && (t2.isBoolean() || t2.isDateType())) continue; + // Disable casting from boolean/timestamp to decimal if ((t1.isBoolean() || t1.isDateType()) && t2.isDecimal()) continue; db.addBuiltin(ScalarFunction.createBuiltinOperator( CAST_FN_NAME, Lists.newArrayList(t1, t2), t2)); @@ -130,6 +128,15 @@ public class CastExpr extends Expr { private void analyze() throws AnalysisException { targetType_.analyze(); + + if (children_.get(0) instanceof DecimalLiteral && + targetType_.isFloatingPointType()) { + // Special case casting a decimal literal to a floating point number. The + // decimal literal can be interpreted as either and we want to avoid casts + // since that can result in loss of accuracy. + ((DecimalLiteral)children_.get(0)).explicitlyCastToFloat(targetType_); + } + // Our cast fn currently takes two arguments. The first is the value to cast and the // second is a dummy of the type to cast to. We need this to be able to resolve the // proper function. @@ -156,7 +163,8 @@ public class CastExpr extends Expr { throw new AnalysisException("Invalid type cast of " + getChild(0).toSql() + " from " + args[0] + " to " + args[1]); } - Preconditions.checkState(targetType_.matchesType(fn_.getReturnType())); + Preconditions.checkState(targetType_.matchesType(fn_.getReturnType()), + targetType_ + " != " + fn_.getReturnType()); type_ = targetType_; } diff --git a/fe/src/main/java/com/cloudera/impala/analysis/DecimalLiteral.java b/fe/src/main/java/com/cloudera/impala/analysis/DecimalLiteral.java new file mode 100644 index 000000000..51daabbbf --- /dev/null +++ b/fe/src/main/java/com/cloudera/impala/analysis/DecimalLiteral.java @@ -0,0 +1,197 @@ +// Copyright 2012 Cloudera Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.cloudera.impala.analysis; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import com.cloudera.impala.catalog.AuthorizationException; +import com.cloudera.impala.catalog.ColumnType; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.common.NotImplementedException; +import com.cloudera.impala.thrift.TDecimalLiteral; +import com.cloudera.impala.thrift.TExprNode; +import com.cloudera.impala.thrift.TExprNodeType; +import com.cloudera.impala.thrift.TFloatLiteral; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +/** + * Literal for all decimal values. Things that contain a decimal point are + * parsed to this class. (e.g. "1.0") + */ +public class DecimalLiteral extends LiteralExpr { + private BigDecimal value_; + + // If true, this literal has been expicitly cast to a type and should not + // be analyzed (which infers the type from value_). + private boolean expliciltyCast_; + + public DecimalLiteral(BigDecimal value) { + init(value); + } + + public DecimalLiteral(BigInteger value) { + init(new BigDecimal(value)); + } + + public DecimalLiteral(String value, ColumnType t) + throws AnalysisException, AuthorizationException { + BigDecimal val = null; + try { + val = new BigDecimal(value); + } catch (NumberFormatException e) { + throw new AnalysisException("invalid decimal literal: " + value, e); + } + init(val); + this.analyze(null); + if (t.isFloatingPointType()) explicitlyCastToFloat(t); + } + + /** + * The versions of the ctor that take types assume the type is correct + * and the DecimalLiteral is created as analyzed with that type. + */ + public DecimalLiteral(BigInteger value, ColumnType type) { + isAnalyzed_ = true; + value_ = new BigDecimal(value); + type_ = type; + } + + public DecimalLiteral(BigDecimal value, ColumnType type) { + isAnalyzed_ = true; + value_ = value; + type_ = type; + } + + + @Override + public String debugString() { + return Objects.toStringHelper(this) + .add("value", value_) + .add("type", type_) + .toString(); + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) return false; + return ((DecimalLiteral) obj).value_.equals(value_); + } + + @Override + public String toSqlImpl() { return getStringValue(); } + @Override + public String getStringValue() { return value_.toString(); } + public double getDoubleValue() { return value_.doubleValue(); } + public long getLongValue() { return value_.longValue(); } + + @Override + protected void toThrift(TExprNode msg) { + switch (type_.getPrimitiveType()) { + case FLOAT: + case DOUBLE: + msg.node_type = TExprNodeType.FLOAT_LITERAL; + msg.float_literal = new TFloatLiteral(value_.doubleValue()); + break; + case DECIMAL: + msg.node_type = TExprNodeType.DECIMAL_LITERAL; + TDecimalLiteral literal = new TDecimalLiteral(); + literal.setValue(value_.unscaledValue().toByteArray()); + msg.decimal_literal = literal; + break; + default: + Preconditions.checkState(false); + } + } + + public BigDecimal getValue() { return value_; } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, + AuthorizationException { + if (isAnalyzed_) return; + super.analyze(analyzer); + if (!expliciltyCast_) { + // Compute the precision and scale from the BigDecimal. + type_ = TypesUtil.computeDecimalType(value_); + if (type_ == null) { + Double d = new Double(value_.doubleValue()); + if (d.isInfinite()) { + throw new AnalysisException("Decimal literal '" + toSql() + + "' exceeds maximum range of doubles."); + } else if (d.doubleValue() == 0 && value_ != BigDecimal.ZERO) { + throw new AnalysisException("Decimal literal '" + toSql() + + "' underflows minimum resolution of doubles."); + } + + // Literal could not be stored in any of the supported decimal precisions and + // scale. Store it as a float/double instead. + float fvalue; + fvalue = value_.floatValue(); + if (fvalue == value_.doubleValue()) { + type_ = ColumnType.FLOAT; + } else { + type_ = ColumnType.DOUBLE; + } + } + } + type_.analyze(); + isAnalyzed_ = true; + } + + /** + * Explicitly cast this literal to 'targetType'. The targetType must be a + * float point type. + */ + protected void explicitlyCastToFloat(ColumnType targetType) { + Preconditions.checkState(targetType.isFloatingPointType()); + type_ = targetType; + expliciltyCast_ = true; + } + + @Override + protected Expr uncheckedCastTo(ColumnType targetType) throws AnalysisException { + Preconditions.checkState(targetType.isNumericType()); + if (targetType.isFloatingPointType() || targetType.isDecimal()) { + type_ = targetType; + } else if (targetType.isIntegerType()) { + Preconditions.checkState(type_.isDecimal()); + Preconditions.checkState(type_.decimalScale() == 0); + type_ = targetType; + } else { + return new CastExpr(targetType, this, true); + } + return this; + } + + @Override + public void swapSign() throws NotImplementedException { + // swapping sign does not change the type + value_ = value_.negate(); + } + + @Override + public int compareTo(LiteralExpr o) { + if (!(o instanceof DecimalLiteral)) return -1; + DecimalLiteral other = (DecimalLiteral) o; + return value_.compareTo(other.value_); + } + + private void init(BigDecimal value) { + isAnalyzed_ = false; + value_ = value; + } +} diff --git a/fe/src/main/java/com/cloudera/impala/analysis/Expr.java b/fe/src/main/java/com/cloudera/impala/analysis/Expr.java index 430cf4680..899b3893d 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/Expr.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/Expr.java @@ -269,6 +269,72 @@ abstract public class Expr extends TreeNode implements ParseNode, Cloneabl return result; } + /** + * Returns true if e is a CastExpr and the target type is a decimal. + */ + private boolean isExplicitCastToDecimal(Expr e) { + if (!(e instanceof CastExpr)) return false; + CastExpr c = (CastExpr)e; + return !c.isImplicit() && c.getType().isDecimal(); + } + + /** + * Converts DecimalLiterals in the tree rooted at child to targetType and + * reanalyzes that subtree. + */ + private void convertNumericLiteralsToFloat(Analyzer analyzer, Expr child, + ColumnType targetType) throws AnalysisException, AuthorizationException { + if (!targetType.isFloatingPointType() && !targetType.isIntegerType()) return; + if (targetType.isIntegerType()) targetType = ColumnType.DOUBLE; + List literals = Lists.newArrayList(); + child.collectAll(Predicates.instanceOf(DecimalLiteral.class), literals); + for (DecimalLiteral l: literals) { + l.explicitlyCastToFloat(targetType); + } + child.reanalyze(analyzer); + } + + /** + * Converts numeric literal in the expr tree rooted at this expr to return floating + * point types instead of decimals, if possible. + * + * Decimal has a higher processing cost than floating point and we should not pay + * the cost if the user does not require the accuracy. For example: + * "select float_col + 1.1" would start out with 1.1 as a decimal(2,1) and the + * float_col would be promoted to a high accuracy decimal. This function will identify + * this case and treat 1.1 as a float. + * In the case of "decimal_col + 1.1", 1.1 would remain a decimal. + * In the case of "float_col + cast(1.1 as decimal(2,1))", the result would be a + * decimal. + * + * Another way to think about it is that DecimalLiterals are analyzed as returning + * decimals (of the narrowest precision/scale) and we later convert them to a floating + * point type when it is consistent with the user's intent. + * + * TODO: another option is to do constant folding in the FE and then apply this rule. + */ + protected void convertNumericLiteralsFromDecimal(Analyzer analyzer) + throws AnalysisException, AuthorizationException { + Preconditions.checkState(this instanceof ArithmeticExpr || + this instanceof BinaryPredicate); + Preconditions.checkState(children_.size() == 2); + ColumnType t0 = getChild(0).getType(); + ColumnType t1 = getChild(1).getType(); + boolean c0IsConstantDecimal = getChild(0).isConstant() && t0.isDecimal(); + boolean c1IsConstantDecimal = getChild(1).isConstant() && t1.isDecimal(); + if (c0IsConstantDecimal && c1IsConstantDecimal) return; + if (!c0IsConstantDecimal && !c1IsConstantDecimal) return; + + // Only child(0) or child(1) is a const decimal. See if we can cast it to + // the type of the other child. + if (c0IsConstantDecimal && !isExplicitCastToDecimal(getChild(0))) { + convertNumericLiteralsToFloat(analyzer, getChild(0), t1); + } + if (c1IsConstantDecimal && !isExplicitCastToDecimal(getChild(1))) { + convertNumericLiteralsToFloat(analyzer, getChild(1), t0); + } + } + /** * Helper function: analyze list of exprs */ diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FloatLiteral.java b/fe/src/main/java/com/cloudera/impala/analysis/FloatLiteral.java deleted file mode 100644 index 43a9b45c8..000000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/FloatLiteral.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2012 Cloudera Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.catalog.ColumnType; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.NotImplementedException; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.cloudera.impala.thrift.TFloatLiteral; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; - -public class FloatLiteral extends LiteralExpr { - private double value_; - - private void init(Double value) { - this.value_ = value.doubleValue(); - // Figure out if this will fit in a FLOAT without loosing precision. - float fvalue; - fvalue = value.floatValue(); - if (fvalue == value.doubleValue()) { - type_ = ColumnType.FLOAT; - } else { - type_ = ColumnType.DOUBLE; - } - } - - public FloatLiteral(Double value) { - init(value); - } - - /** - * C'tor forcing type, e.g., due to implicit cast - */ - public FloatLiteral(Double value, ColumnType type) { - this.value_ = value.doubleValue(); - this.type_ = type; - } - - public FloatLiteral(String value) throws AnalysisException { - Double floatValue = null; - try { - floatValue = new Double(value); - } catch (NumberFormatException e) { - throw new AnalysisException("invalid floating-point literal: " + value, e); - } - init(floatValue); - } - - @Override - public String debugString() { - return Objects.toStringHelper(this) - .add("value", value_) - .toString(); - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) return false; - return ((FloatLiteral) obj).value_ == value_; - } - - @Override - public String toSqlImpl() { - return getStringValue(); - } - - @Override - public String getStringValue() { - return Double.toString(value_); - } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.FLOAT_LITERAL; - msg.float_literal = new TFloatLiteral(value_); - } - - public double getValue() { - return value_; - } - - @Override - protected Expr uncheckedCastTo(ColumnType targetType) throws AnalysisException { - Preconditions.checkState(targetType.isFloatingPointType() || targetType.isDecimal()); - if (targetType.isFloatingPointType()) { - type_ = targetType; - } else { - return new CastExpr(targetType, this, true); - } - return this; - } - - @Override - public void swapSign() throws NotImplementedException { - // swapping sign does not change the type - value_ = -value_; - } - - @Override - public int compareTo(LiteralExpr o) { - if (!(o instanceof FloatLiteral)) return -1; - FloatLiteral other = (FloatLiteral) o; - if (value_ > other.getValue()) return 1; - if (value_ < other.getValue()) return -1; - return 0; - } -} diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java index 99645d2b3..bf1321869 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java @@ -23,7 +23,6 @@ import com.cloudera.impala.catalog.Catalog; import com.cloudera.impala.catalog.ColumnType; import com.cloudera.impala.catalog.Db; import com.cloudera.impala.catalog.Function; -import com.cloudera.impala.catalog.PrimitiveType; import com.cloudera.impala.catalog.ScalarFunction; import com.cloudera.impala.common.AnalysisException; import com.cloudera.impala.common.TreeNode; @@ -178,7 +177,7 @@ public class FunctionCallExpr extends Expr { * We should add a prepare_fn() to UDFs for doing this. */ private ColumnType resolveDecimalReturnType(Analyzer analyzer) - throws AnalysisException { + throws AnalysisException, AuthorizationException { Preconditions.checkState(type_.isWildcardDecimal()); Preconditions.checkState(fn_.getBinaryType() == TFunctionBinaryType.BUILTIN); Preconditions.checkState(children_.size() > 0); @@ -206,8 +205,6 @@ public class FunctionCallExpr extends Expr { "() cannot be called with a NULL second argument."); } - Preconditions.checkState( - children_.get(1).type_.getPrimitiveType() == PrimitiveType.INT); if (!children_.get(1).isConstant()) { // We don't allow calling truncate or round with a non-constant second // (desired scale) argument. e.g. select round(col1, col2). This would @@ -217,14 +214,16 @@ public class FunctionCallExpr extends Expr { throw new AnalysisException(fnName_.getFunction() + "() must be called with a constant second argument."); } - IntLiteral scaleLiteral = - (IntLiteral)LiteralExpr.create(children_.get(1), analyzer.getQueryContext()); + IntLiteral scaleLiteral = (IntLiteral)LiteralExpr.create( + children_.get(1), analyzer.getQueryContext()); resultScale = (int)scaleLiteral.getValue(); if (Math.abs(resultScale) > ColumnType.MAX_SCALE) { throw new AnalysisException("Cannot round/truncate to scales greater than " + ColumnType.MAX_SCALE + "."); } + children_.set(1, scaleLiteral.uncheckedCastTo(ColumnType.INT)); } + if (resultScale < 0) { // Round/Truncate to a negative scale means to round to the digit before // the decimal e.g. round(1234.56, -2) would be 1200. diff --git a/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java index acc7e832f..5e98a4924 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java @@ -639,6 +639,8 @@ public class InsertStmt extends StatementBase { return strBuilder.toString(); } + @Override public void setIsExplain(boolean isExplain) { isExplain_ = isExplain; } + @Override public boolean isExplain() { return isExplain_; } } diff --git a/fe/src/main/java/com/cloudera/impala/analysis/IntLiteral.java b/fe/src/main/java/com/cloudera/impala/analysis/IntLiteral.java index 648b46e5a..6034e30ac 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/IntLiteral.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/IntLiteral.java @@ -114,10 +114,14 @@ public class IntLiteral extends LiteralExpr { if (targetType.isIntegerType()) { this.type_ = targetType; return this; - } else if (targetType.isFloatingPointType()) { - return new FloatLiteral(new Double(value_.longValue()), targetType); - } else if (targetType.isDecimal()) { - return new CastExpr(targetType, this, true); + } else if (targetType.isFloatingPointType() || targetType.isDecimal()) { + BigInteger val = value_; + if (targetType.isDecimal()) { + for (int i = 0; i < targetType.decimalScale(); ++i) { + val = val.multiply(BigInteger.TEN); + } + } + return new DecimalLiteral(val, targetType); } Preconditions.checkState(false, "Unhandled case"); return this; diff --git a/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java index 49d964758..c1ebc907c 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java @@ -14,6 +14,7 @@ package com.cloudera.impala.analysis; +import java.math.BigDecimal; import java.math.BigInteger; import com.cloudera.impala.catalog.AuthorizationException; @@ -23,6 +24,7 @@ import com.cloudera.impala.common.InternalException; import com.cloudera.impala.common.NotImplementedException; import com.cloudera.impala.service.FeSupport; import com.cloudera.impala.thrift.TColumnValue; +import com.cloudera.impala.thrift.TExprNode; import com.cloudera.impala.thrift.TQueryContext; import com.google.common.base.Preconditions; @@ -37,36 +39,80 @@ public abstract class LiteralExpr extends Expr implements Comparable i = exprs.listIterator(); while (i.hasNext()) { Expr expr = i.next(); - if (!(expr instanceof IntLiteral)) { - continue; - } + if (!(expr instanceof IntLiteral)) continue; long pos = ((IntLiteral) expr).getValue(); if (pos < 1) { throw new AnalysisException( diff --git a/fe/src/main/java/com/cloudera/impala/analysis/SlotRef.java b/fe/src/main/java/com/cloudera/impala/analysis/SlotRef.java index 2d2d0de33..2342d3963 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/SlotRef.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/SlotRef.java @@ -125,6 +125,7 @@ public class SlotRef extends Expr { Objects.ToStringHelper toStrHelper = Objects.toStringHelper(this); String tblNameStr = (tblName_ == null ? "null" : tblName_.toString()); toStrHelper.add("tblName", tblNameStr); + toStrHelper.add("type", type_); toStrHelper.add("col", col_); String idStr = (desc_ == null ? "null" : Integer.toString(desc_.getId().asInt())); toStrHelper.add("id", idStr); diff --git a/fe/src/main/java/com/cloudera/impala/analysis/StringLiteral.java b/fe/src/main/java/com/cloudera/impala/analysis/StringLiteral.java index 9332d891d..08405d3e0 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/StringLiteral.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/StringLiteral.java @@ -16,6 +16,7 @@ package com.cloudera.impala.analysis; import java.io.IOException; import java.io.StringReader; +import java.math.BigDecimal; import java.math.BigInteger; import java_cup.runtime.Symbol; @@ -104,7 +105,7 @@ public class StringLiteral extends LiteralExpr { StringReader reader = new StringReader(value_); SqlScanner scanner = new SqlScanner(reader); // For distinguishing positive and negative numbers. - double multiplier = 1; + boolean negative = false; Symbol sym; try { // We allow simple chaining of MINUS to recognize negative numbers. @@ -113,7 +114,7 @@ public class StringLiteral extends LiteralExpr { // This would require invoking the parser. sym = scanner.next_token(); while (sym.sym == SqlParserSymbols.SUBTRACT) { - multiplier *= -1; + negative = !negative; sym = scanner.next_token(); } } catch (IOException e) { @@ -123,11 +124,14 @@ public class StringLiteral extends LiteralExpr { throw new AnalysisException("Number too large: " + value_); } if (sym.sym == SqlParserSymbols.INTEGER_LITERAL) { - Long val = (Long) sym.value; - return new IntLiteral(BigInteger.valueOf(val * (long)multiplier)); + BigInteger val = (BigInteger) sym.value; + if (negative) val = val.negate(); + return new IntLiteral(val); } - if (sym.sym == SqlParserSymbols.FLOATINGPOINT_LITERAL) { - return new FloatLiteral((Double) sym.value * multiplier); + if (sym.sym == SqlParserSymbols.DECIMAL_LITERAL) { + BigDecimal val = (BigDecimal) sym.value; + if (negative) val = val.negate(); + return new DecimalLiteral(val); } // Symbol is not an integer or floating point literal. throw new AnalysisException( diff --git a/fe/src/main/java/com/cloudera/impala/analysis/TypesUtil.java b/fe/src/main/java/com/cloudera/impala/analysis/TypesUtil.java index 9af114db3..fda31d4ed 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/TypesUtil.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/TypesUtil.java @@ -14,6 +14,8 @@ package com.cloudera.impala.analysis; +import java.math.BigDecimal; + import com.cloudera.impala.catalog.ColumnType; import com.cloudera.impala.common.AnalysisException; import com.google.common.base.Preconditions; @@ -163,4 +165,42 @@ public class TypesUtil { "Operation '" + op + "' is not allowed for decimal types."); } } + + /** + * Computes the ColumnType that can represent 'v' with no loss of resolution. + * The scale/precision in BigDecimal is not compatible with SQL decimal semantics + * (much more like significant figures and exponent). + * Returns null if the value cannot be represented. + */ + public static ColumnType computeDecimalType(BigDecimal v) { + // PlainString returns the string with no exponent. We walk it to compute + // the digits before and after. + // TODO: better way? + String str = v.toPlainString(); + int digitsBefore = 0; + int digitsAfter = 0; + boolean decimalFound = false; + boolean leadingZeros = true; + for (int i = 0; i < str.length(); ++i) { + char c = str.charAt(i); + if (c == '-') continue; + if (c == '.') { + decimalFound = true; + continue; + } + if (decimalFound) { + ++digitsAfter; + } else { + // Strip out leading 0 before the decimal point. We want "0.1" to + // be parsed as ".1" (1 digit instead of 2). + if (c == '0' && leadingZeros) continue; + leadingZeros = false; + ++digitsBefore; + } + } + if (digitsAfter >= ColumnType.MAX_SCALE) return null; + if (digitsBefore + digitsAfter >= ColumnType.MAX_PRECISION) return null; + if (digitsBefore == 0 && digitsAfter == 0) digitsBefore = 1; + return ColumnType.createDecimalType(digitsBefore + digitsAfter, digitsAfter); + } } diff --git a/fe/src/main/java/com/cloudera/impala/catalog/ColumnType.java b/fe/src/main/java/com/cloudera/impala/catalog/ColumnType.java index f4bedd3b3..c8d2a3f13 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/ColumnType.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/ColumnType.java @@ -273,7 +273,7 @@ public class ColumnType { public boolean isIntegerType() { return type_ == PrimitiveType.TINYINT || type_ == PrimitiveType.SMALLINT - || type_ == PrimitiveType.INT || type_ == PrimitiveType.BIGINT; + || type_ == PrimitiveType.INT || type_ == PrimitiveType.BIGINT; } public boolean isFixedLengthType() { @@ -332,13 +332,25 @@ public class ColumnType { case TINYINT: return createDecimalType(3); case SMALLINT: return createDecimalType(5); case INT: return createDecimalType(10); - case BIGINT: return createDecimalType(20); + case BIGINT: return createDecimalType(19); case FLOAT: return createDecimalTypeInternal(MAX_PRECISION, 9); case DOUBLE: return createDecimalTypeInternal(MAX_PRECISION, 17); } return ColumnType.INVALID; } + /* Returns true if this decimal type is a supertype of the other decimal type. + * e.g. (10,3) is a super type of (3,3) but (5,4) is not a supertype of (3,0). + * To be a super type of another decimal, the number of digits before and after + * the decimal point must be greater or equal. + */ + public boolean isSupertypeOf(ColumnType o) { + Preconditions.checkState(isDecimal()); + Preconditions.checkState(o.isDecimal()); + if (isWildcardDecimal()) return true; + return scale_ >= o.scale_ && precision_ - scale_ >= o.precision_ - o.scale_; + } + /** * Return type t such that values from both t1 and t2 can be assigned to t * without loss of precision. Returns INVALID_TYPE if there is no such type @@ -349,19 +361,21 @@ public class ColumnType { if (!t1.isValid() || !t2.isValid()) return INVALID; if (t1.equals(t2)) return t1; - if (t1.isDecimal() || t2.isDecimal()) { if (t1.isNull()) return t2; if (t2.isNull()) return t1; // Allow casts between decimal and numeric types by converting // numeric types to the containing decimal type. - t1 = t1.getMinResolutionDecimal(); - t2 = t2.getMinResolutionDecimal(); - if (t1.isInvalid() || t2.isInvalid()) return ColumnType.INVALID; - Preconditions.checkState(t1.isDecimal()); - Preconditions.checkState(t2.isDecimal()); - return TypesUtil.getDecimalAssignmentCompatibleType(t1, t2); + ColumnType t1Decimal = t1.getMinResolutionDecimal(); + ColumnType t2Decimal = t2.getMinResolutionDecimal(); + if (t1Decimal.isInvalid() || t2Decimal.isInvalid()) return ColumnType.INVALID; + Preconditions.checkState(t1Decimal.isDecimal()); + Preconditions.checkState(t2Decimal.isDecimal()); + + if (t1Decimal.isSupertypeOf(t2Decimal)) return t1; + if (t2Decimal.isSupertypeOf(t1Decimal)) return t2; + return TypesUtil.getDecimalAssignmentCompatibleType(t1Decimal, t2Decimal); } PrimitiveType smallerType = @@ -379,7 +393,8 @@ public class ColumnType { * t1 to t2 results in no loss of precision). */ public static boolean isImplicitlyCastable(ColumnType t1, ColumnType t2) { - return getAssignmentCompatibleType(t1, t2).matchesType(t2); + return getAssignmentCompatibleType(t1, t2).matchesType(t2) || + getAssignmentCompatibleType(t2, t1).matchesType(t2); } /** @@ -450,6 +465,7 @@ public class ColumnType { if (type_ == PrimitiveType.CHAR) { return "CHAR(" + len_ + ")"; } else if (type_ == PrimitiveType.DECIMAL) { + if (isWildcardDecimal()) return "DECIMAL(*,*)"; return "DECIMAL(" + precision_ + "," + scale_ + ")"; } return type_.toString(); diff --git a/fe/src/main/java/com/cloudera/impala/catalog/Function.java b/fe/src/main/java/com/cloudera/impala/catalog/Function.java index 266ce4467..7ee89d476 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/Function.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/Function.java @@ -55,6 +55,8 @@ public class Function implements CatalogObject { // when matching a particular instantiation. That is, their fixed arguments // match exactly and the remaining varargs have the same type. // e.g. fn(int, int, int) and fn(int...) + // Argument types that are NULL are ignored when doing this comparison. + // e.g. fn(NULL, int) is indistinguishable from fn(int, int) IS_INDISTINGUISHABLE, // X is a supertype of Y if Y.arg[i] can be implicitly cast to X.arg[i]. If X has @@ -207,6 +209,7 @@ public class Function implements CatalogObject { int minArgs = Math.min(o.argTypes_.length, this.argTypes_.length); // The first fully specified args must be identical. for (int i = 0; i < minArgs; ++i) { + if (o.argTypes_[i].isNull() || this.argTypes_[i].isNull()) continue; if (!o.argTypes_[i].matchesType(this.argTypes_[i])) return false; } if (o.argTypes_.length == this.argTypes_.length) return true; @@ -215,10 +218,12 @@ public class Function implements CatalogObject { if (!o.getVarArgsType().matchesType(this.getVarArgsType())) return false; if (this.getNumArgs() > o.getNumArgs()) { for (int i = minArgs; i < this.getNumArgs(); ++i) { + if (this.argTypes_[i].isNull()) continue; if (!this.argTypes_[i].matchesType(o.getVarArgsType())) return false; } } else { for (int i = minArgs; i < o.getNumArgs(); ++i) { + if (o.argTypes_[i].isNull()) continue; if (!o.argTypes_[i].matchesType(this.getVarArgsType())) return false; } } @@ -227,6 +232,7 @@ public class Function implements CatalogObject { // o has var args so check the remaining arguments from this if (o.getNumArgs() > minArgs) return false; for (int i = minArgs; i < this.getNumArgs(); ++i) { + if (this.argTypes_[i].isNull()) continue; if (!this.argTypes_[i].matchesType(o.getVarArgsType())) return false; } return true; @@ -234,6 +240,7 @@ public class Function implements CatalogObject { // this has var args so check the remaining arguments from s if (this.getNumArgs() > minArgs) return false; for (int i = minArgs; i < o.getNumArgs(); ++i) { + if (o.argTypes_[i].isNull()) continue; if (!o.argTypes_[i].matchesType(this.getVarArgsType())) return false; } return true; diff --git a/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java b/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java index 6819ea34d..c69c7a398 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/HdfsPartition.java @@ -25,7 +25,6 @@ import org.slf4j.LoggerFactory; import com.cloudera.impala.analysis.Expr; import com.cloudera.impala.analysis.LiteralExpr; -import com.cloudera.impala.analysis.NullLiteral; import com.cloudera.impala.analysis.PartitionKeyValue; import com.cloudera.impala.thrift.ImpalaInternalServiceConstants; import com.cloudera.impala.thrift.TAccessLevel; @@ -451,7 +450,7 @@ public class HdfsPartition implements Comparable { clusterCols.size(), exprNodes.size())); for (int i = 0; i < exprNodes.size(); ++i) { - literalExpr.add(TExprNodeToLiteralExpr( + literalExpr.add(LiteralExpr.fromThrift( exprNodes.get(i), clusterCols.get(i).getType())); } } @@ -475,32 +474,6 @@ public class HdfsPartition implements Comparable { return partition; } - private static LiteralExpr TExprNodeToLiteralExpr(TExprNode exprNode, - ColumnType primitiveType) { - try { - switch (exprNode.node_type) { - case FLOAT_LITERAL: - return (LiteralExpr) (LiteralExpr.create(Double.toString( - exprNode.float_literal.value), primitiveType).castTo(primitiveType)); - case INT_LITERAL: - return (LiteralExpr) (LiteralExpr.create(Long.toString( - exprNode.int_literal.value), primitiveType).castTo(primitiveType)); - case STRING_LITERAL: - return LiteralExpr.create(exprNode.string_literal.value, primitiveType); - case BOOL_LITERAL: - return LiteralExpr.create(Boolean.toString(exprNode.bool_literal.value), - primitiveType); - case NULL_LITERAL: - return new NullLiteral(); - default: - throw new UnsupportedOperationException("Unsupported partition key type: " + - exprNode.node_type); - } - } catch (Exception e) { - throw new IllegalStateException("Error creating LiteralExpr: ", e); - } - } - public THdfsPartition toThrift(boolean includeFileDescriptorMetadata) { List thriftExprs = Expr.treesToThrift(getPartitionValues()); diff --git a/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java b/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java index 28b4c40a5..ed1f77e43 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java @@ -49,7 +49,6 @@ import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.cloudera.impala.analysis.Expr; import com.cloudera.impala.analysis.LiteralExpr; import com.cloudera.impala.analysis.NullLiteral; import com.cloudera.impala.analysis.PartitionKeyValue; @@ -565,10 +564,8 @@ public class HdfsTable extends Table { } else { ColumnType type = colsByPos_.get(keyValues.size()).getType(); try { - Expr expr = LiteralExpr.create(partitionKey, type); - // Force the literal to be of type declared in the metadata. - expr = expr.castTo(type); - keyValues.add((LiteralExpr) expr); + LiteralExpr expr = LiteralExpr.create(partitionKey, type); + keyValues.add(expr); } catch (Exception ex) { LOG.warn("Failed to create literal expression of type: " + type, ex); throw new InvalidStorageDescriptorException(ex); diff --git a/fe/src/main/java/com/cloudera/impala/common/TreeNode.java b/fe/src/main/java/com/cloudera/impala/common/TreeNode.java index ee3bee058..d1636ae1f 100644 --- a/fe/src/main/java/com/cloudera/impala/common/TreeNode.java +++ b/fe/src/main/java/com/cloudera/impala/common/TreeNode.java @@ -53,6 +53,11 @@ public class TreeNode> { */ public , D extends C> void collect( Predicate predicate, List matches) { + // TODO: the semantics of this function are very strange. contains() + // checks using .equals() on the nodes. In the case of literals, slotrefs + // and maybe others, two different tree node objects can be equal and + // this function would only return one of them. This is not intuitive. + // We rely on these semantics to not have duplicate nodes. Investigate this. if (predicate.apply((C) this) && !matches.contains(this)) { matches.add((D) this); return; @@ -63,6 +68,22 @@ public class TreeNode> { } } + /** + * Add all nodes in the tree that satisfy 'predicate' to the list 'matches' + * This node is checked first, followed by its children in order. All nodes + * that match in the subtree are added. + */ + public , D extends C> void collectAll( + Predicate predicate, List matches) { + if (predicate.apply((C) this)) { + matches.add((D) this); + } + + for (NodeType child: children_) { + child.collectAll(predicate, matches); + } + } + /** * For each expression in 'nodeList', collect all subexpressions satisfying 'predicate' * into 'matches' diff --git a/fe/src/main/java/com/cloudera/impala/planner/DataSourceScanNode.java b/fe/src/main/java/com/cloudera/impala/planner/DataSourceScanNode.java index 14f751a48..a9e48d40d 100644 --- a/fe/src/main/java/com/cloudera/impala/planner/DataSourceScanNode.java +++ b/fe/src/main/java/com/cloudera/impala/planner/DataSourceScanNode.java @@ -23,8 +23,8 @@ import com.cloudera.impala.analysis.Analyzer; import com.cloudera.impala.analysis.BinaryPredicate; import com.cloudera.impala.analysis.BoolLiteral; import com.cloudera.impala.analysis.CompoundPredicate; +import com.cloudera.impala.analysis.DecimalLiteral; import com.cloudera.impala.analysis.Expr; -import com.cloudera.impala.analysis.FloatLiteral; import com.cloudera.impala.analysis.IntLiteral; import com.cloudera.impala.analysis.LiteralExpr; import com.cloudera.impala.analysis.SlotRef; @@ -119,7 +119,7 @@ public class DataSourceScanNode extends ScanNode { return new TColumnValue().setLong_val(((IntLiteral) expr).getValue()); case FLOAT: case DOUBLE: - return new TColumnValue().setDouble_val(((FloatLiteral) expr).getValue()); + return new TColumnValue().setDouble_val(((DecimalLiteral) expr).getDoubleValue()); case STRING: return new TColumnValue().setString_val(((StringLiteral) expr).getValue()); case DECIMAL: diff --git a/fe/src/main/jflex/sql-scanner.flex b/fe/src/main/jflex/sql-scanner.flex index f40e8d984..d28c04531 100644 --- a/fe/src/main/jflex/sql-scanner.flex +++ b/fe/src/main/jflex/sql-scanner.flex @@ -16,6 +16,7 @@ package com.cloudera.impala.analysis; import java_cup.runtime.Symbol; import java.lang.Integer; +import java.math.BigDecimal; import java.math.BigInteger; import java.util.HashMap; import java.util.LinkedHashMap; @@ -216,8 +217,8 @@ import com.cloudera.impala.analysis.SqlParserSymbols; tokenIdMap.put(new Integer(SqlParserSymbols.RPAREN), ")"); tokenIdMap.put(new Integer(SqlParserSymbols.LBRACKET), "["); tokenIdMap.put(new Integer(SqlParserSymbols.RBRACKET), "]"); - tokenIdMap.put(new Integer(SqlParserSymbols.FLOATINGPOINT_LITERAL), - "FLOATING POINT LITERAL"); + tokenIdMap.put(new Integer(SqlParserSymbols.DECIMAL_LITERAL), + "DECIMAL LITERAL"); tokenIdMap.put(new Integer(SqlParserSymbols.INTEGER_LITERAL), "INTEGER LITERAL"); tokenIdMap.put(new Integer(SqlParserSymbols.NOT), "!"); tokenIdMap.put(new Integer(SqlParserSymbols.LESSTHAN), "<"); @@ -269,7 +270,7 @@ FLit1 = [0-9]+ \. [0-9]* FLit2 = \. [0-9]+ FLit3 = [0-9]+ Exponent = [eE] [+-]? [0-9]+ -DoubleLiteral = ({FLit1}|{FLit2}|{FLit3}) {Exponent}? +DecimalLiteral = ({FLit1}|{FLit2}|{FLit3}) {Exponent}? IdentifierOrKw = [:digit:]*[:jletter:][:jletterdigit:]* | "&&" | "||" QuotedIdentifier = \`(\\.|[^\\\`])*\` @@ -318,18 +319,14 @@ EndOfLineComment = "--" {NonTerminator}* {LineTerminator}? return newToken(SqlParserSymbols.INTEGER_LITERAL, val); } -{DoubleLiteral} { - Double val = null; +{DecimalLiteral} { + BigDecimal val = null; try { - val = new Double(yytext()); + val = new BigDecimal(yytext()); } catch (NumberFormatException e) { return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext()); } - // conversion succeeded but literal is infinity or not a number - if (val.isInfinite() || val.isNaN()) { - return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext()); - } - return newToken(SqlParserSymbols.FLOATINGPOINT_LITERAL, val); + return newToken(SqlParserSymbols.DECIMAL_LITERAL, val); } {QuotedIdentifier} { diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeExprsTest.java index 913e2eeb2..4f07fddb0 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeExprsTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeExprsTest.java @@ -76,7 +76,6 @@ public class AnalyzeExprsTest extends AnalyzerTest { "Literal '9223372036854775808' exceeds maximum range of integers."); // Test floating-point types. - // TODO: Fix detecting the min resolution type for floating-point literals. testNumericLiteral(Float.toString(Float.MIN_VALUE), ColumnType.DOUBLE); testNumericLiteral(Float.toString(Float.MAX_VALUE), ColumnType.DOUBLE); testNumericLiteral("-" + Float.toString(Float.MIN_VALUE), ColumnType.DOUBLE); @@ -85,6 +84,11 @@ public class AnalyzeExprsTest extends AnalyzerTest { testNumericLiteral(Double.toString(Double.MAX_VALUE), ColumnType.DOUBLE); testNumericLiteral("-" + Double.toString(Double.MIN_VALUE), ColumnType.DOUBLE); testNumericLiteral("-" + Double.toString(Double.MAX_VALUE), ColumnType.DOUBLE); + + AnalysisError(String.format("select %s1", Double.toString(Double.MAX_VALUE)), + "Decimal literal '1.7976931348623157E+3081' exceeds maximum range of doubles."); + AnalysisError(String.format("select %s1", Double.toString(Double.MIN_VALUE)), + "Decimal literal '4.9E-3241' underflows minimum resolution of doubles."); } /** @@ -167,15 +171,11 @@ public class AnalyzeExprsTest extends AnalyzerTest { @Test public void TestDecimalCasts() throws AnalysisException { - String decimal = "cast('1.1' as decimal)"; - AnalysisError("select cast(" + decimal + " as boolean)", - "Invalid type cast of CAST('1.1' AS DECIMAL(9,0)) " + - "from DECIMAL(9,0) to BOOLEAN"); + AnalyzesOk("select cast(1.1 as boolean)"); + AnalyzesOk("select cast(1.1 as timestamp)"); + AnalysisError("select cast(true as decimal)", "Invalid type cast of TRUE from BOOLEAN to DECIMAL(9,0)"); - AnalysisError("select cast(" + decimal + " as timestamp)", - "Invalid type cast of CAST('1.1' AS DECIMAL(9,0)) " + - "from DECIMAL(9,0) to TIMESTAMP"); AnalysisError("select cast(cast(1 as timestamp) as decimal)", "Invalid type cast of CAST(1 AS TIMESTAMP) from TIMESTAMP to DECIMAL(9,0)"); @@ -183,7 +183,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { if (type.isNull() || type.isDecimal() || type.isBoolean() || type.isDateType()) { continue; } - AnalyzesOk("select cast(" + decimal + " as " + type + ")"); + AnalyzesOk("select cast(1.1 as " + type + ")"); AnalyzesOk("select cast(cast(1 as " + type + ") as decimal)"); } @@ -191,7 +191,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { for (int precision = 1; precision <= ColumnType.MAX_PRECISION; ++precision) { for (int scale = 0; scale < precision; ++scale) { ColumnType t = ColumnType.createDecimalType(precision, scale); - AnalyzesOk("select cast(" + decimal + " as " + t + ")"); + AnalyzesOk("select cast(1.1 as " + t + ")"); AnalyzesOk("select cast(cast(1 as " + t + ") as decimal)"); } } @@ -616,30 +616,103 @@ public class AnalyzeExprsTest extends AnalyzerTest { assertEquals(PrimitiveType.BOOLEAN, expr.getType().getPrimitiveType()); } } - checkCasts(expr); // The children's types must be NULL or equal to the requested opType. ColumnType child1Type = expr.getChild(0).getType(); ColumnType child2Type = type1 == null ? null : expr.getChild(1).getType(); - Assert.assertTrue(opType.equals(child1Type) - || opType.isNull() || child1Type.isNull()); + Assert.assertTrue("opType= " + opType + " child1Type=" + child1Type, + opType.equals(child1Type) || opType.isNull() || child1Type.isNull()); if (type1 != null) { - Assert.assertTrue(opType.equals(child2Type) - || opType.isNull() || child2Type.isNull()); + Assert.assertTrue("opType= " + opType + " child2Type=" + child2Type, + opType.equals(child2Type) || opType.isNull() || child2Type.isNull()); } } + private void checkReturnType(String stmt, ColumnType resultType) { + SelectStmt select = (SelectStmt) AnalyzesOk(stmt); + ArrayList selectListExprs = select.getResultExprs(); + assertNotNull(selectListExprs); + assertEquals(selectListExprs.size(), 1); + // check the first expr in select list + Expr expr = selectListExprs.get(0); + assertEquals("Expected: " + resultType + " != " + expr.getType(), + resultType, expr.getType()); + } + + @Test + public void TestNumericLiteralTypeResolution() throws AnalysisException { + checkReturnType("select 1", ColumnType.TINYINT); + checkReturnType("select 1.1", ColumnType.createDecimalType(2,1)); + checkReturnType("select 01.1", ColumnType.createDecimalType(2,1)); + checkReturnType("select 1 + 1.1", ColumnType.DOUBLE); + checkReturnType("select 0.23 + 1.1", ColumnType.createDecimalType(4,2)); + + checkReturnType("select float_col + float_col from functional.alltypestiny", + ColumnType.DOUBLE); + checkReturnType("select int_col + int_col from functional.alltypestiny", + ColumnType.BIGINT); + + // floating point + numeric literal = floating point + checkReturnType("select float_col + 1.1 from functional.alltypestiny", + ColumnType.DOUBLE); + // decimal + numeric literal = decimal + checkReturnType("select d1 + 1.1 from functional.decimal_tbl", + ColumnType.createDecimalType(11,1)); + // int + numeric literal = floating point + checkReturnType("select int_col + 1.1 from functional.alltypestiny", + ColumnType.DOUBLE); + + // Explicitly casting the literal to a decimal will override the behavior + checkReturnType("select int_col + cast(1.1 as decimal(2,1)) from " + + " functional.alltypestiny", ColumnType.createDecimalType(12,1)); + checkReturnType("select float_col + cast(1.1 as decimal(2,1)) from " + + " functional.alltypestiny", ColumnType.createDecimalType(38,9)); + checkReturnType("select float_col + cast(1.1*1.2+1.3 as decimal(2,1)) from " + + " functional.alltypestiny", ColumnType.createDecimalType(38,9)); + + // The location and complexity of the expr should not matter. + checkReturnType("select 1.0 + float_col + 1.1 from functional.alltypestiny", + ColumnType.DOUBLE); + checkReturnType("select 1.0 + 2.0 + float_col from functional.alltypestiny", + ColumnType.DOUBLE); + checkReturnType("select 1.0 + 2.0 + pi() * float_col from functional.alltypestiny", + ColumnType.DOUBLE); + checkReturnType("select 1.0 + d1 + 1.1 from functional.decimal_tbl", + ColumnType.createDecimalType(12,1)); + checkReturnType("select 1.0 + 2.0 + d1 from functional.decimal_tbl", + ColumnType.createDecimalType(11,1)); + checkReturnType("select 1.0 + 2.0 + pi()*d1 from functional.decimal_tbl", + ColumnType.createDecimalType(38,17)); + + // Test with multiple cols + checkReturnType("select double_col + 1.23 + float_col + 1.0 " + + " from functional.alltypestiny", ColumnType.DOUBLE); + checkReturnType("select double_col + 1.23 + float_col + 1.0 + int_col " + + " + bigint_col from functional.alltypestiny", ColumnType.DOUBLE); + checkReturnType("select d1 + 1.23 + d2 + 1.0 " + + " from functional.decimal_tbl", ColumnType.createDecimalType(14,2)); + + // Test with slot of both decimal and non-decimal + checkReturnType("select t1.int_col + t2.c1 from functional.alltypestiny t1 " + + " cross join functional.decimal_tiny t2", ColumnType.createDecimalType(15,4)); + checkReturnType("select 1.1 + t1.int_col + t2.c1 from functional.alltypestiny t1 " + + " cross join functional.decimal_tiny t2", ColumnType.createDecimalType(38,17)); + } + /** * Check that: - * - we don't cast literals (we should have simply converted the literal + * - we don't implicitly cast literals (we should have simply converted the literal * to the target type) * - we don't do redundant casts (ie, we don't cast a bigint expr to a bigint) */ private void checkCasts(Expr expr) { if (expr instanceof CastExpr) { - Assert.assertFalse(expr.getType() + " == " + expr.getChild(0).getType(), - expr.getType().equals(expr.getChild(0).getType())); - Assert.assertFalse(expr.getChild(0) instanceof LiteralExpr); + CastExpr cast = (CastExpr)expr; + if (cast.isImplicit()) { + Assert.assertFalse(expr.getType() + " == " + expr.getChild(0).getType(), + expr.getType().equals(expr.getChild(0).getType())); + Assert.assertFalse(expr.debugString(), expr.getChild(0) instanceof LiteralExpr); + } } for (Expr child: expr.getChildren()) { checkCasts(child); @@ -672,7 +745,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { public void TestFixedPointArithmeticOps() throws AnalysisException { // negative tests, no floating point types allowed AnalysisError("select ~float_col from functional.alltypes", - "Bitwise operations only allowed on fixed-point types"); + "Bitwise operations only allowed on integer types"); AnalysisError("select float_col ^ int_col from functional.alltypes", "Invalid non-integer argument to operation '^'"); AnalysisError("select float_col & int_col from functional.alltypes", @@ -757,7 +830,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { // Non-function-call like version. AnalysisError("select timestamp_col + interval 5.2 years from functional.alltypes", "Operand '5.2' of timestamp arithmetic expression " + - "'timestamp_col + INTERVAL 5.2 years' returns type 'DOUBLE'. " + + "'timestamp_col + INTERVAL 5.2 years' returns type 'DECIMAL(2,1)'. " + "Expected an integer type."); // No implicit cast from STRING to integer types. @@ -776,7 +849,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { // Reversed interval and timestamp using addition. AnalysisError("select interval 5.2 years + timestamp_col from functional.alltypes", "Operand '5.2' of timestamp arithmetic expression " + - "'INTERVAL 5.2 years + timestamp_col' returns type 'DOUBLE'. " + + "'INTERVAL 5.2 years + timestamp_col' returns type 'DECIMAL(2,1)'. " + "Expected an integer type."); // Cast from STRING to INT. AnalyzesOk("select interval cast('10' as int) years + timestamp_col " + @@ -785,7 +858,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { AnalysisError("select date_add(timestamp_col, interval 5.2 years) " + "from functional.alltypes", "Operand '5.2' of timestamp arithmetic expression " + - "'DATE_ADD(timestamp_col, INTERVAL 5.2 years)' returns type 'DOUBLE'. " + + "'DATE_ADD(timestamp_col, INTERVAL 5.2 years)' returns type 'DECIMAL(2,1)'. " + "Expected an integer type."); // Cast from STRING to INT. AnalyzesOk("select date_add(timestamp_col, interval cast('10' as int) years) " + @@ -826,6 +899,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { "Cannot pass '*' to scalar function."); // Call function that only accepts decimal + AnalyzesOk("select precision(1)"); AnalyzesOk("select precision(cast('1.1' as decimal))"); AnalyzesOk("select scale(1.1)"); AnalysisError("select scale('1.1')", @@ -1043,7 +1117,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { "No matching function with signature: default.udf(TINYINT, STRING, TINYINT)."); AnalysisError("select udf(1.1)", - "No matching function with signature: default.udf(DOUBLE)"); + "No matching function with signature: default.udf(DECIMAL(2,1))"); AnalyzesOk("select functional.udf(1.1)"); AnalysisError("select functional.udf('Hello')", @@ -1202,7 +1276,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { testDecimalExpr(decimal_10_0 + " + cast(1 as int)", ColumnType.createDecimalType(11, 0)); testDecimalExpr(decimal_10_0 + " + cast(1 as bigint)", - ColumnType.createDecimalType(21, 0)); + ColumnType.createDecimalType(20, 0)); testDecimalExpr(decimal_10_0 + " + cast(1 as float)", ColumnType.createDecimalType(38, 9)); testDecimalExpr(decimal_10_0 + " + cast(1 as double)", @@ -1262,7 +1336,7 @@ public class AnalyzeExprsTest extends AnalyzerTest { AnalysisError("select d1 ^ d1 from functional.decimal_tbl", "Invalid non-integer argument to operation '^': d1 ^ d1"); AnalysisError("select ~d1 from functional.decimal_tbl", - "Bitwise operations only allowed on fixed-point types: ~d1"); + "Bitwise operations only allowed on integer types: ~d1"); AnalyzesOk("select d3 = d4 from functional.decimal_tbl"); AnalyzesOk("select d5 != d1 from functional.decimal_tbl"); @@ -1311,13 +1385,22 @@ public class AnalyzeExprsTest extends AnalyzerTest { AnalyzesOk("select truncate(cast(1.123 as decimal(10,3)), -1)"); AnalysisError("select round(cast(1.123 as decimal(10,3)), 5.1)", - "No matching function with signature: round(DECIMAL(10,3), DOUBLE)"); - AnalysisError("select round(cast(1.123 as decimal(10,3)), 40)", + "No matching function with signature: round(DECIMAL(10,3), DECIMAL(2,1))"); + AnalysisError("select round(cast(1.123 as decimal(30,20)), 40)", "Cannot round/truncate to scales greater than 38."); AnalysisError("select truncate(cast(1.123 as decimal(10,3)), 40)", "Cannot round/truncate to scales greater than 38."); AnalysisError("select round(cast(1.123 as decimal(10,3)), NULL)", "round() cannot be called with a NULL second argument."); + + testDecimalExpr("round(1.23)", ColumnType.createDecimalType(1, 0)); + testDecimalExpr("round(1.23, 1)", ColumnType.createDecimalType(2, 1)); + testDecimalExpr("round(1.23, 0)", ColumnType.createDecimalType(1, 0)); + testDecimalExpr("round(1.23, 3)", ColumnType.createDecimalType(4, 3)); + testDecimalExpr("round(1.23, -1)", ColumnType.createDecimalType(1, 0)); + testDecimalExpr("round(1.23, -2)", ColumnType.createDecimalType(1, 0)); + testDecimalExpr("round(cast(1.23 as decimal(3,2)), -2)", + ColumnType.createDecimalType(1, 0)); } /** diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java index 084370208..383442c7f 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java @@ -728,7 +728,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest { // arbitrary exprs not returning boolean AnalysisError("select count(*) from functional.alltypes " + "group by bool_col having 5 + 10 * 5.6", - "HAVING clause '5.0 + 10.0 * 5.6' requires return type 'BOOLEAN'. " + + "HAVING clause '5 + 10 * 5.6' requires return type 'BOOLEAN'. " + "Actual type is 'DOUBLE'."); AnalysisError("select count(*) from functional.alltypes " + "group by bool_col having int_col", diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java index 43550a724..059ae665a 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java @@ -61,8 +61,9 @@ public class AnalyzerTest { typeToLiteralValue_.put(ColumnType.SMALLINT, (Byte.MAX_VALUE + 1) + ""); typeToLiteralValue_.put(ColumnType.INT, (Short.MAX_VALUE + 1) + ""); typeToLiteralValue_.put(ColumnType.BIGINT, ((long) Integer.MAX_VALUE + 1) + ""); - typeToLiteralValue_.put(ColumnType.FLOAT, "1.0"); - typeToLiteralValue_.put(ColumnType.DOUBLE, (Float.MAX_VALUE + 1) + ""); + typeToLiteralValue_.put(ColumnType.FLOAT, "cast(1.0 as float)"); + typeToLiteralValue_.put(ColumnType.DOUBLE, + "cast(" + (Float.MAX_VALUE + 1) + " as double)"); typeToLiteralValue_.put(ColumnType.TIMESTAMP, "cast('2012-12-21 00:00:00.000' as timestamp)"); typeToLiteralValue_.put(ColumnType.STRING, "'Hello, World!'"); @@ -497,14 +498,14 @@ public class AnalyzerTest { // Analysis error from non-integral values AnalysisError("select * from functional.AllTypes limit 10.0", - "LIMIT expression must be an integer type but is 'FLOAT': 10.0"); + "LIMIT expression must be an integer type but is 'DECIMAL(3,1)': 10.0"); AnalysisError("select * from functional.AllTypes limit NOT FALSE", "LIMIT expression must be an integer type but is 'BOOLEAN': NOT FALSE"); AnalysisError("select * from functional.AllTypes limit CAST(\"asdf\" AS INT)", "LIMIT expression evaluates to NULL: CAST('asdf' AS INT)"); AnalysisError("select * from functional.AllTypes order by id limit 10 " + "OFFSET 10.0", - "OFFSET expression must be an integer type but is 'FLOAT': 10.0"); + "OFFSET expression must be an integer type but is 'DECIMAL(3,1)': 10.0"); AnalysisError("select * from functional.AllTypes order by id limit 10 " + "offset CAST('asdf' AS INT)", "OFFSET expression evaluates to NULL: CAST('asdf' AS INT)"); diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ExprTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ExprTest.java index 3611da574..93f895998 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ExprTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ExprTest.java @@ -33,6 +33,7 @@ public class ExprTest { testLiteralExprPositive("1.0", ColumnType.FLOAT); testLiteralExprPositive("1.0", ColumnType.DOUBLE); testLiteralExprPositive("ABC", ColumnType.STRING); + testLiteralExprPositive("1.1", ColumnType.createDecimalType(2, 1)); // INVALID_TYPE should always fail testLiteralExprNegative("ABC", ColumnType.INVALID); @@ -52,9 +53,6 @@ public class ExprTest { testLiteralExprNegative("2010-01-01", ColumnType.DATE); testLiteralExprNegative("2010-01-01", ColumnType.DATETIME); testLiteralExprNegative("2010-01-01", ColumnType.TIMESTAMP); - - // No decimal literals. - testLiteralExprNegative("1.1", ColumnType.createDecimalType()); } private void testLiteralExprPositive(String value, ColumnType type) { diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java index 8fa32ff0a..cbf708080 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java @@ -570,12 +570,9 @@ public class ParserTest { ParsesOk(String.format("select -%s", Double.toString(Double.MIN_VALUE))); ParsesOk(String.format("select -%s", Double.toString(Double.MAX_VALUE))); - // Java converts a float underflow to 0.0. - // Since there is no easy, reliable way to detect underflow, - // we don't consider it an error. + // Test overflow and underflow ParsesOk(String.format("select %s1", Double.toString(Double.MIN_VALUE))); - // java converts a float overflow to infinity, we consider it an error - ParserError(String.format("select %s1", Double.toString(Double.MAX_VALUE))); + ParsesOk(String.format("select %s1", Double.toString(Double.MAX_VALUE))); } @Test @@ -658,30 +655,31 @@ public class ParserTest { ParsesOk("select a + + + 1 from t where a + + + 1", ArithmeticExpr.class); // float literals - ParsesOk("select +1.0 from t where +1.0", FloatLiteral.class); - ParsesOk("select +-1.0 from t where +-1.0", FloatLiteral.class); + ParsesOk("select +1.0 from t where +1.0", DecimalLiteral.class); + ParsesOk("select +-1.0 from t where +-1.0", DecimalLiteral.class); ParsesOk("select +1.-0 from t where +1.-0", ArithmeticExpr.class); + // test scientific notation - ParsesOk("select 8e6 from t where 8e6", FloatLiteral.class); - ParsesOk("select +8e6 from t where +8e6", FloatLiteral.class); - ParsesOk("select 8e+6 from t where 8e+6", FloatLiteral.class); - ParsesOk("select -8e6 from t where -8e6", FloatLiteral.class); - ParsesOk("select 8e-6 from t where 8e-6", FloatLiteral.class); - ParsesOk("select -8e-6 from t where -8e-6", FloatLiteral.class); + ParsesOk("select 8e6 from t where 8e6", DecimalLiteral.class); + ParsesOk("select +8e6 from t where +8e6", DecimalLiteral.class); + ParsesOk("select 8e+6 from t where 8e+6", DecimalLiteral.class); + ParsesOk("select -8e6 from t where -8e6", DecimalLiteral.class); + ParsesOk("select 8e-6 from t where 8e-6", DecimalLiteral.class); + ParsesOk("select -8e-6 from t where -8e-6", DecimalLiteral.class); // with a decimal point - ParsesOk("select 4.5e2 from t where 4.5e2", FloatLiteral.class); - ParsesOk("select +4.5e2 from t where +4.5e2", FloatLiteral.class); - ParsesOk("select 4.5e+2 from t where 4.5e+2", FloatLiteral.class); - ParsesOk("select -4.5e2 from t where -4.5e2", FloatLiteral.class); - ParsesOk("select 4.5e-2 from t where 4.5e-2", FloatLiteral.class); - ParsesOk("select -4.5e-2 from t where -4.5e-2", FloatLiteral.class); + ParsesOk("select 4.5e2 from t where 4.5e2", DecimalLiteral.class); + ParsesOk("select +4.5e2 from t where +4.5e2", DecimalLiteral.class); + ParsesOk("select 4.5e+2 from t where 4.5e+2", DecimalLiteral.class); + ParsesOk("select -4.5e2 from t where -4.5e2", DecimalLiteral.class); + ParsesOk("select 4.5e-2 from t where 4.5e-2", DecimalLiteral.class); + ParsesOk("select -4.5e-2 from t where -4.5e-2", DecimalLiteral.class); // with a decimal point but without a number before the decimal - ParsesOk("select .7e9 from t where .7e9", FloatLiteral.class); - ParsesOk("select +.7e9 from t where +.7e9", FloatLiteral.class); - ParsesOk("select .7e+9 from t where .7e+9", FloatLiteral.class); - ParsesOk("select -.7e9 from t where -.7e9", FloatLiteral.class); - ParsesOk("select .7e-9 from t where .7e-9", FloatLiteral.class); - ParsesOk("select -.7e-9 from t where -.7e-9", FloatLiteral.class); + ParsesOk("select .7e9 from t where .7e9", DecimalLiteral.class); + ParsesOk("select +.7e9 from t where +.7e9", DecimalLiteral.class); + ParsesOk("select .7e+9 from t where .7e+9", DecimalLiteral.class); + ParsesOk("select -.7e9 from t where -.7e9", DecimalLiteral.class); + ParsesOk("select .7e-9 from t where .7e-9", DecimalLiteral.class); + ParsesOk("select -.7e-9 from t where -.7e-9", DecimalLiteral.class); // mixed signs ParsesOk("select -+-1 from t where -+-1", IntLiteral.class); diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java index 2b1855eec..eb43833f1 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java @@ -83,16 +83,16 @@ public class ToSqlTest extends AnalyzerTest { public void selectListTest() { testToSql("select 1234, 1234.0, 1234.0 + 1, 1234.0 + 1.0, 1 + 1, \"abc\" " + "from functional.alltypes", - "SELECT 1234, 1234.0, 1234.0 + 1.0, 1234.0 + 1.0, 1 + 1, 'abc' " + + "SELECT 1234, 1234.0, 1234.0 + 1, 1234.0 + 1.0, 1 + 1, 'abc' " + "FROM functional.alltypes"); // Test aliases. testToSql("select 1234 i, 1234.0 as j, (1234.0 + 1) k, (1234.0 + 1.0) as l " + "from functional.alltypes", - "SELECT 1234 i, 1234.0 j, (1234.0 + 1.0) k, (1234.0 + 1.0) l " + + "SELECT 1234 i, 1234.0 j, (1234.0 + 1) k, (1234.0 + 1.0) l " + "FROM functional.alltypes"); // Test select without from. testToSql("select 1234 i, 1234.0 as j, (1234.0 + 1) k, (1234.0 + 1.0) as l", - "SELECT 1234 i, 1234.0 j, (1234.0 + 1.0) k, (1234.0 + 1.0) l"); + "SELECT 1234 i, 1234.0 j, (1234.0 + 1) k, (1234.0 + 1.0) l"); // Test select without from. testToSql("select null, 1234 < 5678, 1234.0 < 5678.0, 1234 < null " + "from functional.alltypes", @@ -405,7 +405,7 @@ public class ToSqlTest extends AnalyzerTest { "SELECT t1.id, t2.id FROM " + "(SELECT id, string_col FROM functional.alltypes) t1, " + "(SELECT id, float_col FROM functional.alltypes) t2 " + - "WHERE t1.id = t2.id AND t1.string_col = 'abc' AND t2.float_col < 10.0"); + "WHERE t1.id = t2.id AND t1.string_col = 'abc' AND t2.float_col < 10"); } @Test @@ -546,11 +546,11 @@ public class ToSqlTest extends AnalyzerTest { testToSql("select 1 * 1, (1 * 1), 2 / 2, (2 / 2), 3 % 3, (3 % 3), " + "4 DIV 4, (4 DIV 4), 5 + 5, (5 + 5), 6 - 6, (6 - 6), 7 & 7, (7 & 7), " + "8 | 8, (8 | 8), 9 ^ 9, (9 ^ 9), ~10, (~10)", - "SELECT 1 * 1, (1 * 1), 2.0 / 2.0, (2.0 / 2.0), 3 % 3, (3 % 3), " + + "SELECT 1 * 1, (1 * 1), 2 / 2, (2 / 2), 3 % 3, (3 % 3), " + "4 DIV 4, (4 DIV 4), 5 + 5, (5 + 5), 6 - 6, (6 - 6), 7 & 7, (7 & 7), " + "8 | 8, (8 | 8), 9 ^ 9, (9 ^ 9), ~10, (~10)"); testToSql("select (((1 + 2) * (3 - 4) + 6) / 7)", - "SELECT (((1 + 2) * (3 - 4) + 6) / 7.0)"); + "SELECT (((1 + 2) * (3 - 4) + 6) / 7)"); // CaseExpr. // Single case without else clause. No case expr. diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index e1cf80b4f..6621a3e27 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -1273,13 +1273,17 @@ d2 DECIMAL(10, 0) d3 DECIMAL(20, 10) d4 DECIMAL(38, 38) d5 DECIMAL(10, 5) +---- PARTITION_COLUMNS +d6 DECIMAL(9, 0) +---- ALTER +ALTER TABLE {table_name} ADD IF NOT EXISTS PARTITION(d6=1); ---- ROW_FORMAT delimited fields terminated by ',' ---- LOAD -`hadoop fs -mkdir -p /test-warehouse/decimal_tbl && hadoop fs -put -f \ -${IMPALA_HOME}/testdata/data/decimal_tbl.txt /test-warehouse/decimal_tbl/ +`hadoop fs -mkdir -p /test-warehouse/decimal_tbl/d6=1 && hadoop fs -put -f \ +${IMPALA_HOME}/testdata/data/decimal_tbl.txt /test-warehouse/decimal_tbl/d6=1/ ---- DEPENDENT_LOAD -INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} +INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} partition(d6) select * from functional.{table_name}; ==== ---- DATASET diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test b/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test index dd0e6a69c..aa19c06ce 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/aggregation.test @@ -154,7 +154,7 @@ where t.x > 10 01:AGGREGATE [FINALIZE] | output: sum(bigint_col), count(bigint_col) | group by: int_col -| having: sum(bigint_col) / count(bigint_col) > 10.0 +| having: sum(bigint_col) / count(bigint_col) > 10 | 00:SCAN HDFS [functional.alltypes] partitions=24/24 size=478.45KB diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test b/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test index a14ab0406..e4f10b77c 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test @@ -7,7 +7,7 @@ where tinyint_col < 256 and ---- PLAN 00:SCAN DATA SOURCE [functional.alltypes_datasource] data source predicates: tinyint_col < 256 -predicates: float_col != 0.0, CAST(int_col AS BIGINT) < 10 +predicates: float_col != 0, CAST(int_col AS BIGINT) < 10 ==== # The first four predicates are in a form that can be offered to the data source # and the first and third will be accepted (it accepts every other conjunct). @@ -24,7 +24,7 @@ where 10 > int_col and ---- PLAN 00:SCAN DATA SOURCE [functional.alltypes_datasource] data source predicates: 10 > int_col, string_col != 'Foo' -predicates: 5.0 > double_col, string_col != 'Bar', NOT TRUE = bool_col, NOT 5.0 = double_col +predicates: 5 > double_col, string_col != 'Bar', NOT TRUE = bool_col, NOT 5.0 = double_col ==== # The 3rd predicate is not in a form that can be offered to the data source so # the 4th will be offered and accepted instead. @@ -36,7 +36,7 @@ where int_col < 10 and ---- PLAN 00:SCAN DATA SOURCE [functional.alltypes_datasource] data source predicates: int_col < 10, bool_col != FALSE -predicates: double_col > 5.0, string_col IN ('Foo', 'Bar') +predicates: double_col > 5, string_col IN ('Foo', 'Bar') ==== # Tests that all predicates from the On-clause are applied (IMPALA-805) # and that slot equivalences are enforced at lowest possible plan node diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/join-order.test b/testdata/workloads/functional-planner/queries/PlannerTest/join-order.test index 3d3ad0a4e..54e96ca78 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/join-order.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/join-order.test @@ -24,10 +24,10 @@ order by limit 10 ---- PLAN 06:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 05:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_orderkey, o_orderdate, o_shippriority | 04:HASH JOIN [INNER JOIN] @@ -49,21 +49,21 @@ limit 10 predicates: l_shipdate > '1995-03-15' ---- DISTRIBUTEDPLAN 12:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 11:EXCHANGE [PARTITION=UNPARTITIONED] | 06:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 10:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | group by: l_orderkey, o_orderdate, o_shippriority | 09:EXCHANGE [PARTITION=HASH(l_orderkey,o_orderdate,o_shippriority)] | 05:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_orderkey, o_orderdate, o_shippriority | 04:HASH JOIN [INNER JOIN, BROADCAST] @@ -114,10 +114,10 @@ order by limit 10 ---- PLAN 06:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 05:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_orderkey, o_orderdate, o_shippriority | 04:HASH JOIN [INNER JOIN] @@ -139,21 +139,21 @@ limit 10 predicates: c.c_mktsegment = 'BUILDING' ---- DISTRIBUTEDPLAN 14:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 13:EXCHANGE [PARTITION=UNPARTITIONED] | 06:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 12:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | group by: l_orderkey, o_orderdate, o_shippriority | 11:EXCHANGE [PARTITION=HASH(l_orderkey,o_orderdate,o_shippriority)] | 05:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_orderkey, o_orderdate, o_shippriority | 04:HASH JOIN [INNER JOIN, PARTITIONED] @@ -210,10 +210,10 @@ order by limit 100 ---- PLAN 12:TOP-N [LIMIT=100] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 11:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: n_name | 10:HASH JOIN [INNER JOIN] @@ -252,21 +252,21 @@ limit 100 partitions=1/1 size=718.94MB ---- DISTRIBUTEDPLAN 21:TOP-N [LIMIT=100] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 20:EXCHANGE [PARTITION=UNPARTITIONED] | 12:TOP-N [LIMIT=100] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 19:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | group by: n_name | 18:EXCHANGE [PARTITION=HASH(n_name)] | 11:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: n_name | 10:HASH JOIN [INNER JOIN, BROADCAST] diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/joins.test b/testdata/workloads/functional-planner/queries/PlannerTest/joins.test index 99e2c6bb5..0c933f981 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/joins.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/joins.test @@ -165,7 +165,7 @@ and (b.double_col * c.tinyint_col > 1000 or c.tinyint_col < 1000) ---- PLAN 04:HASH JOIN [RIGHT OUTER JOIN] | hash predicates: a.id = c.id, b.string_col = c.string_col -| other predicates: a.day >= 6, b.month > 2, a.tinyint_col = 15, b.string_col = '15', a.tinyint_col + b.tinyint_col < 15, a.float_col - c.double_col < 0.0, (b.double_col * c.tinyint_col > 1000.0 OR c.tinyint_col < 1000) +| other predicates: a.day >= 6, b.month > 2, a.tinyint_col = 15, b.string_col = '15', a.tinyint_col + b.tinyint_col < 15, a.float_col - c.double_col < 0, (b.double_col * c.tinyint_col > 1000 OR c.tinyint_col < 1000) | |--02:SCAN HDFS [functional.alltypesaggnonulls c] | partitions=2/10 size=148.10KB compact @@ -185,7 +185,7 @@ and (b.double_col * c.tinyint_col > 1000 or c.tinyint_col < 1000) | 04:HASH JOIN [RIGHT OUTER JOIN, PARTITIONED] | hash predicates: a.id = c.id, b.string_col = c.string_col -| other predicates: a.day >= 6, b.month > 2, a.tinyint_col = 15, b.string_col = '15', a.tinyint_col + b.tinyint_col < 15, a.float_col - c.double_col < 0.0, (b.double_col * c.tinyint_col > 1000.0 OR c.tinyint_col < 1000) +| other predicates: a.day >= 6, b.month > 2, a.tinyint_col = 15, b.string_col = '15', a.tinyint_col + b.tinyint_col < 15, a.float_col - c.double_col < 0, (b.double_col * c.tinyint_col > 1000 OR c.tinyint_col < 1000) | |--08:EXCHANGE [PARTITION=HASH(c.id,c.string_col)] | | diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/subquery.test b/testdata/workloads/functional-planner/queries/PlannerTest/subquery.test index 41581d74d..bf3c918b4 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/subquery.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/subquery.test @@ -295,7 +295,7 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000 ---- PLAN 04:HASH JOIN [INNER JOIN] | hash predicates: a.tinyint_col = c.id -| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000.0 +| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000 | |--00:SCAN HDFS [functional.alltypessmall c] | partitions=4/4 size=6.32KB compact @@ -329,7 +329,7 @@ NODE 2: | 04:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: a.tinyint_col = c.id -| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000.0 +| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000 | |--06:EXCHANGE [BROADCAST] | | @@ -605,7 +605,7 @@ and c2 > 10 01:AGGREGATE [FINALIZE] | output: count(*), sum(functional.alltypesagg.int_col), count(functional.alltypesagg.int_col) | group by: functional.alltypesagg.int_col % 7 -| having: sum(int_col) / count(int_col) > 500.0 OR count(*) = 10, count(*) > 10, int_col % 7 IS NOT NULL +| having: sum(int_col) / count(int_col) > 500 OR count(*) = 10, count(*) > 10, int_col % 7 IS NOT NULL | 00:SCAN HDFS [functional.alltypesagg] partitions=10/10 size=743.67KB @@ -615,7 +615,7 @@ and c2 > 10 03:AGGREGATE [MERGE FINALIZE] | output: sum(count(*)), sum(sum(int_col)), sum(count(int_col)) | group by: int_col % 7 -| having: sum(int_col) / count(int_col) > 500.0 OR count(*) = 10, count(*) > 10, int_col % 7 IS NOT NULL +| having: sum(int_col) / count(int_col) > 500 OR count(*) = 10, count(*) > 10, int_col % 7 IS NOT NULL | 02:EXCHANGE [PARTITION=HASH(int_col % 7)] | @@ -764,7 +764,7 @@ and x.int_col + x.float_col + CAST(c.string_col AS FLOAT) < 1000 ---- PLAN 04:HASH JOIN [INNER JOIN] | hash predicates: a.tinyint_col = c.id -| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000.0 +| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000 | |--03:SCAN HDFS [functional.alltypessmall c] | partitions=4/4 size=6.32KB compact @@ -785,7 +785,7 @@ and x.int_col + x.float_col + CAST(c.string_col AS FLOAT) < 1000 | 04:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: a.tinyint_col = c.id -| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000.0 +| other predicates: a.int_col + b.float_col + CAST(c.string_col AS FLOAT) < 1000 | |--06:EXCHANGE [BROADCAST] | | diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds-all.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds-all.test index 83ab41e51..c51762695 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds-all.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds-all.test @@ -589,7 +589,7 @@ LIMIT 100 | |--02:SCAN HDFS [tpcds.store s] | partitions=1/1 size=3.08KB compact -| predicates: s_gmt_offset = -5.0 +| predicates: s_gmt_offset = -5 | 03:HASH JOIN [INNER JOIN] | hash predicates: ss.ss_sold_date_sk = d.d_date_sk @@ -626,7 +626,7 @@ LIMIT 100 | | | 02:SCAN HDFS [tpcds.store s] | partitions=1/1 size=3.08KB -| predicates: s_gmt_offset = -5.0 +| predicates: s_gmt_offset = -5 | 03:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: ss.ss_sold_date_sk = d.d_date_sk @@ -1818,7 +1818,7 @@ LIMIT 100 | | | |--02:SCAN HDFS [tpcds.household_demographics hd] | | partitions=1/1 size=148.10KB compact -| | predicates: (hd.hd_buy_potential = '>10000' OR hd.hd_buy_potential = 'unknown'), hd.hd_vehicle_count > 0, CASE WHEN hd.hd_vehicle_count > 0 THEN hd.hd_dep_count / hd.hd_vehicle_count ELSE NULL END > 1.0 +| | predicates: (hd.hd_buy_potential = '>10000' OR hd.hd_buy_potential = 'unknown'), hd.hd_vehicle_count > 0, CASE WHEN hd.hd_vehicle_count > 0 THEN hd.hd_dep_count / hd.hd_vehicle_count ELSE NULL END > 1 | | | 04:HASH JOIN [INNER JOIN] | | hash predicates: ss.ss_sold_date_sk = d.d_date_sk @@ -1873,7 +1873,7 @@ LIMIT 100 | | | | | 02:SCAN HDFS [tpcds.household_demographics hd] | | partitions=1/1 size=148.10KB -| | predicates: (hd.hd_buy_potential = '>10000' OR hd.hd_buy_potential = 'unknown'), hd.hd_vehicle_count > 0, CASE WHEN hd.hd_vehicle_count > 0 THEN hd.hd_dep_count / hd.hd_vehicle_count ELSE NULL END > 1.0 +| | predicates: (hd.hd_buy_potential = '>10000' OR hd.hd_buy_potential = 'unknown'), hd.hd_vehicle_count > 0, CASE WHEN hd.hd_vehicle_count > 0 THEN hd.hd_dep_count / hd.hd_vehicle_count ELSE NULL END > 1 | | | 04:HASH JOIN [INNER JOIN, BROADCAST] | | hash predicates: ss.ss_sold_date_sk = d.d_date_sk diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test index 7c1746801..172fcbac5 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test @@ -20,7 +20,7 @@ group by l_linestatus ---- PLAN 01:AGGREGATE [FINALIZE] -| output: sum(l_quantity), sum(l_extendedprice), sum(l_extendedprice * (1.0 - l_discount)), sum(l_extendedprice * (1.0 - l_discount) * (1.0 + l_tax)), count(l_quantity), count(l_extendedprice), sum(l_discount), count(l_discount), count(1) +| output: sum(l_quantity), sum(l_extendedprice), sum(l_extendedprice * (1 - l_discount)), sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)), count(l_quantity), count(l_extendedprice), sum(l_discount), count(l_discount), count(1) | group by: l_returnflag, l_linestatus | 00:SCAN HDFS [tpch.lineitem] @@ -30,13 +30,13 @@ group by 04:EXCHANGE [PARTITION=UNPARTITIONED] | 03:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_quantity)), sum(sum(l_extendedprice)), sum(sum(l_extendedprice * (1.0 - l_discount))), sum(sum(l_extendedprice * (1.0 - l_discount) * (1.0 + l_tax))), sum(count(l_quantity)), sum(count(l_extendedprice)), sum(sum(l_discount)), sum(count(l_discount)), sum(count(1)) +| output: sum(sum(l_quantity)), sum(sum(l_extendedprice)), sum(sum(l_extendedprice * (1 - l_discount))), sum(sum(l_extendedprice * (1 - l_discount) * (1 + l_tax))), sum(count(l_quantity)), sum(count(l_extendedprice)), sum(sum(l_discount)), sum(count(l_discount)), sum(count(1)) | group by: l_returnflag, l_linestatus | 02:EXCHANGE [PARTITION=HASH(l_returnflag,l_linestatus)] | 01:AGGREGATE -| output: sum(l_quantity), sum(l_extendedprice), sum(l_extendedprice * (1.0 - l_discount)), sum(l_extendedprice * (1.0 - l_discount) * (1.0 + l_tax)), count(l_quantity), count(l_extendedprice), sum(l_discount), count(l_discount), count(1) +| output: sum(l_quantity), sum(l_extendedprice), sum(l_extendedprice * (1 - l_discount)), sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)), count(l_quantity), count(l_extendedprice), sum(l_discount), count(l_discount), count(1) | group by: l_returnflag, l_linestatus | 00:SCAN HDFS [tpch.lineitem] @@ -246,10 +246,10 @@ order by limit 10 ---- PLAN 06:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 05:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_orderkey, o_orderdate, o_shippriority | 04:HASH JOIN [INNER JOIN] @@ -271,21 +271,21 @@ limit 10 predicates: l_shipdate > '1995-03-15' ---- DISTRIBUTEDPLAN 12:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 11:EXCHANGE [PARTITION=UNPARTITIONED] | 06:TOP-N [LIMIT=10] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC, o_orderdate ASC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC, o_orderdate ASC | 10:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | group by: l_orderkey, o_orderdate, o_shippriority | 09:EXCHANGE [PARTITION=HASH(l_orderkey,o_orderdate,o_shippriority)] | 05:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_orderkey, o_orderdate, o_shippriority | 04:HASH JOIN [INNER JOIN, BROADCAST] @@ -412,10 +412,10 @@ order by limit 100 ---- PLAN 12:TOP-N [LIMIT=100] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 11:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: n_name | 10:HASH JOIN [INNER JOIN] @@ -454,21 +454,21 @@ limit 100 partitions=1/1 size=718.94MB ---- DISTRIBUTEDPLAN 21:TOP-N [LIMIT=100] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 20:EXCHANGE [PARTITION=UNPARTITIONED] | 12:TOP-N [LIMIT=100] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 19:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | group by: n_name | 18:EXCHANGE [PARTITION=HASH(n_name)] | 11:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: n_name | 10:HASH JOIN [INNER JOIN, BROADCAST] @@ -531,7 +531,7 @@ where l_shipdate >= '1994-01-01' and | 00:SCAN HDFS [tpch.lineitem] partitions=1/1 size=718.94MB - predicates: l_shipdate >= '1994-01-01', l_shipdate < '1995-01-01', l_discount >= 0.05, l_discount <= 0.07, l_quantity < 24.0 + predicates: l_shipdate >= '1994-01-01', l_shipdate < '1995-01-01', l_discount >= 0.05, l_discount <= 0.07, l_quantity < 24 ---- DISTRIBUTEDPLAN 03:AGGREGATE [MERGE FINALIZE] | output: sum(sum(l_extendedprice * l_discount)) @@ -543,7 +543,7 @@ where l_shipdate >= '1994-01-01' and | 00:SCAN HDFS [tpch.lineitem] partitions=1/1 size=718.94MB - predicates: l_shipdate >= '1994-01-01', l_shipdate < '1995-01-01', l_discount >= 0.05, l_discount <= 0.07, l_quantity < 24.0 + predicates: l_shipdate >= '1994-01-01', l_shipdate < '1995-01-01', l_discount >= 0.05, l_discount <= 0.07, l_quantity < 24 ==== # Q8 - National Market Share Query # Modifications: Got rid of subquery, converted select from multiple tables to joins, @@ -582,7 +582,7 @@ limit 100 | order by: year(o_orderdate) ASC | 15:AGGREGATE [FINALIZE] -| output: sum(CASE WHEN n2.n_name = 'BRAZIL' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END), sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(CASE WHEN n2.n_name = 'BRAZIL' THEN l_extendedprice * (1 - l_discount) ELSE 0 END), sum(l_extendedprice * (1 - l_discount)) | group by: year(o_orderdate) | 14:HASH JOIN [INNER JOIN] @@ -642,13 +642,13 @@ limit 100 | order by: year(o_orderdate) ASC | 25:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(CASE WHEN n2.n_name = 'BRAZIL' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END)), sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(CASE WHEN n2.n_name = 'BRAZIL' THEN l_extendedprice * (1 - l_discount) ELSE 0 END)), sum(sum(l_extendedprice * (1 - l_discount))) | group by: year(o_orderdate) | 24:EXCHANGE [PARTITION=HASH(year(o_orderdate))] | 15:AGGREGATE -| output: sum(CASE WHEN n2.n_name = 'BRAZIL' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END), sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(CASE WHEN n2.n_name = 'BRAZIL' THEN l_extendedprice * (1 - l_discount) ELSE 0 END), sum(l_extendedprice * (1 - l_discount)) | group by: year(o_orderdate) | 14:HASH JOIN [INNER JOIN, BROADCAST] @@ -746,7 +746,7 @@ limit 200 | order by: n.n_name ASC, year(o.o_orderdate) DESC | 11:AGGREGATE [FINALIZE] -| output: sum(l.l_extendedprice * (1.0 - l.l_discount) - ps.ps_supplycost * l.l_quantity) +| output: sum(l.l_extendedprice * (1 - l.l_discount) - ps.ps_supplycost * l.l_quantity) | group by: n.n_name, year(o.o_orderdate) | 10:HASH JOIN [INNER JOIN] @@ -792,13 +792,13 @@ limit 200 | order by: n.n_name ASC, year(o.o_orderdate) DESC | 19:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l.l_extendedprice * (1.0 - l.l_discount) - ps.ps_supplycost * l.l_quantity)) +| output: sum(sum(l.l_extendedprice * (1 - l.l_discount) - ps.ps_supplycost * l.l_quantity)) | group by: n.n_name, year(o.o_orderdate) | 18:EXCHANGE [PARTITION=HASH(n.n_name,year(o.o_orderdate))] | 11:AGGREGATE -| output: sum(l.l_extendedprice * (1.0 - l.l_discount) - ps.ps_supplycost * l.l_quantity) +| output: sum(l.l_extendedprice * (1 - l.l_discount) - ps.ps_supplycost * l.l_quantity) | group by: n.n_name, year(o.o_orderdate) | 10:HASH JOIN [INNER JOIN, BROADCAST] @@ -882,10 +882,10 @@ order by limit 20 ---- PLAN 08:TOP-N [LIMIT=20] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 07:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: c_custkey, c_name, CAST(c_acctbal AS BIGINT), c_phone, n_name, c_address, c_comment | 06:HASH JOIN [INNER JOIN] @@ -912,21 +912,21 @@ limit 20 predicates: l.l_returnflag = 'R' ---- DISTRIBUTEDPLAN 15:TOP-N [LIMIT=20] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 14:EXCHANGE [PARTITION=UNPARTITIONED] | 08:TOP-N [LIMIT=20] -| order by: round(sum(l_extendedprice * (1.0 - l_discount)), 5) DESC +| order by: round(sum(l_extendedprice * (1 - l_discount)), 5) DESC | 13:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | group by: c_custkey, c_name, CAST(c_acctbal AS BIGINT), c_phone, n_name, c_address, c_comment | 12:EXCHANGE [PARTITION=HASH(c_custkey,c_name,CAST(c_acctbal AS BIGINT),c_phone,n_name,c_address,c_comment)] | 07:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: c_custkey, c_name, CAST(c_acctbal AS BIGINT), c_phone, n_name, c_address, c_comment | 06:HASH JOIN [INNER JOIN, BROADCAST] @@ -1228,7 +1228,7 @@ join tpch.part p l.l_shipdate < '1995-10-01' ---- PLAN 03:AGGREGATE [FINALIZE] -| output: sum(CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END), sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1 - l_discount) ELSE 0.0 END), sum(l_extendedprice * (1 - l_discount)) | 02:HASH JOIN [INNER JOIN] | hash predicates: p.p_partkey = l.l_partkey @@ -1241,12 +1241,12 @@ join tpch.part p partitions=1/1 size=22.83MB ---- DISTRIBUTEDPLAN 06:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END)), sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1 - l_discount) ELSE 0.0 END)), sum(sum(l_extendedprice * (1 - l_discount))) | 05:EXCHANGE [PARTITION=UNPARTITIONED] | 03:AGGREGATE -| output: sum(CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END), sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1 - l_discount) ELSE 0.0 END), sum(l_extendedprice * (1 - l_discount)) | 02:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: p.p_partkey = l.l_partkey @@ -1273,7 +1273,7 @@ WRITE TO HDFS [tpch.revenue, OVERWRITE=true] | partitions=1 | 01:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_suppkey | 00:SCAN HDFS [tpch.lineitem] @@ -1284,13 +1284,13 @@ WRITE TO HDFS [tpch.revenue, OVERWRITE=true] | partitions=1 | 03:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | group by: l_suppkey | 02:EXCHANGE [PARTITION=HASH(l_suppkey)] | 01:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | group by: l_suppkey | 00:SCAN HDFS [tpch.lineitem] @@ -1672,7 +1672,7 @@ limit 100 | |--03:SCAN HDFS [tpch.q18_tmp t] | partitions=1/1 size=16.13MB compact -| predicates: t.t_sum_quantity > 300.0 +| predicates: t.t_sum_quantity > 300 | 05:HASH JOIN [INNER JOIN] | hash predicates: o.o_custkey = c.c_custkey @@ -1714,7 +1714,7 @@ limit 100 | | | 03:SCAN HDFS [tpch.q18_tmp t] | partitions=1/1 size=16.13MB -| predicates: t.t_sum_quantity > 300.0 +| predicates: t.t_sum_quantity > 300 | 05:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: o.o_custkey = c.c_custkey @@ -1784,11 +1784,11 @@ or ) ---- PLAN 03:AGGREGATE [FINALIZE] -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | 02:HASH JOIN [INNER JOIN] | hash predicates: l.l_partkey = p.p_partkey -| other predicates: (p_brand = 'Brand#12' AND (p_container LIKE 'SM CASE' OR p_container LIKE 'SM BOX' OR p_container LIKE 'SM PACK' OR p_container LIKE 'SM PKG') AND l_quantity >= 1.0 AND l_quantity <= 11.0 AND p_size >= 1 AND p_size <= 5 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#23' AND (p_container LIKE 'MED BAG' OR p_container LIKE 'MED BOX' OR p_container LIKE 'MED PKG' OR p_container LIKE 'MED PACK') AND l_quantity >= 10.0 AND l_quantity <= 20.0 AND p_size >= 1 AND p_size <= 10 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#34' AND (p_container LIKE 'LG BAG' OR p_container LIKE 'LG BOX' OR p_container LIKE 'LG PKG' OR p_container LIKE 'LG PACK') AND l_quantity >= 20.0 AND l_quantity <= 30.0 AND p_size >= 1 AND p_size <= 15 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') +| other predicates: (p_brand = 'Brand#12' AND (p_container LIKE 'SM CASE' OR p_container LIKE 'SM BOX' OR p_container LIKE 'SM PACK' OR p_container LIKE 'SM PKG') AND l_quantity >= 1 AND l_quantity <= 11 AND p_size >= 1 AND p_size <= 5 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#23' AND (p_container LIKE 'MED BAG' OR p_container LIKE 'MED BOX' OR p_container LIKE 'MED PKG' OR p_container LIKE 'MED PACK') AND l_quantity >= 10 AND l_quantity <= 20 AND p_size >= 1 AND p_size <= 10 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#34' AND (p_container LIKE 'LG BAG' OR p_container LIKE 'LG BOX' OR p_container LIKE 'LG PKG' OR p_container LIKE 'LG PACK') AND l_quantity >= 20 AND l_quantity <= 30 AND p_size >= 1 AND p_size <= 15 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') | |--01:SCAN HDFS [tpch.part p] | partitions=1/1 size=22.83MB compact @@ -1797,16 +1797,16 @@ or partitions=1/1 size=718.94MB ---- DISTRIBUTEDPLAN 06:AGGREGATE [MERGE FINALIZE] -| output: sum(sum(l_extendedprice * (1.0 - l_discount))) +| output: sum(sum(l_extendedprice * (1 - l_discount))) | 05:EXCHANGE [PARTITION=UNPARTITIONED] | 03:AGGREGATE -| output: sum(l_extendedprice * (1.0 - l_discount)) +| output: sum(l_extendedprice * (1 - l_discount)) | 02:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: l.l_partkey = p.p_partkey -| other predicates: (p_brand = 'Brand#12' AND (p_container LIKE 'SM CASE' OR p_container LIKE 'SM BOX' OR p_container LIKE 'SM PACK' OR p_container LIKE 'SM PKG') AND l_quantity >= 1.0 AND l_quantity <= 11.0 AND p_size >= 1 AND p_size <= 5 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#23' AND (p_container LIKE 'MED BAG' OR p_container LIKE 'MED BOX' OR p_container LIKE 'MED PKG' OR p_container LIKE 'MED PACK') AND l_quantity >= 10.0 AND l_quantity <= 20.0 AND p_size >= 1 AND p_size <= 10 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#34' AND (p_container LIKE 'LG BAG' OR p_container LIKE 'LG BOX' OR p_container LIKE 'LG PKG' OR p_container LIKE 'LG PACK') AND l_quantity >= 20.0 AND l_quantity <= 30.0 AND p_size >= 1 AND p_size <= 15 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') +| other predicates: (p_brand = 'Brand#12' AND (p_container LIKE 'SM CASE' OR p_container LIKE 'SM BOX' OR p_container LIKE 'SM PACK' OR p_container LIKE 'SM PKG') AND l_quantity >= 1 AND l_quantity <= 11 AND p_size >= 1 AND p_size <= 5 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#23' AND (p_container LIKE 'MED BAG' OR p_container LIKE 'MED BOX' OR p_container LIKE 'MED PKG' OR p_container LIKE 'MED PACK') AND l_quantity >= 10 AND l_quantity <= 20 AND p_size >= 1 AND p_size <= 10 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') OR (p_brand = 'Brand#34' AND (p_container LIKE 'LG BAG' OR p_container LIKE 'LG BOX' OR p_container LIKE 'LG PKG' OR p_container LIKE 'LG PACK') AND l_quantity >= 20 AND l_quantity <= 30 AND p_size >= 1 AND p_size <= 15 AND (l_shipmode LIKE 'AIR' OR l_shipmode LIKE 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON') | |--04:EXCHANGE [BROADCAST] | | @@ -2215,7 +2215,7 @@ WRITE TO HDFS [tpch.q22_customer_tmp1, OVERWRITE=true] | 00:SCAN HDFS [tpch.customer c] partitions=1/1 size=23.08MB - predicates: c.c_acctbal > 0.0, (substr(c.c_phone, 1, 2) = '13' OR substr(c.c_phone, 1, 2) = '31' OR substr(c.c_phone, 1, 2) = '23' OR substr(c.c_phone, 1, 2) = '29' OR substr(c.c_phone, 1, 2) = '30' OR substr(c.c_phone, 1, 2) = '18' OR substr(c.c_phone, 1, 2) = '17') + predicates: c.c_acctbal > 0.00, (substr(c.c_phone, 1, 2) = '13' OR substr(c.c_phone, 1, 2) = '31' OR substr(c.c_phone, 1, 2) = '23' OR substr(c.c_phone, 1, 2) = '29' OR substr(c.c_phone, 1, 2) = '30' OR substr(c.c_phone, 1, 2) = '18' OR substr(c.c_phone, 1, 2) = '17') ---- DISTRIBUTEDPLAN WRITE TO HDFS [tpch.q22_customer_tmp1, OVERWRITE=true] | partitions=1 @@ -2232,7 +2232,7 @@ WRITE TO HDFS [tpch.q22_customer_tmp1, OVERWRITE=true] | 00:SCAN HDFS [tpch.customer c] partitions=1/1 size=23.08MB - predicates: c.c_acctbal > 0.0, (substr(c.c_phone, 1, 2) = '13' OR substr(c.c_phone, 1, 2) = '31' OR substr(c.c_phone, 1, 2) = '23' OR substr(c.c_phone, 1, 2) = '29' OR substr(c.c_phone, 1, 2) = '30' OR substr(c.c_phone, 1, 2) = '18' OR substr(c.c_phone, 1, 2) = '17') + predicates: c.c_acctbal > 0.00, (substr(c.c_phone, 1, 2) = '13' OR substr(c.c_phone, 1, 2) = '31' OR substr(c.c_phone, 1, 2) = '23' OR substr(c.c_phone, 1, 2) = '29' OR substr(c.c_phone, 1, 2) = '30' OR substr(c.c_phone, 1, 2) = '18' OR substr(c.c_phone, 1, 2) = '17') ==== # QUERY_NAME : TPCH-Q22_QUERY_2 # Modifications: Updated to use LEFT OUTER JOIN instead of NOT EXISTS, diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/views.test b/testdata/workloads/functional-planner/queries/PlannerTest/views.test index 14f84ab5b..16155bf05 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/views.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/views.test @@ -397,8 +397,8 @@ select * from functional.alltypes_parens select bool_col FROM ( SELECT bool_col FROM functional.alltypes t ) t WHERE t.bool_col ---- PLAN 00:SCAN HDFS [functional.alltypes t] - partitions=24/24 size=478.45KB - predicates: bool_col + partitions=24/24 size=478.45KB + predicates: bool_col ---- DISTRIBUTEDPLAN 01:EXCHANGE [PARTITION=UNPARTITIONED] | diff --git a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test index ef5fd4346..a30458abf 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test +++ b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test @@ -989,7 +989,7 @@ int,boolean,int ==== ---- QUERY # Regression test for min/max of all negative values. IMPALA-869. -select min(-1.0), max(-1.0) from tinytable +select min(cast(-1.0 as float)), max(cast(-1.0 as float)) from tinytable ---- TYPES float,float ---- RESULTS diff --git a/testdata/workloads/functional-query/queries/QueryTest/decimal.test b/testdata/workloads/functional-query/queries/QueryTest/decimal.test index 9a1e38345..4777430ea 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/decimal.test +++ b/testdata/workloads/functional-query/queries/QueryTest/decimal.test @@ -42,6 +42,20 @@ select sum(d1), sum(d2), sum(d3), sum(d4), sum(d5) from decimal_tbl decimal,decimal,decimal,decimal,decimal ==== ---- QUERY +select count(*), sum(d1), sum(d2), sum(d3), sum(d4), sum(d5) from decimal_tbl where d6 = 1 +---- RESULTS +5,161111,3332,13717.2838257900,0.61728394500000000000000000000000000000,12361.02889 +---- TYPES +bigint,decimal,decimal,decimal,decimal,decimal +==== +---- QUERY +select count(*), sum(d1), sum(d2), sum(d3), sum(d4), sum(d5) from decimal_tbl where d6 = 0 +---- RESULTS +0,NULL,NULL,NULL,NULL,NULL +---- TYPES +bigint,decimal,decimal,decimal,decimal,decimal +==== +---- QUERY select c3, count(*) from decimal_tiny group by c3 ---- RESULTS 0.3,10 diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test index 8e71078f8..da020f89e 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test @@ -236,7 +236,7 @@ bigint 9980 ==== ---- QUERY -select count(*) from alltypesagg where float_col > cast(1.1 as float) +select count(*) from alltypesagg where float_col > cast(1.1 as float) ---- TYPES bigint ---- RESULTS @@ -337,7 +337,7 @@ bigint ==== ---- QUERY # Type synonym check: DOUBLE = REAL -select count(*) from alltypesagg where double_col >= 20.2 and cast(double_col as double) = cast(double_col as real) +select count(*) from alltypesagg where double_col >= 20.2 and cast(double_col as double) = cast(double_col as real) ---- TYPES bigint ---- RESULTS @@ -373,7 +373,7 @@ bigint #540000,540000,5535000,5535000,55485000,55485000,60979499.99976754,60979499.99976754,111428999.9997675,111428999.9997676 #==== ## - -#select +#select #-1 * SUM(tinyint_col) - SUM(smallint_col), #SUM(-1 * tinyint_col - smallint_col), #-1 * SUM(tinyint_col) - SUM(smallint_col) - SUM(int_col), @@ -825,7 +825,7 @@ bigint ==== ---- QUERY select count(*) from alltypesagg -where 1 not in (tinyint_col, smallint_col, int_col, bigint_col) +where 1 not in (tinyint_col, smallint_col, int_col, bigint_col) ---- TYPES bigint ---- RESULTS @@ -833,7 +833,7 @@ bigint ==== ---- QUERY select count(*) from alltypesagg -where 10.1 in (tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col) +where 10.1 in (tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col) ---- TYPES bigint ---- RESULTS @@ -841,7 +841,7 @@ bigint ==== ---- QUERY select count(*) from alltypesagg -where 10.1 not in (tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col) +where 10.1 not in (tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col) ---- TYPES bigint ---- RESULTS @@ -849,7 +849,7 @@ bigint ==== ---- QUERY select count(*) from alltypesagg -where '01/01/10' in (date_string_col, string_col, 'abc') +where '01/01/10' in (date_string_col, string_col, 'abc') ---- TYPES bigint ---- RESULTS @@ -864,7 +864,7 @@ bigint 9000 ==== ---- QUERY -select count(*) from alltypesagg +select count(*) from alltypesagg where cast('2010-01-01 00:00:00' as timestamp) in (timestamp_col) ---- TYPES bigint @@ -872,7 +872,7 @@ bigint 1 ==== ---- QUERY -select count(*) from alltypesagg +select count(*) from alltypesagg where cast('2010-01-01 00:00:00' as timestamp) not in (timestamp_col) ---- TYPES bigint @@ -984,7 +984,7 @@ bigint ==== ---- QUERY select count(*) from alltypesagg where timestamp_col -between cast('2010-01-01 00:00:00' as timestamp) +between cast('2010-01-01 00:00:00' as timestamp) and cast('2010-01-01 01:40:00' as timestamp) ---- TYPES bigint @@ -993,7 +993,7 @@ bigint ==== ---- QUERY select count(*) from alltypesagg where timestamp_col -not between cast('2010-01-01 00:00:00' as timestamp) +not between cast('2010-01-01 00:00:00' as timestamp) and cast('2010-01-01 01:40:00' as timestamp) ---- TYPES bigint @@ -1018,7 +1018,7 @@ select now() ---- TYPES timestamp ---- RESULTS -# Matches a single date of the form 'yyyy-MM-dd HH:mm:ss' +# Matches a single date of the form 'yyyy-MM-dd HH:mm:ss' # or 'yyyy-MM-dd HH:mm:ss.SSSSSS' row_regex: \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}(\.\d{9})? ==== @@ -1105,7 +1105,7 @@ timestamp,timestamp,timestamp,timestamp ==== ---- QUERY # Add/sub weeks, non-function-call like version. -select timestamp_col + interval 2 weeks, +select timestamp_col + interval 2 weeks, timestamp_col + interval 53 weeks, interval 2 weeks + timestamp_col, interval 53 weeks + timestamp_col, @@ -1119,7 +1119,7 @@ timestamp,timestamp,timestamp,timestamp,timestamp,timestamp ==== ---- QUERY # Add/sub weeks, function-call like version. -select date_add(timestamp_col, interval 2 weeks), +select date_add(timestamp_col, interval 2 weeks), date_add(timestamp_col, interval 53 weeks), date_sub(timestamp_col, interval 2 weeks), date_sub(timestamp_col, interval 53 weeks) @@ -1131,9 +1131,9 @@ timestamp,timestamp,timestamp,timestamp ==== ---- QUERY # Add/sub days, non-function-call like version. -select timestamp_col + interval 10 days, +select timestamp_col + interval 10 days, timestamp_col + interval 10 days, - interval 10 days + timestamp_col, + interval 10 days + timestamp_col, interval 10 days + timestamp_col, timestamp_col - interval 10 days, timestamp_col - interval 365 days @@ -1284,7 +1284,7 @@ timestamp,timestamp ---- QUERY # Chaining of arithmetic operations (only non-function-call like version). select timestamp_col + interval 10 years + interval 2 months + interval 5 days, - interval 10 years + timestamp_col + interval 2 months + interval 5 days, + interval 10 years + timestamp_col + interval 2 months + interval 5 days, timestamp_col + interval 10 years - interval 2 months + interval 5 days, interval 10 years + timestamp_col - interval 2 months + interval 5 days, timestamp_col - interval 10 years - interval 2 months - interval 5 days @@ -1363,14 +1363,23 @@ bigint ---- RESULTS 1000 ==== ----- QUERY +#---- QUERY # Test rounding (IMPALA-266) -select round(8.072, 3), round(8, 3), round(8.0719999999,3), round(8.072, 0), -round(8.072, 4), 8, 8.0; +# Disabled due to IMPALA-1018 +#select round(8.072, 3), round(8, 3), round(8.0719999999,3), round(8.072, 0), +#round(8.072, 4), 8, 8.0; +#---- TYPES +#double, double, double, double, double, tinyint, decimal +#---- RESULTS +#8.072,8.000,8.072,8,8.0720,8,8 +#==== +---- QUERY +select round(float_col, 3), round(float_col, 4) from functional.alltypestiny limit 2; ---- TYPES -double, double, double, double, double, tinyint, float +double,double ---- RESULTS -8.072,8.000,8.072,8,8.0720,8,8 +0.000,0.0000 +1.100,1.1000 ==== ---- QUERY # Test a fix of codegen/non-codegen exprs(IMPALA-350) @@ -1433,3 +1442,40 @@ string '01:01:01 02/1971/02' '1972||Mar||03||030303' ==== +---- QUERY +select 1.1 * 1.1 + cast(1.1 as float) +---- TYPES +double +---- RESULTS +2.310000023841858 +==== +---- QUERY +select 1.1 * 1.1 + cast(1.1 as decimal(2,1)) +---- TYPES +decimal +---- RESULTS +2.31 +==== +---- QUERY +select 1.1 * 1.1 + 1.1 +---- TYPES +decimal +---- RESULTS +2.31 +==== +---- QUERY +select 1.1 * 1.1 + float_col from functional.alltypestiny limit 2; +---- TYPES +double +---- RESULTS +1.21 +2.310000023841858 +==== +---- QUERY +select 1.1 * 1.1 + c3 from functional.decimal_tiny limit 2; +---- TYPES +decimal +---- RESULTS +1.21 +1.31 +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/udf.test b/testdata/workloads/functional-query/queries/QueryTest/udf.test index 50397ad16..c82a29905 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/udf.test +++ b/testdata/workloads/functional-query/queries/QueryTest/udf.test @@ -76,16 +76,6 @@ decimal 1.0 ==== ---- QUERY -select identity(cast(1 as decimal(20,1))); ----- TYPES -decimal ----- RESULTS -# The identity UDF has signature decimal(38,10) -> decimal(38,10), so the input is -# promoted and the result has scale 10. The previous decimal test cases are not promoted -# because there are identity UDFs that exactly match the input. -1.0000000000 -==== ----- QUERY select identity(cast(1 as decimal(38,10))); ---- TYPES decimal @@ -146,7 +136,9 @@ string 'eeeeeeee' ==== ---- QUERY -select identity(d1), identity(d3), identity(d5) from functional.decimal_tbl; +select identity(d1), +identity(cast(d3 as decimal(38,10))), identity(cast(d5 as decimal(38,10))) +from functional.decimal_tbl; ---- TYPES decimal,decimal,decimal ---- RESULTS @@ -343,7 +335,7 @@ int 16 ==== ---- QUERY -select var_sum(cast(1 as decimal(1,0)), cast(2 as decimal(1,0)), cast(3 as decimal(1,0))); +select var_sum(cast(1 as decimal(4,2)), cast(2 as decimal(4,2)), cast(3 as decimal(4,2))); ---- TYPES decimal ---- RESULTS @@ -352,8 +344,8 @@ decimal ---- QUERY # More complicated arguments select native_function_test.var_sum( - cast(1 as decimal(1,0)), cast(2 as decimal(2,1)), - cast(3 as decimal(2,1)) + cast("1.1" as decimal(2,1))); + cast(1 as decimal(4,2)), cast(2 as decimal(4,2)), + cast(3 as decimal(3,2)) + cast("1.1" as decimal(3,2))); ---- TYPES decimal ---- RESULTS diff --git a/testdata/workloads/functional-query/queries/QueryTest/union.test b/testdata/workloads/functional-query/queries/QueryTest/union.test index 0bcea0e38..a18c17810 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/union.test +++ b/testdata/workloads/functional-query/queries/QueryTest/union.test @@ -84,7 +84,7 @@ select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, union all select 1,false,1,1,1,10,1.1,10.1,'01/01/09','1',cast('2009-01-01 00:01:00' as timestamp), 2009,1 ---- TYPES -int, boolean, tinyint, smallint, int, bigint, double, double, string, string, timestamp, int, int +int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int ---- RESULTS: VERIFY_IS_EQUAL_SORTED 0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1 0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1 @@ -226,13 +226,13 @@ int, boolean, tinyint, smallint, int, bigint, float, double, string, string, tim 3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2 ==== ---- QUERY -# Union unnesting: UNION ALL doesn't absorb nested union with DISTINCT, +# Union unnesting: UNION ALL doesn't absorb nested union with DISTINCT, # first operand is nested (select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 union distinct select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=2) union all -select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 +select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 ---- TYPES int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int ---- RESULTS: VERIFY_IS_EQUAL_SORTED @@ -262,7 +262,7 @@ int, boolean, tinyint, smallint, int, bigint, float, double, string, string, tim 3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2 ==== ---- QUERY -# Union unnesting: UNION ALL absorbs the children but not directly the operands +# Union unnesting: UNION ALL absorbs the children but not directly the operands # of a nested union with mixed ALL/DISTINCT, first operand is nested (select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 union distinct @@ -284,7 +284,7 @@ int, boolean, tinyint, smallint, int, bigint, float, double, string, string, tim 3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2 ==== ---- QUERY -# Union unnesting: UNION ALL absorbs the children but not directly the operands +# Union unnesting: UNION ALL absorbs the children but not directly the operands # of a nested union with mixed ALL/DISTINCT, second operand is nested select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 union all @@ -306,7 +306,7 @@ int, boolean, tinyint, smallint, int, bigint, float, double, string, string, tim 3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2 ==== ---- QUERY -# Union unnesting: UNION ALL doesn't absorb the children of a nested union +# Union unnesting: UNION ALL doesn't absorb the children of a nested union # with mixed ALL/DISTINCT and limit, second operand is nested select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 union all @@ -336,7 +336,7 @@ int, boolean, tinyint, smallint, int, bigint, float, double, string, string, tim (select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=2) order by 1 limit 3) union all -select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 +select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 ---- TYPES int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int ---- RESULTS: VERIFY_IS_EQUAL_SORTED @@ -440,7 +440,7 @@ int, boolean, tinyint, smallint, int, bigint, float, double, string, string, tim (select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=2) order by 1 limit 3) union distinct -select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 +select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 ---- TYPES int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int ---- RESULTS: VERIFY_IS_EQUAL_SORTED @@ -456,7 +456,7 @@ int, boolean, tinyint, smallint, int, bigint, float, double, string, string, tim (select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=2) order by 1 limit 3) union distinct -select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 +select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypestiny where year=2009 and month=1 ---- TYPES int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int ---- RESULTS: VERIFY_IS_EQUAL_SORTED @@ -678,11 +678,11 @@ int, double, string ==== ---- QUERY # Test UNION ALL on large tables with a few constant selects to excercise backend logic. -select count(*) from ( +select count(*) from ( select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypes union all select 0,true,0,0,0,0,cast(0 as float),0,'01/01/09','0',cast('2009-01-01 00:00:00' as timestamp),2009,1 - union all + union all select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypes union all select 1,false,1,1,1,10,cast(1.1 as float),10.1,'01/01/09','1',cast('2009-01-01 00:01:00' as timestamp),2009,1 @@ -696,11 +696,11 @@ bigint ==== ---- QUERY # Test UNION DISTINCT on large tables with a few constant selects to excercise backend logic. -select count(*) from ( +select count(*) from ( select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypes union distinct select 0,true,0,0,0,0,cast(0 as float),0,'01/01/09','0',cast('2009-01-01 00:00:00' as timestamp),2009,1 - union distinct + union distinct select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypes union distinct select 1,false,1,1,1,10,cast(1.1 as float),10.1,'01/01/09','1',cast('2009-01-01 00:01:00' as timestamp),2009,1 @@ -714,16 +714,16 @@ bigint ==== ---- QUERY # Test UNION DISTINCT on large table constant selects and values statements -select count(*) from ( +select count(*) from ( select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypes union distinct values(0,true,0,0,0,0,cast(0 as float),0,'01/01/09','0',cast('2009-01-01 00:00:00' as timestamp),2009,1) - union distinct + union distinct select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month from alltypes union distinct select 1,false,1,1,1,10,cast(1.1 as float),10.1,'01/01/09','1',cast('2009-01-01 00:01:00' as timestamp),2009,1 union distinct - values(2,true,2,2,2,20,cast(2.2 as float),20.2,'01/01/09','2',cast('2009-01-01 00:02:00.10' as timestamp),2009,1) + values(2,true,2,2,2,20,cast(2.2 as float),cast(20.2 as double),'01/01/09','2',cast('2009-01-01 00:02:00.10' as timestamp),2009,1) ) x ---- TYPES bigint @@ -732,13 +732,13 @@ bigint ==== ---- QUERY # UNION ALL with only constant selects -select 1, 'a', NULL, 10.0f +select 1, 'a', NULL, 10.0 union all -select 2, 'b', NULL, 20.0f +select 2, 'b', NULL, 20.0 union all -select 3, 'c', NULL, 30.0f +select 3, 'c', NULL, 30.0 ---- TYPES -tinyint, string, null, float +tinyint, string, null, decimal ---- RESULTS: VERIFY_IS_EQUAL_SORTED 1,'a',NULL,10.0 2,'b',NULL,20.0 @@ -746,26 +746,26 @@ tinyint, string, null, float ==== ---- QUERY # UNION DISTINCT with only constant selects -select 1, 'a', NULL, 10.0f +select 1, 'a', NULL, 10.0 union distinct -select 2, 'b', NULL, 20.0f +select 2, 'b', NULL, 20.0 union distinct -select 1, 'a', NULL, 10.0f +select 1, 'a', NULL, 10.0 ---- TYPES -tinyint, string, null, float +tinyint, string, null, decimal ---- RESULTS: VERIFY_IS_EQUAL_SORTED 1,'a',NULL,10.0 2,'b',NULL,20.0 ==== ---- QUERY # UNION ALL with values statements -values(1, 'a', NULL, 10.0f) +values(1, 'a', NULL, 10.0) union all -values(2, 'b', NULL, 20.0f) +values(2, 'b', NULL, 20.0) union all -values(3, 'c', NULL, 30.0f) +values(3, 'c', NULL, 30.0) ---- TYPES -tinyint, string, null, float +tinyint, string, null, decimal ---- RESULTS: VERIFY_IS_EQUAL_SORTED 1,'a',NULL,10.0 2,'b',NULL,20.0 @@ -773,13 +773,13 @@ tinyint, string, null, float ==== ---- QUERY # UNION DISTINCT with values statements -values(1, 'a', NULL, 10.0f) +values(1, 'a', NULL, 10.0) union distinct -values(2, 'b', NULL, 20.0f) +values(2, 'b', NULL, 20.0) union distinct -values(1, 'a', NULL, 10.0f) +values(1, 'a', NULL, 10.0) ---- TYPES -tinyint, string, null, float +tinyint, string, null, decimal ---- RESULTS: VERIFY_IS_EQUAL_SORTED 1,'a',NULL,10.0 2,'b',NULL,20.0 diff --git a/testdata/workloads/functional-query/queries/QueryTest/values.test b/testdata/workloads/functional-query/queries/QueryTest/values.test index d6e73b553..39b57f29c 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/values.test +++ b/testdata/workloads/functional-query/queries/QueryTest/values.test @@ -2,31 +2,31 @@ ---- QUERY values(1, 2+1, 1.0, 5.0 + 1.0, 'a') ---- RESULTS -1,3,1,6,'a' +1,3,1.0,6.0,'a' ---- TYPES -TINYINT, SMALLINT, FLOAT, DOUBLE, STRING +TINYINT, SMALLINT, DECIMAL, DECIMAL, STRING ==== ---- QUERY values(1+1, 2, 5.0, 'a') order by 1 limit 10 ---- RESULTS -2,2,5,'a' +2,2,5.0,'a' ---- TYPES -SMALLINT, TINYINT, FLOAT, STRING +SMALLINT, TINYINT, DECIMAL, STRING ==== ---- QUERY values((1+8, 2, 5.0, 'a'), (2, 3, 6.0, 'b'), (3, 4, 7.0, 'c')) ---- RESULTS -9,2,5,'a' -2,3,6,'b' -3,4,7,'c' +9,2,5.0,'a' +2,3,6.0,'b' +3,4,7.0,'c' ---- TYPES -SMALLINT, TINYINT, FLOAT, STRING +SMALLINT, TINYINT, DECIMAL, STRING ==== ---- QUERY values((1+8, 2, 5.0, 'a'), (2, 3, 6.0, 'b'), (3, 4, 7.0, 'c')) order by 1 desc limit 2 ---- RESULTS -9,2,5,'a' -3,4,7,'c' +9,2,5.0,'a' +3,4,7.0,'c' ---- TYPES -SMALLINT, TINYINT, FLOAT, STRING -==== \ No newline at end of file +SMALLINT, TINYINT, DECIMAL, STRING +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/views-ddl.test b/testdata/workloads/functional-query/queries/QueryTest/views-ddl.test index 66ef4e92e..b1befcc7f 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/views-ddl.test +++ b/testdata/workloads/functional-query/queries/QueryTest/views-ddl.test @@ -12,7 +12,7 @@ select * from functional.alltypesagg ---- RESULTS ==== ---- QUERY -# Create another simple view with 'if not exists' on a subset of +# Create another simple view with 'if not exists' on a subset of # alltypes' columns using custom column names and comments create view if not exists ddl_test_db.simple_view_sub (x, y comment 'hello', z) as @@ -207,13 +207,13 @@ bigint,bigint ---- QUERY # Create a view on a constant select and try to query it. create view ddl_test_db.const_view -as select 1, 'a', 10.0 +as select 1, 'a', cast(10.0 as float) ---- RESULTS ==== ---- QUERY select * from ddl_test_db.const_view ---- RESULTS -1,'a',10.0 +1,'a',10 ---- TYPES tinyint,string,float ====