jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-25 02:03:09 -05:00
Code Issues Projects Releases Wiki Activity

Added the remaining math functions supported in Hive.

Browse Source
This commit is contained in:
Alexander Behm
2012-02-16 22:30:14 -08:00
parent a780ffdcb3
commit bacee81c5a
8 changed files with 875 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
be/src/exprs/expr-test.cc
View File

@@ -1,6 +1,7 @@
// Copyright (c) 2011 Cloudera, Inc. All rights reserved.
#include <string>
#include <math.h>
#include <boost/unordered_map.hpp>
#include <gtest/gtest.h>
#include <boost/lexical_cast.hpp>
@@ -23,7 +24,6 @@
#include "exprs/null-literal.h"
#include "exprs/string-literal.h"
using namespace std;
using namespace boost;
@@ -640,9 +640,224 @@ TEST_F(ExprTest, StringFunctions) {
// e.g. TestValue("length(NULL)", TYPE_INT, NULL);
}
TEST_F(ExprTest, MathTrigonometricFunctions) {
// It is important to calculate the expected values
// using math functions, and not simply use constants.
// Otherwise, floating point imprecisions may lead to failed tests.
TestValue("sin(0.0)", TYPE_DOUBLE, sin(0.0));
TestValue("sin(pi())", TYPE_DOUBLE, sin(M_PI));
TestValue("sin(pi() / 2.0)", TYPE_DOUBLE, sin(M_PI / 2.0));
TestValue("asin(-1.0)", TYPE_DOUBLE, asin(-1.0));
TestValue("asin(1.0)", TYPE_DOUBLE, asin(1.0));
TestValue("cos(0.0)", TYPE_DOUBLE, cos(0.0));
TestValue("cos(pi())", TYPE_DOUBLE, cos(M_PI));
TestValue("acos(-1.0)", TYPE_DOUBLE, acos(-1.0));
TestValue("acos(1.0)", TYPE_DOUBLE, acos(1.0));
TestValue("tan(pi() * -1.0)", TYPE_DOUBLE, tan(M_PI * -1.0));
TestValue("tan(pi())", TYPE_DOUBLE, tan(M_PI));
TestValue("atan(pi())", TYPE_DOUBLE, atan(M_PI));
TestValue("atan(pi() * - 1.0)", TYPE_DOUBLE, atan(M_PI * -1.0));
TestValue("radians(0)", TYPE_DOUBLE, 0);
TestValue("radians(180.0)", TYPE_DOUBLE, M_PI);
TestValue("degrees(0)", TYPE_DOUBLE, 0.0);
TestValue("degrees(pi())", TYPE_DOUBLE, 180.0);
// TODO: tests with NULL arguments, currently we can't parse them inside function calls.
}
TEST_F(ExprTest, MathConversionFunctions) {
TestStringValue("bin(0)", "0");
TestStringValue("bin(1)", "1");
TestStringValue("bin(12)", "1100");
TestStringValue("bin(1234567)", "100101101011010000111");
TestStringValue("bin(" + lexical_cast<string>(numeric_limits<int64_t>::max()) + ")",
"111111111111111111111111111111111111111111111111111111111111111");
TestStringValue("bin(" + lexical_cast<string>(numeric_limits<int64_t>::min()+1) + ")",
"1000000000000000000000000000000000000000000000000000000000000001");
TestStringValue("hex(0)", "0");
TestStringValue("hex(15)", "F");
TestStringValue("hex(16)", "10");
TestStringValue("hex(" + lexical_cast<string>(numeric_limits<int64_t>::max()) + ")",
"7FFFFFFFFFFFFFFF");
TestStringValue("hex(" + lexical_cast<string>(numeric_limits<int64_t>::min()+1) + ")",
"8000000000000001");
TestStringValue("hex('0')", "30");
TestStringValue("hex('aAzZ')", "61417A5A");
TestStringValue("hex('Impala')", "496D70616C61");
TestStringValue("hex('impalA')", "696D70616C41");
TestStringValue("unhex('30')", "0");
TestStringValue("unhex('61417A5A')", "aAzZ");
TestStringValue("unhex('496D70616C61')", "Impala");
TestStringValue("unhex('696D70616C41')", "impalA");
// Character not in hex alphabet results in empty string.
TestStringValue("unhex('30GA')", "");
// Uneven number of chars results in empty string.
TestStringValue("unhex('30A')", "");
// Run the test suite twice, once with a bigint parameter, and once with string parameters.
for (int i = 0; i < 2; ++i) {
// First iteration is with bigint, second with string parameter.
string q = (i == 0) ? "" : "'";
// Invalid input: Base below -36 or above 36.
TestIsNull("conv(" + q + "10" + q + ", 10, 37)", TYPE_STRING);
TestIsNull("conv(" + q + "10" + q + ", 37, 10)", TYPE_STRING);
TestIsNull("conv(" + q + "10" + q + ", 10, -37)", TYPE_STRING);
TestIsNull("conv(" + q + "10" + q + ", -37, 10)", TYPE_STRING);
// Invalid input: Base between -2 and 2.
TestIsNull("conv(" + q + "10" + q + ", 10, 1)", TYPE_STRING);
TestIsNull("conv(" + q + "10" + q + ", 1, 10)", TYPE_STRING);
TestIsNull("conv(" + q + "10" + q + ", 10, -1)", TYPE_STRING);
TestIsNull("conv(" + q + "10" + q + ", -1, 10)", TYPE_STRING);
// Invalid input: Positive number but negative src base.
TestIsNull("conv(" + q + "10" + q + ", -10, 10)", TYPE_STRING);
// Test positive numbers.
TestStringValue("conv(" + q + "10" + q + ", 10, 10)", "10");
TestStringValue("conv(" + q + "10" + q + ", 2, 10)", "2");
TestStringValue("conv(" + q + "11" + q + ", 36, 10)", "37");
TestStringValue("conv(" + q + "11" + q + ", 36, 2)", "100101");
TestStringValue("conv(" + q + "100101" + q + ", 2, 36)", "11");
TestStringValue("conv(" + q + "0" + q + ", 10, 2)", "0");
// Test negative numbers (tests from Hive).
// If to_base is positive, the number should be handled as a 2's complement (64-bit).
TestStringValue("conv(" + q + "-641" + q + ", 10, -10)", "-641");
TestStringValue("conv(" + q + "1011" + q + ", 2, -16)", "B");
TestStringValue("conv(" + q + "-1" + q + ", 10, 16)", "FFFFFFFFFFFFFFFF");
TestStringValue("conv(" + q + "-15" + q + ", 10, 16)", "FFFFFFFFFFFFFFF1");
// Test digits that are not available in srcbase. We expect those digits
// from left-to-right that can be interpreted in srcbase to form the result
// (i.e., the paring bails only when it encounters a digit not in srcbase).
TestStringValue("conv(" + q + "17" + q + ", 7, 10)", "1");
TestStringValue("conv(" + q + "371" + q + ", 7, 10)", "3");
TestStringValue("conv(" + q + "371" + q + ", 7, 10)", "3");
TestStringValue("conv(" + q + "445" + q + ", 5, 10)", "24");
// Test overflow (tests from Hive).
// If a number is two large, the result should be -1 (if signed),
// or MAX_LONG (if unsigned).
TestStringValue("conv(" + q + lexical_cast<string>(numeric_limits<int64_t>::max())
+ q + ", 36, 16)", "FFFFFFFFFFFFFFFF");
TestStringValue("conv(" + q + lexical_cast<string>(numeric_limits<int64_t>::max())
+ q + ", 36, -16)", "-1");
TestStringValue("conv(" + q + lexical_cast<string>(numeric_limits<int64_t>::min()+1)
+ q + ", 36, 16)", "FFFFFFFFFFFFFFFF");
TestStringValue("conv(" + q + lexical_cast<string>(numeric_limits<int64_t>::min()+1)
+ q + ", 36, -16)", "-1");
}
// Test invalid input strings that start with an invalid digit.
// Hive returns "0" in such cases.
TestStringValue("conv('@', 16, 10)", "0");
TestStringValue("conv('$123', 12, 2)", "0");
TestStringValue("conv('*12g', 32, 5)", "0");
// TODO: tests with NULL arguments, currently we can't parse them inside function calls.
}
TEST_F(ExprTest, MathFunctions) {
TestValue("pi()", TYPE_DOUBLE, M_PI);
TestValue("e()", TYPE_DOUBLE, M_E);
TestValue("abs(-1.0)", TYPE_DOUBLE, 1.0);
TestValue("abs(1.0)", TYPE_DOUBLE, 1.0);
TestValue("sign(0.0)", TYPE_FLOAT, 1.0f);
TestValue("sign(10.0)", TYPE_FLOAT, 1.0f);
TestValue("sign(-10.0)", TYPE_FLOAT, -1.0f);
// It is important to calculate the expected values
// using math functions, and not simply use constants.
// Otherwise, floating point imprecisions may lead to failed tests.
TestValue("exp(2)", TYPE_DOUBLE, exp(2));
TestValue("exp(e())", TYPE_DOUBLE, exp(M_E));
TestValue("ln(e())", TYPE_DOUBLE, 1.0);
TestValue("ln(255.0)", TYPE_DOUBLE, log(255.0));
TestValue("log10(1000.0)", TYPE_DOUBLE, 3.0);
TestValue("log10(50.0)", TYPE_DOUBLE, log10(50.0));
TestValue("log2(64.0)", TYPE_DOUBLE, 6.0);
TestValue("log2(678.0)", TYPE_DOUBLE, log(678.0) / log(2.0));
TestValue("log(10.0, 1000.0)", TYPE_DOUBLE, log(1000.0) / log(10.0));
TestValue("log(2.0, 64.0)", TYPE_DOUBLE, 6.0);
TestValue("pow(2.0, 10.0)", TYPE_DOUBLE, pow(2.0, 10.0));
TestValue("pow(e(), 2.0)", TYPE_DOUBLE, M_E * M_E);
TestValue("power(2.0, 10.0)", TYPE_DOUBLE, pow(2.0, 10.0));
TestValue("power(e(), 2.0)", TYPE_DOUBLE, M_E * M_E);
TestValue("sqrt(121.0)", TYPE_DOUBLE, 11.0);
TestValue("sqrt(2.0)", TYPE_DOUBLE, sqrt(2.0));
// Run twice to test deterministic behavior.
uint32_t seed = 0;
double expected = static_cast<double>(rand_r(&seed)) / static_cast<double>(RAND_MAX);
TestValue("rand()", TYPE_DOUBLE, expected);
TestValue("rand()", TYPE_DOUBLE, expected);
seed = 1234;
expected = static_cast<double>(rand_r(&seed)) / static_cast<double>(RAND_MAX);
TestValue("rand(1234)", TYPE_DOUBLE, expected);
TestValue("rand(1234)", TYPE_DOUBLE, expected);
// Test bigint param.
TestValue("pmod(10, 3)", TYPE_BIGINT, 1);
TestValue("pmod(-10, 3)", TYPE_BIGINT, 2);
TestValue("pmod(10, -3)", TYPE_BIGINT, -2);
TestValue("pmod(-10, -3)", TYPE_BIGINT, -1);
TestValue("pmod(1234567890, 13)", TYPE_BIGINT, 10);
TestValue("pmod(-1234567890, 13)", TYPE_BIGINT, 3);
TestValue("pmod(1234567890, -13)", TYPE_BIGINT, -3);
TestValue("pmod(-1234567890, -13)", TYPE_BIGINT, -10);
// Test double param.
TestValue("pmod(12.3, 4.0)", TYPE_DOUBLE, fmod(fmod(12.3, 4.0) + 4.0, 4.0));
TestValue("pmod(-12.3, 4.0)", TYPE_DOUBLE, fmod(fmod(-12.3, 4.0) + 4.0, 4.0));
TestValue("pmod(12.3, -4.0)", TYPE_DOUBLE, fmod(fmod(12.3, -4.0) - 4.0, -4.0));
TestValue("pmod(-12.3, -4.0)", TYPE_DOUBLE, fmod(fmod(-12.3, -4.0) - 4.0, -4.0));
TestValue("pmod(123456.789, 13.456)", TYPE_DOUBLE,
fmod(fmod(123456.789, 13.456) + 13.456, 13.456));
TestValue("pmod(-123456.789, 13.456)", TYPE_DOUBLE,
fmod(fmod(-123456.789, 13.456) + 13.456, 13.456));
TestValue("pmod(123456.789, -13.456)", TYPE_DOUBLE,
fmod(fmod(123456.789, -13.456) - 13.456, -13.456));
TestValue("pmod(-123456.789, -13.456)", TYPE_DOUBLE,
fmod(fmod(-123456.789, -13.456) - 13.456, -13.456));
// Test bigint param.
TestValue("positive(1234567890)", TYPE_BIGINT, 1234567890);
TestValue("positive(-1234567890)", TYPE_BIGINT, -1234567890);
TestValue("negative(1234567890)", TYPE_BIGINT, -1234567890);
TestValue("negative(-1234567890)", TYPE_BIGINT, 1234567890);
// Test double param.
TestValue("positive(3.14159265)", TYPE_DOUBLE, 3.14159265);
TestValue("positive(-3.14159265)", TYPE_DOUBLE, -3.14159265);
TestValue("negative(3.14159265)", TYPE_DOUBLE, -3.14159265);
TestValue("negative(-3.14159265)", TYPE_DOUBLE, 3.14159265);
// TODO: tests with NULL arguments, currently we can't parse them inside function calls.
}
TEST_F(ExprTest, MathRoundingFunctions) {
TestValue("ceil(0.1)", TYPE_BIGINT, 1);
TestValue("ceil(-10.05)", TYPE_BIGINT, -10);
TestValue("ceiling(0.1)", TYPE_BIGINT, 1);
TestValue("ceiling(-10.05)", TYPE_BIGINT, -10);
TestValue("floor(0.1)", TYPE_BIGINT, 0);
TestValue("floor(-10.007)", TYPE_BIGINT, -11);
TestValue("round(1.499999)", TYPE_BIGINT, 1);
TestValue("round(1.5)", TYPE_BIGINT, 2);
TestValue("round(1.500001)", TYPE_BIGINT, 2);
TestValue("round(-1.499999)", TYPE_BIGINT, -1);
TestValue("round(-1.5)", TYPE_BIGINT, -2);
TestValue("round(-1.500001)", TYPE_BIGINT, -2);
TestValue("round(3.14159265, 0)", TYPE_DOUBLE, 3.0);
TestValue("round(3.14159265, 1)", TYPE_DOUBLE, 3.1);
TestValue("round(3.14159265, 2)", TYPE_DOUBLE, 3.14);
TestValue("round(3.14159265, 3)", TYPE_DOUBLE, 3.142);
TestValue("round(3.14159265, 4)", TYPE_DOUBLE, 3.1416);
TestValue("round(3.14159265, 5)", TYPE_DOUBLE, 3.14159);
TestValue("round(-3.14159265, 0)", TYPE_DOUBLE, -3.0);
TestValue("round(-3.14159265, 1)", TYPE_DOUBLE, -3.1);
TestValue("round(-3.14159265, 2)", TYPE_DOUBLE, -3.14);
TestValue("round(-3.14159265, 3)", TYPE_DOUBLE, -3.142);
TestValue("round(-3.14159265, 4)", TYPE_DOUBLE, -3.1416);
TestValue("round(-3.14159265, 5)", TYPE_DOUBLE, -3.14159);
}
}
int main(int argc, char **argv) {

4
be/src/exprs/expr.h
View File

@@ -24,6 +24,10 @@ class TExpr;
class TExprNode;
// The materialized value returned by Expr::GetValue().
// Some exprs may set multiple fields of this value at once
// for maintaining state across evaluations.
// For example, the rand() math function uses double_val as its return value,
// and int_val as the state for the random number generator.
struct ExprValue {
bool bool_val;
int8_t tinyint_val;

505
be/src/exprs/math-functions.cc
View File

@@ -2,17 +2,522 @@
#include "exprs/math-functions.h"
#include <iomanip>
#include <sstream>
#include <math.h>
#include "exprs/expr.h"
#include "runtime/tuple-row.h"
#include "util/string-parser.h"
using namespace std;
namespace impala {
const char* MathFunctions::ALPHANUMERIC_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
void* MathFunctions::Pi(Expr* e, TupleRow* row) {
e->result_.double_val = M_PI;
return &e->result_.double_val;
}
void* MathFunctions::E(Expr* e, TupleRow* row) {
e->result_.double_val = M_E;
return &e->result_.double_val;
}
void* MathFunctions::Abs(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = fabs(*d);
return &e->result_.double_val;
}
void* MathFunctions::Sign(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.float_val = (*d < 0) ? -1.0 : 1.0;
return &e->result_.float_val;
}
void* MathFunctions::Sin(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = sin(*d);
return &e->result_.double_val;
}
void* MathFunctions::Asin(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = asin(*d);
return &e->result_.double_val;
}
void* MathFunctions::Cos(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = cos(*d);
return &e->result_.double_val;
}
void* MathFunctions::Acos(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = acos(*d);
return &e->result_.double_val;
}
void* MathFunctions::Tan(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = tan(*d);
return &e->result_.double_val;
}
void* MathFunctions::Atan(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = atan(*d);
return &e->result_.double_val;
}
void* MathFunctions::Radians(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = *d * M_PI / 180.0;
return &e->result_.double_val;
}
void* MathFunctions::Degrees(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = *d * 180 / M_PI;
return &e->result_.double_val;
}
void* MathFunctions::Ceil(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.bigint_val = ceil(*d);
return &e->result_.bigint_val;
}
void* MathFunctions::Floor(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.bigint_val = floor(*d);
return &e->result_.bigint_val;
}
void* MathFunctions::Round(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.bigint_val = static_cast<int64_t>(*d + ((*d < 0) ? -0.5 : 0.5));
return &e->result_.bigint_val;
}
void* MathFunctions::RoundUpTo(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 2);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
int32_t* precision = reinterpret_cast<int32_t*>(e->children()[1]->GetValue(row));
if (d == NULL || precision == NULL) return NULL;
e->result_.double_val = floor(*d * pow(10.0, *precision) + 0.5) / pow(10.0, *precision);
return &e->result_.double_val;
}
void* MathFunctions::Exp(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = exp(*d);
return &e->result_.double_val;
}
void* MathFunctions::Ln(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = log(*d);
return &e->result_.double_val;
}
void* MathFunctions::Log10(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = log10(*d);
return &e->result_.double_val;
}
void* MathFunctions::Log2(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = log(*d) / log (2.0);
return &e->result_.double_val;
}
void* MathFunctions::Log(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 2);
double* base = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
double* val = reinterpret_cast<double*>(e->children()[1]->GetValue(row));
if (val == NULL || base == NULL) return NULL;
e->result_.double_val = log(*val) / log(*base);
return &e->result_.double_val;
}
void* MathFunctions::Pow(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 2);
double* base = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
double* exp = reinterpret_cast<double*>(e->children()[1]->GetValue(row));
if (exp == NULL || base == NULL) return NULL;
e->result_.double_val = pow(*base, *exp);
return &e->result_.double_val;
}
void* MathFunctions::Sqrt(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = sqrt(*d);
return &e->result_.double_val;
}
void* MathFunctions::Rand(Expr* e, TupleRow* row) {
// Use e->result_.int_val as state for the random number generator.
// Cast is necessary, otherwise rand_r will complain.
e->result_.int_val = rand_r(reinterpret_cast<uint32_t*>(&e->result_.int_val));
// Normalize to [0,1].
e->result_.double_val =
static_cast<double>(e->result_.int_val) / static_cast<double>(RAND_MAX);
return &e->result_.double_val;
}
void* MathFunctions::RandSeed(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
int32_t* seed = reinterpret_cast<int32_t*>(e->children()[0]->GetValue(row));
if (seed == NULL) return NULL;
// Use e->result_.bool_val to indicate whether initial seed has been set.
if (!e->result_.bool_val) {
// Set user-defined seed upon this first call.
e->result_.int_val = *seed;
e->result_.bool_val = true;
}
return Rand(e, row);
}
void* MathFunctions::Bin(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
// Cast to an unsigned integer because it is compiler dependent
// whether the sign bit will be shifted like a regular bit.
// (logical vs. arithmetic shift for signed numbers)
uint64_t* num = reinterpret_cast<uint64_t*>(e->children()[0]->GetValue(row));
if (num == NULL) return NULL;
uint64_t n = *num;
const size_t max_bits = sizeof(uint64_t) * 8;
char result[max_bits];
uint32_t index = max_bits;
do {
result[--index] = '0' + (n & 1);
} while (n >>= 1);
StringValue val(result + index, max_bits - index);
// Copies the data in result.
e->result_.SetStringVal(val);
return &e->result_.string_val;
}
void* MathFunctions::HexInt(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
int64_t* num = reinterpret_cast<int64_t*>(e->children()[0]->GetValue(row));
if (num == NULL) return NULL;
stringstream ss;
ss << hex << uppercase << *num;
e->result_.SetStringVal(ss.str());
return &e->result_.string_val;
}
void* MathFunctions::HexString(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
StringValue* s = reinterpret_cast<StringValue*>(e->children()[0]->GetValue(row));
if (s == NULL) return NULL;
stringstream ss;
ss << hex << uppercase << setw(2) << setfill('0');
for (int i = 0; i < s->len; ++i) {
ss << static_cast<int32_t>(s->ptr[i]);
}
e->result_.SetStringVal(ss.str());
return &e->result_.string_val;
}
void* MathFunctions::Unhex(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
StringValue* s = reinterpret_cast<StringValue*>(e->children()[0]->GetValue(row));
if (s == NULL) return NULL;
// For uneven number of chars return empty string like Hive does.
if (s->len % 2 != 0) {
e->result_.string_val.len = 0;
e->result_.string_val.ptr = NULL;
return &e->result_.string_val;
}
int result_len = s->len / 2;
char result[result_len];
int res_index = 0;
int s_index = 0;
while (s_index < s->len) {
char c = 0;
for (int j = 0; j < 2; ++j, ++s_index) {
switch(s->ptr[s_index]) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
c += (s->ptr[s_index] - '0') * ((j == 0) ? 16 : 1);
break;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
// Map to decimal values [10, 15]
c += (s->ptr[s_index] - 'A' + 10) * ((j == 0) ? 16 : 1);
break;
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
// Map to decimal [10, 15]
c += (s->ptr[s_index] - 'a' + 10) * ((j == 0) ? 16 : 1);
break;
default:
// Character not in hex alphabet, return empty string.
e->result_.string_val.len = 0;
e->result_.string_val.ptr = NULL;
return &e->result_.string_val;
}
}
result[res_index] = c;
++res_index;
}
StringValue val(result, result_len);
// Copies the data in result.
e->result_.SetStringVal(val);
return &e->result_.string_val;
}
void* MathFunctions::ConvInt(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 3);
int64_t* num = reinterpret_cast<int64_t*>(e->children()[0]->GetValue(row));
int8_t* src_base = reinterpret_cast<int8_t*>(e->children()[1]->GetValue(row));
int8_t* dest_base = reinterpret_cast<int8_t*>(e->children()[2]->GetValue(row));
if (num == NULL || src_base == NULL || dest_base == NULL) {
return NULL;
}
// As in MySQL and Hive, min base is 2 and max base is 36.
// (36 is max base representable by alphanumeric chars)
// If a negative target base is given, num should be interpreted in 2's complement.
if (abs(*src_base) < MIN_BASE || abs(*src_base) > MAX_BASE
|| abs(*dest_base) < MIN_BASE || abs(*dest_base) > MAX_BASE) {
// Return NULL like Hive does.
return NULL;
}
if (*src_base < 0 && *num >= 0) {
// Invalid input.
return NULL;
}
int64_t decimal_num = *num;
if (*src_base != 10) {
// Convert src_num representing a number in src_base but encoded in decimal
// into its actual decimal number.
if (!DecimalInBaseToDecimal(*num, *src_base, &decimal_num)) {
// Handle overflow, setting decimal_num appropriately.
HandleParseResult(*dest_base, &decimal_num, StringParser::PARSE_OVERFLOW);
}
}
DecimalToBase(decimal_num, *dest_base, &e->result_);
return &e->result_.string_val;
}
void* MathFunctions::ConvString(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 3);
StringValue* num_str = reinterpret_cast<StringValue*>(e->children()[0]->GetValue(row));
int8_t* src_base = reinterpret_cast<int8_t*>(e->children()[1]->GetValue(row));
int8_t* dest_base = reinterpret_cast<int8_t*>(e->children()[2]->GetValue(row));
if (num_str == NULL || src_base == NULL || dest_base == NULL) {
return NULL;
}
// As in MySQL and Hive, min base is 2 and max base is 36.
// (36 is max base representable by alphanumeric chars)
// If a negative target base is given, num should be interpreted in 2's complement.
if (abs(*src_base) < MIN_BASE || abs(*src_base) > MAX_BASE
|| abs(*dest_base) < MIN_BASE || abs(*dest_base) > MAX_BASE) {
// Return NULL like Hive does.
return NULL;
}
// Convert digits in num_str in src_base to decimal.
StringParser::ParseResult parse_res;
int64_t decimal_num = StringParser::StringToInt<int64_t>(num_str->ptr, num_str->len,
*src_base, &parse_res);
if (*src_base < 0 && decimal_num >= 0) {
// Invalid input.
return NULL;
}
if (!HandleParseResult(*dest_base, &decimal_num, parse_res)) {
// Return 0 for invalid input strings like Hive does.
StringValue val(const_cast<char*>("0"), 1);
e->result_.SetStringVal(val);
return &e->result_.string_val;
}
DecimalToBase(decimal_num, *dest_base, &e->result_);
return &e->result_.string_val;
}
void MathFunctions::DecimalToBase(int64_t src_num, int8_t dest_base,
ExprValue* expr_val) {
// Max number of digits of any base (base 2 gives max digits), plus sign.
const size_t max_digits = sizeof(uint64_t) * 8 + 1;
char buf[max_digits];
int32_t result_len = 0;
int32_t buf_index = max_digits - 1;
uint64_t temp_num;
if (dest_base < 0) {
// Dest base is negative, treat src_num as signed.
temp_num = abs(src_num);
} else {
// Dest base is positive. We must interpret src_num in 2's complement.
// Convert to an unsigned int to properly deal with 2's complement conversion.
temp_num = static_cast<uint64_t>(src_num);
}
int abs_base = abs(dest_base);
do {
buf[buf_index] = ALPHANUMERIC_CHARS[temp_num % abs_base];
temp_num /= abs_base;
--buf_index;
++result_len;
} while (temp_num > 0);
// Add optional sign.
if (src_num < 0 && dest_base < 0) {
buf[buf_index] = '-';
++result_len;
}
StringValue val(buf + max_digits - result_len, result_len);
// Copies the data in result.
expr_val->SetStringVal(val);
}
bool MathFunctions::DecimalInBaseToDecimal(int64_t src_num, int8_t src_base,
int64_t* result) {
uint64_t temp_num = abs(src_num);
int32_t place = 1;
*result = 0;
do {
int32_t digit = temp_num % 10;
// Reset result if digit is not representable in src_base.
if (digit >= src_base) {
*result = 0;
place = 1;
} else {
*result += digit * place;
place *= src_base;
// Overflow.
if (UNLIKELY(*result < digit)) {
return false;
}
}
temp_num /= 10;
} while (temp_num > 0);
*result = (src_num < 0) ? -(*result) : *result;
return true;
}
bool MathFunctions::HandleParseResult(int8_t dest_base, int64_t* num,
StringParser::ParseResult parse_res) {
// On overflow set special value depending on dest_base.
// This is consistent with Hive and MySQL's behavior.
if (parse_res == StringParser::PARSE_OVERFLOW) {
if (dest_base < 0) {
*num = -1;
} else {
*num = numeric_limits<uint64_t>::max();
}
} else if (parse_res == StringParser::PARSE_FAILURE) {
// Some other error condition.
return false;
}
return true;
}
void* MathFunctions::PmodBigInt(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 2);
int64_t* a = reinterpret_cast<int64_t*>(e->children()[0]->GetValue(row));
int64_t* b = reinterpret_cast<int64_t*>(e->children()[1]->GetValue(row));
if (a == NULL || b == NULL) return NULL;
e->result_.bigint_val = ((*a % *b) + *b) % *b;
return &e->result_.bigint_val;
}
void* MathFunctions::PmodDouble(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 2);
double* a = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
double* b = reinterpret_cast<double*>(e->children()[1]->GetValue(row));
if (a == NULL || b == NULL) return NULL;
e->result_.double_val = fmod(fmod(*a, *b) + *b, *b);
return &e->result_.double_val;
}
void* MathFunctions::PositiveBigInt(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
int64_t* i = reinterpret_cast<int64_t*>(e->children()[0]->GetValue(row));
if (i == NULL) return NULL;
e->result_.bigint_val = *i;
return &e->result_.bigint_val;
}
void* MathFunctions::PositiveDouble(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = *d;
return &e->result_.double_val;
}
void* MathFunctions::NegativeBigInt(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
int64_t* i = reinterpret_cast<int64_t*>(e->children()[0]->GetValue(row));
if (i == NULL) return NULL;
e->result_.bigint_val = -*i;
return &e->result_.bigint_val;
}
void* MathFunctions::NegativeDouble(Expr* e, TupleRow* row) {
DCHECK_EQ(e->GetNumChildren(), 1);
double* d = reinterpret_cast<double*>(e->children()[0]->GetValue(row));
if (d == NULL) return NULL;
e->result_.double_val = -*d;
return &e->result_.double_val;
}
}

68
be/src/exprs/math-functions.h
View File

@@ -3,17 +3,85 @@
#ifndef IMPALA_EXPRS_MATH_FUNCTIONS_H
#define IMPALA_EXPRS_MATH_FUNCTIONS_H
#include <stdint.h>
// For StringParser::ParseResult
#include "util/string-parser.h"
namespace impala {
class Expr;
class ExprValue;
class OpcodeRegistry;
class TupleRow;
// Math functions that were not auto-generated.
class MathFunctions {
public:
static void Init(OpcodeRegistry*);
static void* Pi(Expr* e, TupleRow* row);
static void* E(Expr* e, TupleRow* row);
static void* Abs(Expr* e, TupleRow* row);
static void* Sign(Expr* e, TupleRow* row);
static void* Sin(Expr* e, TupleRow* row);
static void* Asin(Expr* e, TupleRow* row);
static void* Cos(Expr* e, TupleRow* row);
static void* Acos(Expr* e, TupleRow* row);
static void* Tan(Expr* e, TupleRow* row);
static void* Atan(Expr* e, TupleRow* row);
static void* Radians(Expr* e, TupleRow* row);
static void* Degrees(Expr* e, TupleRow* row);
static void* Ceil(Expr* e, TupleRow* row);
static void* Floor(Expr* e, TupleRow* row);
static void* Round(Expr* e, TupleRow* row);
static void* RoundUpTo(Expr* e, TupleRow* row);
static void* Exp(Expr* e, TupleRow* row);
static void* Ln(Expr* e, TupleRow* row);
static void* Log10(Expr* e, TupleRow* row);
static void* Log2(Expr* e, TupleRow* row);
static void* Log(Expr* e, TupleRow* row);
static void* Pow(Expr* e, TupleRow* row);
static void* Sqrt(Expr* e, TupleRow* row);
static void* Rand(Expr* e, TupleRow* row);
static void* RandSeed(Expr* e, TupleRow* row);
static void* Bin(Expr* e, TupleRow* row);
static void* HexInt(Expr* e, TupleRow* row);
static void* HexString(Expr* e, TupleRow* row);
static void* Unhex(Expr* e, TupleRow* row);
static void* ConvInt(Expr* e, TupleRow* row);
static void* ConvString(Expr* e, TupleRow* row);
static void* PmodBigInt(Expr* e, TupleRow* row);
static void* PmodDouble(Expr* e, TupleRow* row);
static void* PositiveBigInt(Expr* e, TupleRow* row);
static void* PositiveDouble(Expr* e, TupleRow* row);
static void* NegativeBigInt(Expr* e, TupleRow* row);
static void* NegativeDouble(Expr* e, TupleRow* row);
private:
static const int32_t MIN_BASE = 2;
static const int32_t MAX_BASE = 36;
static const char* ALPHANUMERIC_CHARS;
// Converts src_num in decimal to dest_base,
// and fills expr_val.string_val with the result.
static void DecimalToBase(int64_t src_num, int8_t dest_base, ExprValue* expr_val);
// Converts src_num representing a number in src_base but encoded in decimal
// into its actual decimal number.
// For example, if src_num is 21 and src_base is 5,
// then this function sets *result to 2*5^1 + 1*5^0 = 11.
// Returns false if overflow occurred, true upon success.
static bool DecimalInBaseToDecimal(int64_t src_num, int8_t src_base, int64_t* result);
// Helper function used in Conv to implement behavior consistent
// with MySQL and Hive in case of numeric overflow during Conv.
// Inspects parse_res, and in case of overflow sets num to MAXINT64 if dest_base
// is positive, otherwise to -1.
// Returns true if no parse_res == PARSE_SUCCESS || parse_res == PARSE_OVERFLOW.
// Returns false otherwise, indicating some other error condition.
static bool HandleParseResult(int8_t dest_base, int64_t* num,
StringParser::ParseResult parse_res);
};
}

38
be/src/util/string-parser.h
View File

@@ -30,6 +30,7 @@ class StringParser {
// This is considerably faster than glibc's implementation (25x).
// In the case of overflow, the max/min value for the data type will be returned.
// Assumes s represents a decimal number.
template <typename T>
static inline T StringToInt(const char* s, int len, ParseResult* result) {
T val = 0;
@@ -57,6 +58,43 @@ class StringParser {
return (T)(negative ? -val : val);
}
// Convert a string s representing a number in given base into a decimal number.
template <typename T>
static inline T StringToInt(const char* s, int len, int base, ParseResult* result) {
T val = 0;
bool negative = false;
int i = 0;
switch (*s) {
case '-': negative = true;
case '+': i = 1;
}
for (; i < len; ++i) {
T digit;
if (LIKELY(s[i] >= '0' && s[i] <= '9')) {
digit = s[i] - '0';
} else if (s[i] >= 'a' && s[i] <= 'z') {
digit = (s[i] - 'a' + 10);
} else if (s[i] >= 'A' && s[i] <= 'Z') {
digit = (s[i] - 'A' + 10);
} else {
*result = PARSE_FAILURE;
return 0;
}
// Bail, if we encounter a digit that is not available in base.
if (digit >= base) {
break;
}
val = val * base + digit;
// Overflow
if (UNLIKELY(val < digit)) {
*result = PARSE_OVERFLOW;
return negative ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
}
}
*result = PARSE_SUCCESS;
return (T)(negative ? -val : val);
}
// This is considerably faster than glibc's implementation (>100x why???)
// No special case handling needs to be done for overflows, the floating point spec
// already does it and will cap the values to -inf/inf

36
common/function-registry/impala_functions.py
View File

@@ -24,6 +24,42 @@ functions = [
['Regex', 'BOOLEAN', ['STRING', 'STRING'], 'LikePredicate::RegexFn', []],
['Math_Pi', 'DOUBLE', [], 'MathFunctions::Pi', ['pi']],
['Math_E', 'DOUBLE', [], 'MathFunctions::E', ['e']],
['Math_Abs', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Abs', ['abs']],
['Math_Sign', 'FLOAT', ['DOUBLE'], 'MathFunctions::Sign', ['sign']],
['Math_Sin', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Sin', ['sin']],
['Math_Asin', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Asin', ['asin']],
['Math_Cos', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Cos', ['cos']],
['Math_Acos', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Acos', ['acos']],
['Math_Tan', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Tan', ['tan']],
['Math_Atan', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Atan', ['atan']],
['Math_Radians', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Radians', ['radians']],
['Math_Degrees', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Degrees', ['degrees']],
['Math_Ceil', 'BIGINT', ['DOUBLE'], 'MathFunctions::Ceil', ['ceil', 'ceiling']],
['Math_Floor', 'BIGINT', ['DOUBLE'], 'MathFunctions::Floor', ['floor']],
['Math_Round', 'BIGINT', ['DOUBLE'], 'MathFunctions::Round', ['round']],
['Math_Round', 'DOUBLE', ['DOUBLE', 'INT'], 'MathFunctions::RoundUpTo', ['round']],
['Math_Exp', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Exp', ['exp']],
['Math_Ln', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Ln', ['ln']],
['Math_Log10', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Log10', ['log10']],
['Math_Log2', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Log2', ['log2']],
['Math_Log', 'DOUBLE', ['DOUBLE', 'DOUBLE'], 'MathFunctions::Log', ['log']],
['Math_Pow', 'DOUBLE', ['DOUBLE', 'DOUBLE'], 'MathFunctions::Pow', ['pow', 'power']],
['Math_Sqrt', 'DOUBLE', ['DOUBLE'], 'MathFunctions::Sqrt', ['sqrt']],
['Math_Rand', 'DOUBLE', [], 'MathFunctions::Rand', ['rand']],
['Math_Rand', 'DOUBLE', ['INT'], 'MathFunctions::RandSeed', ['rand']],
['Math_Bin', 'STRING', ['BIGINT'], 'MathFunctions::Bin', ['bin']],
['Math_Hex', 'STRING', ['BIGINT'], 'MathFunctions::HexInt', ['hex']],
['Math_Hex', 'STRING', ['STRING'], 'MathFunctions::HexString', ['hex']],
['Math_Unhex', 'STRING', ['STRING'], 'MathFunctions::Unhex', ['unhex']],
['Math_Conv', 'STRING', ['BIGINT', 'TINYINT', 'TINYINT'], 'MathFunctions::ConvInt', ['conv']],
['Math_Conv', 'STRING', ['STRING', 'TINYINT', 'TINYINT'], 'MathFunctions::ConvString', ['conv']],
['Math_Pmod', 'BIGINT', ['BIGINT', 'BIGINT'], 'MathFunctions::PmodBigInt', ['pmod']],
['Math_Pmod', 'DOUBLE', ['DOUBLE', 'DOUBLE'], 'MathFunctions::PmodDouble', ['pmod']],
['Math_Positive', 'BIGINT', ['BIGINT'], 'MathFunctions::PositiveBigInt', ['positive']],
['Math_Positive', 'DOUBLE', ['DOUBLE'], 'MathFunctions::PositiveDouble', ['positive']],
['Math_Negative', 'BIGINT', ['BIGINT'], 'MathFunctions::NegativeBigInt', ['negative']],
['Math_Negative', 'DOUBLE', ['DOUBLE'], 'MathFunctions::NegativeDouble', ['negative']],
['String_Substring', 'STRING', ['STRING', 'INT'], 'StringFunctions::Substring', ['substr', 'substring']],
['String_Substring', 'STRING', ['STRING', 'INT', 'INT'], 'StringFunctions::Substring', ['substr', 'substring']],

1
fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java
View File

@@ -48,6 +48,7 @@ public class FunctionCallExpr extends Expr {
PrimitiveType[] argTypes = new PrimitiveType[this.children.size()];
for (int i = 0; i < this.children.size(); ++i) {
this.children.get(i).analyze(analyzer);
argTypes[i] = this.children.get(i).getType();
}
OpcodeRegistry.Signature match =

7
fe/src/test/java/com/cloudera/impala/analysis/AnalyzerTest.java
View File

@@ -878,6 +878,13 @@ public class AnalyzerTest {
"Arithmetic operation requires numeric or string operands");
}
@Test
public void TestNestedFunctions() {
AnalyzesOk("select sin(pi())");
AnalyzesOk("select sin(cos(pi()))");
AnalyzesOk("select sin(cos(tan(e())))");
}
@Test
public void TestInsert() {
testInsertStatic(true);