IMPALA-12925: Fix decimal data type for external JDBC table

Decimal type is a primitive data type for Impala. Current code returns wrong values for columns with decimal data type in external JDBC tables. This patch fixes wrong values returned from JDBC data source, and supports pushing down decimal type of predicates to remote database and remote Impala. The decimal precision and scale of the columns in external JDBC table must be no less than the decimal precision and scale of the corresponding columns in the table of remote database. Otherwise, Impala fails with an error since it may cause truncation of decimal data. Testing: - Added Planner test for pushing down decimal type of predicates. - Added end-to-end unit-tests for tables with decimal type of columns for Postgres, MySQL, and Impala-to-Impala. - Passed core-tests. Change-Id: I8c9d2e0667c42c0e52436b158e3dfe3ec14b9e3b Reviewed-on: http://gerrit.cloudera.org:8080/21218 Reviewed-by: Michael Smith <michael.smith@cloudera.com> Reviewed-by: Abhishek Rawat <arawat@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
2025-12-19 18:12:08 -05:00 · 2024-03-23 17:18:35 -07:00
parent 4be5fd8896
commit e50bfa8376
14 changed files with 576 additions and 21 deletions
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -487,7 +487,9 @@ error_codes = (

  ("STATESTORE_INCOMPATIBLE_PROTOCOL", 158,
   "Subscriber '$0' has incompatible protocol version V$1 conflicting with statestored's "
-   "version V$2")
+   "version V$2"),
+
+  ("JDBC_CONFIGURATION_ERROR", 159, "Error in JDBC table configuration: $0.")
 )

 import sys
--- a/fe/src/main/java/org/apache/impala/extdatasource/jdbc/JdbcDataSource.java
+++ b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/JdbcDataSource.java
@@ -131,7 +131,7 @@ public class JdbcDataSource implements ExternalDataSource {
    Preconditions.checkState(state_ == DataSourceState.CREATED);
    if (!convertInitStringToConfiguration(params.getInit_string())) {
      return new TPrepareResult(
-          new TStatus(TErrorCode.INTERNAL_ERROR,
+          new TStatus(TErrorCode.JDBC_CONFIGURATION_ERROR,
              Lists.newArrayList("Invalid init_string value")));
    }
    List<Integer> acceptedPredicates = acceptedPredicates(params.getPredicates());
@@ -149,7 +149,7 @@ public class JdbcDataSource implements ExternalDataSource {
    // the frontend and used a different instance of this JdbcDataSource class.
    if (!convertInitStringToConfiguration(params.getInit_string())) {
      return new TOpenResult(
-          new TStatus(TErrorCode.INTERNAL_ERROR,
+          new TStatus(TErrorCode.JDBC_CONFIGURATION_ERROR,
              Lists.newArrayList("Invalid init_string value")));
    }
    // 2. Build the query and execute it
@@ -159,7 +159,7 @@ public class JdbcDataSource implements ExternalDataSource {
      buildQueryAndExecute(params);
    } catch (JdbcDatabaseAccessException e) {
      return new TOpenResult(
-          new TStatus(TErrorCode.INTERNAL_ERROR, Lists.newArrayList(e.getMessage())));
+          new TStatus(TErrorCode.RUNTIME_ERROR, Lists.newArrayList(e.getMessage())));
    }
    scanHandle_ = UUID.randomUUID().toString();
    return new TOpenResult(STATUS_OK).setScan_handle(scanHandle_);
@@ -176,7 +176,7 @@ public class JdbcDataSource implements ExternalDataSource {
    if (schema_.getColsSize() != 0) {
      if (iterator_ == null) {
        return new TGetNextResult(
-            new TStatus(TErrorCode.INTERNAL_ERROR,
+            new TStatus(TErrorCode.RUNTIME_ERROR,
                Lists.newArrayList("Iterator of JDBC resultset is null")));
      }
      for (int i = 0; i < schema_.getColsSize(); ++i) {
@@ -189,6 +189,9 @@ public class JdbcDataSource implements ExternalDataSource {
          iterator_.next(schema_.getCols(), cols);
          ++numRows;
        }
+      } catch (UnsupportedOperationException e) {
+        return new TGetNextResult(new TStatus(
+            TErrorCode.JDBC_CONFIGURATION_ERROR, Lists.newArrayList(e.getMessage())));
      } catch (Exception e) {
        hasNext = false;
      }
@@ -214,7 +217,7 @@ public class JdbcDataSource implements ExternalDataSource {
      return new TCloseResult(STATUS_OK);
    } catch (Exception e) {
      return new TCloseResult(
-          new TStatus(TErrorCode.INTERNAL_ERROR, Lists.newArrayList(e.getMessage())));
+          new TStatus(TErrorCode.RUNTIME_ERROR, Lists.newArrayList(e.getMessage())));
    }
  }

--- a/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JdbcRecordIterator.java
+++ b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/dao/JdbcRecordIterator.java
@@ -92,7 +92,8 @@ public class JdbcRecordIterator {
    }
  }

-  public void next(List<TColumnDesc> colDescs, List<TColumnData> colDatas) {
+  public void next(List<TColumnDesc> colDescs, List<TColumnData> colDatas)
+      throws UnsupportedOperationException {
    Preconditions.checkState(colDescs.size() == colDatas.size());
    for (int i = 0; i < colDescs.size(); ++i) {
      TColumnType type = colDescs.get(i).getType();
@@ -149,8 +150,20 @@ public class JdbcRecordIterator {
            break;
          case DECIMAL:
            BigDecimal val = rs.getBigDecimal(i + 1);
-            colData.addToBinary_vals(
-                SerializationUtils.encodeDecimal(new BigDecimal(val.byteValue())));
+            int valPrecision = val.precision();
+            int valScale = val.scale();
+            // Check if there is enough precision and scale in the destination decimal.
+            if (scalarType.scale < valScale ||
+                scalarType.precision < valPrecision + scalarType.scale - valScale) {
+              throw new UnsupportedOperationException(String.format("Invalid DECIMAL" +
+                  "(%d, %d) for column %s since there is possible loss of precision " +
+                  "when casting from DECIMAL(%d, %d)",
+                  scalarType.precision, scalarType.scale, colDescs.get(i).getName(),
+                  valPrecision, valScale));
+            } else if (scalarType.scale > valScale) {
+              val = val.setScale(scalarType.scale);
+            }
+            colData.addToBinary_vals(SerializationUtils.encodeDecimal(val));
            break;
          case BINARY:
          case CHAR:
--- a/fe/src/main/java/org/apache/impala/extdatasource/jdbc/util/QueryConditionUtil.java
+++ b/fe/src/main/java/org/apache/impala/extdatasource/jdbc/util/QueryConditionUtil.java
@@ -27,6 +27,8 @@ import org.apache.impala.extdatasource.jdbc.dao.DatabaseAccessor;
 import org.apache.impala.extdatasource.thrift.TBinaryPredicate;
 import org.apache.impala.extdatasource.thrift.TComparisonOp;
 import org.apache.impala.thrift.TColumnValue;
+import org.apache.impala.thrift.TPrimitiveType;
+import org.apache.impala.thrift.TTypeNode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

@@ -51,7 +53,8 @@ public class QueryConditionUtil {
        String name = predicate.getCol().getName();
        name = columnMapping.getOrDefault(name, name);
        String op = converse(predicate.getOp());
-        String value = getTColumnValueAsString(predicate.getValue(), dbAccessor_);
+        String value = getTColumnValueAsString(predicate.getValue(),
+            predicate.getCol().getType().getTypes().get(0), dbAccessor_);
        joiner.add(String.format("%s %s %s", name, op, value));
      }
      condition.add(joiner.toString());
@@ -65,7 +68,7 @@ public class QueryConditionUtil {
   *
   * @see org.apache.impala.planner.DataSourceScanNode#literalToColumnValue
   */
-  public static String getTColumnValueAsString(TColumnValue value,
+  public static String getTColumnValueAsString(TColumnValue value, TTypeNode node,
      DatabaseAccessor dbAccessor_) {
    Preconditions.checkState(value != null);
    StringBuilder sb = new StringBuilder();
@@ -82,11 +85,17 @@ public class QueryConditionUtil {
    } else if (value.isSetDouble_val()) {
      sb.append(value.double_val);
    } else if (value.isSetString_val()) {
-      sb.append(String.format("'%s'", value.string_val));
+      // DECIMAL and TIMESTAMP types of predicates are represented as string.
+      if (node.getScalar_type().getType() == TPrimitiveType.DECIMAL) {
+        // Check column data type and don't add quotes for decimal string.
+        sb.append(String.format("%s", value.string_val));
+      } else {
+        sb.append(String.format("'%s'", value.string_val));
+      }
    } else if (value.isSetDate_val()) {
      sb.append(String.format("'%s'", dbAccessor_.getDateString(value.date_val)));
    } else {
-      // TODO: Support data types of DECIMAL, TIMESTAMP, DATE and binary for predicates.
+      // TODO: Support data types of binary for predicates.
      // Keep in-sync with DataSourceScanNode.literalToColumnValue().
      throw new IllegalArgumentException("Unsupported data type.");
    }
--- a/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
@@ -150,7 +150,24 @@ public class DataSourceScanNode extends ScanNode {
        return new TColumnValue().setString_val(
            ((TimestampLiteral) expr).getStringValue());
      case DECIMAL:
-      case DATETIME:
+        /**
+         * Decimal type of predicate data in TBinaryPredicate is saved as 'binary'.
+         * Predicates are generated by Planner and sent to backend DataSourceScanNode.
+         * When backend DataSourceScanNode calls ExternalDataSource.open() API, it sends
+         * predicates to Java ExternalDataSourceExecutor object as TOpenParams.predicates
+         * through JNI API. There is an issue to deserialize 'binary' type of data in
+         * TBinaryPredicate when JniUtil.deserializeThrift() is called in function
+         * ExternalDataSourceExecutor.open(). 'binary' type of data are changed to wrong
+         * value. JniUtil.deserializeThrift() calls Thrift API TDeserializer.deserialize.
+         * To work around this Thrift deserialization issue, string representation of the
+         * decimal is returned by converting decimal to string with BigDecimal.toString()
+         * Java API.
+         * TODO: Retry using JNI & Thrift TDeserializer.deserialize for decimal saved as
+         * 'binary' when Thrift is upgraded.
+         */
+        return new TColumnValue().setString_val(
+            ((NumericLiteral) expr).getValue().toString());
+       case DATETIME:
        return null;
      default:
        Preconditions.checkState(false);
--- a/testdata/bin/clean-mysql-env.sh
+++ b/testdata/bin/clean-mysql-env.sh
@@ -35,6 +35,7 @@ rm -rf /tmp/mysql-8.2.0.*
 # Clean tmp files
 rm -f /tmp/mysql_jdbc_alltypes.*
 rm -f /tmp/mysql_jdbc.*sql
+rm -f /tmp/mysql_jdbc_decimal_tbl.*

 EXT_DATA_SOURCES_HDFS_PATH=${FILESYSTEM_PREFIX}/test-warehouse/data-sources
 JDBC_DRIVERS_HDFS_PATH=${EXT_DATA_SOURCES_HDFS_PATH}/jdbc-drivers
--- a/testdata/bin/create-ext-data-source-table.sql
+++ b/testdata/bin/create-ext-data-source-table.sql
@@ -92,3 +92,20 @@ TBLPROPERTIES (
 "dbcp.password"="password",
 "table"="AllTypesWithQuote",
 "column.mapping"="id=id, bool_col=Bool_col, tinyint_col=Tinyint_col, smallint_col=Smallint_col, int_col=Int_col, bigint_col=Bigint_col, float_col=Float_col, double_col=Double_col, date_string_col=Date_string_col, string_col=String_col, timestamp=Timestamp");
+
+DROP TABLE IF EXISTS jdbc_decimal_tbl;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl (
+ d1 DECIMAL(9,0),
+ d2 DECIMAL(10,0),
+ d3 DECIMAL(20,10),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(10,5))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="POSTGRES",
+"jdbc.url"="jdbc:postgresql://localhost:5432/functional",
+"jdbc.driver"="org.postgresql.Driver",
+"driver.url"="hdfs://localhost:20500/test-warehouse/data-sources/jdbc-drivers/postgresql-jdbc.jar",
+"dbcp.username"="hiveuser",
+"dbcp.password"="password",
+"table"="decimal_tbl");
--- a/testdata/bin/load-ext-data-sources.sh
+++ b/testdata/bin/load-ext-data-sources.sh
@@ -69,6 +69,20 @@ CREATE TABLE "AllTypesWithQuote"
 __EOT__
 sudo -u postgres psql -U hiveuser -d functional -f /tmp/jdbc_alltypes_with_quote.sql

+# Create a table with decimal type of columns
+cat > /tmp/jdbc_decimal_tbl.sql <<__EOT__
+DROP TABLE IF EXISTS decimal_tbl;
+CREATE TABLE decimal_tbl
+(
+    d1 DECIMAL(9,0),
+    d2 DECIMAL(10,0),
+    d3 DECIMAL(20,10),
+    d4 DECIMAL(38,38),
+    d5 DECIMAL(10,5)
+);
+__EOT__
+sudo -u postgres psql -U hiveuser -d functional -f /tmp/jdbc_decimal_tbl.sql
+
 # Load data to jdbc table
 cat ${IMPALA_HOME}/testdata/target/AllTypes/* > /tmp/jdbc_alltypes.csv
 loadCmd="COPY alltypes FROM '/tmp/jdbc_alltypes.csv' DELIMITER ',' CSV"
@@ -77,6 +91,10 @@ sudo -u postgres psql -d functional -c "$loadCmd"
 loadCmd="COPY \"AllTypesWithQuote\" FROM '/tmp/jdbc_alltypes.csv' DELIMITER ',' CSV"
 sudo -u postgres psql -d functional -c "$loadCmd"

+cat ${IMPALA_HOME}/testdata/data/decimal_tbl.txt > /tmp/jdbc_decimal_tbl.csv
+loadCmd="COPY decimal_tbl FROM '/tmp/jdbc_decimal_tbl.csv' DELIMITER ',' CSV"
+sudo -u postgres psql -d functional -c "$loadCmd"
+
 # Create impala tables and load data
 cat > /tmp/impala_jdbc_alltypes.sql <<__EOT__
 USE FUNCTIONAL;
@@ -109,4 +127,5 @@ ${IMPALA_HOME}/bin/impala-shell.sh -i ${IMPALAD} -f /tmp/impala_jdbc_alltypes.sq
 # Clean tmp files
 rm /tmp/jdbc_alltypes.*
 rm /tmp/jdbc_alltypes_with_quote.*
+rm /tmp/jdbc_decimal_tbl.*
 rm /tmp/impala_jdbc_alltypes.sql
--- a/testdata/bin/setup-mysql-env.sh
+++ b/testdata/bin/setup-mysql-env.sh
@@ -123,6 +123,24 @@ __EOT__
 docker exec -i mysql mysql -uroot -psecret functional < \
  /tmp/mysql_jdbc_alltypes_with_case_sensitive_names.sql

+# Create a table with decimal type of columns.
+# Note that the decimal scale in MySQL has a range of 0 to 30, which is smaller than the
+# scale range in Impala (0 to 38).
+cat > /tmp/mysql_jdbc_decimal_tbl.sql <<__EOT__
+DROP TABLE IF EXISTS decimal_tbl;
+CREATE TABLE decimal_tbl
+(
+    d1 DECIMAL(9,0),
+    d2 DECIMAL(10,0),
+    d3 DECIMAL(20,10),
+    d4 DECIMAL(38,30),
+    d5 DECIMAL(10,5)
+);
+__EOT__
+
+docker exec -i mysql mysql -uroot -psecret functional < \
+  /tmp/mysql_jdbc_decimal_tbl.sql
+
 # Load data to jdbc table
 cat ${IMPALA_HOME}/testdata/target/AllTypes/* > /tmp/mysql_jdbc_alltypes.csv
 docker cp /tmp/mysql_jdbc_alltypes.csv mysql:/tmp
@@ -141,6 +159,15 @@ loadCmd="LOAD DATA LOCAL INFILE '/tmp/mysql_jdbc_alltypes.csv' INTO TABLE \

 docker exec -i mysql mysql -uroot -psecret functional --local-infile=1 <<<  "$loadCmd"

+cat ${IMPALA_HOME}/testdata/data/decimal_tbl.txt > /tmp/mysql_jdbc_decimal_tbl.csv
+docker cp /tmp/mysql_jdbc_decimal_tbl.csv mysql:/tmp
+
+loadCmd="LOAD DATA LOCAL INFILE '/tmp/mysql_jdbc_decimal_tbl.csv' INTO TABLE \
+  decimal_tbl COLUMNS TERMINATED BY ','"
+
+docker exec -i mysql mysql -uroot -psecret functional --local-infile=1 <<<  "$loadCmd"
+
+
 EXT_DATA_SOURCE_SRC_PATH=${IMPALA_HOME}/java/ext-data-source
 EXT_DATA_SOURCES_HDFS_PATH=${FILESYSTEM_PREFIX}/test-warehouse/data-sources
 JDBC_DRIVERS_HDFS_PATH=${EXT_DATA_SOURCES_HDFS_PATH}/jdbc-drivers
--- a/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/data-source-tables.test
@@ -125,3 +125,11 @@ data source predicates: timestamp_col >= TIMESTAMP '2009-01-02 00:00:00', timest
   limit: 15
   row-size=64B cardinality=1
 ====
+select * from functional.jdbc_decimal_tbl where d3 > 123.456 and d5 < 10.0
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN DATA SOURCE [functional.jdbc_decimal_tbl]
+data source predicates: d3 > 123.456, d5 < 10.0
+   row-size=52B cardinality=1
+====
--- a/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables-predicates.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables-predicates.test
@@ -1,6 +1,6 @@
 ====
 ---- QUERY
-# Create external JDBC DataSource table
+# Create external JDBC table
 DROP TABLE IF EXISTS alltypes_jdbc_datasource;
 CREATE EXTERNAL TABLE alltypes_jdbc_datasource (
 id INT,
@@ -28,6 +28,31 @@ TBLPROPERTIES (
 'Table has been created.'
 ====
 ---- QUERY
+# Create external JDBC table with decimal type of columns.
+# The decimal precisions and scales of decimal columns equal to the decimal precisions
+# and scales of the corresponding decimal columns in the remote table.
+DROP TABLE IF EXISTS jdbc_decimal_tbl;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl (
+ d1 DECIMAL(9,0),
+ d2 DECIMAL(10,0),
+ d3 DECIMAL(20,10),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(10,5),
+ d6 DECIMAL(9,0))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="IMPALA",
+"jdbc.url"="jdbc:impala://$INTERNAL_LISTEN_HOST:21050/functional",
+"jdbc.auth"="AuthMech=0",
+"jdbc.driver"="com.cloudera.impala.jdbc.Driver",
+"driver.url"="$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/ImpalaJDBC42.jar",
+"dbcp.username"="impala",
+"dbcp.password"="cloudera",
+"table"="decimal_tbl");
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
 # Gets specified columns based on date predicate with operator '='.
 select id, bool_col, smallint_col, float_col, double_col, date_col
 from alltypes_jdbc_datasource
@@ -291,8 +316,72 @@ where timestamp_col > '2009-01-03 00:22:00' order by id limit 5;
 INT, BOOLEAN, SMALLINT, FLOAT, DOUBLE, DATE, TIMESTAMP
 ====
 ---- QUERY
+select * from jdbc_decimal_tbl where d3 < 10.00;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 <= 123.45678900;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900,1
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100,1
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 = 123.45678900;
+---- RESULTS
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 != 123.45678900;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900,1
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100,1
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000,1
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 >= 123.45678900;
+---- RESULTS
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000,1
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000,1
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 > 123.45678900;
+---- RESULTS
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000,1
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 >= 123.45678900 and d5 < 10.0;
+---- RESULTS
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000,1
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
 # Drop table
 DROP TABLE alltypes_jdbc_datasource;
 ---- RESULTS
 'Table has been dropped.'
 ====
+---- QUERY
+# Drop table
+DROP TABLE jdbc_decimal_tbl;
+---- RESULTS
+'Table has been dropped.'
+====
--- a/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/impala-ext-jdbc-tables.test
@@ -1,6 +1,6 @@
 ====
 ---- QUERY
-# Create external JDBC DataSource table
+# Create external JDBC table
 DROP TABLE IF EXISTS alltypes_jdbc_datasource;
 CREATE EXTERNAL TABLE alltypes_jdbc_datasource (
 id INT,
@@ -29,7 +29,7 @@ TBLPROPERTIES (
 'Table has been created.'
 ====
 ---- QUERY
-# Create external JDBC DataSource table
+# Create external JDBC table
 DROP TABLE IF EXISTS alltypes_jdbc_datasource_2;
 CREATE EXTERNAL TABLE alltypes_jdbc_datasource_2 (
 id INT,
@@ -58,6 +58,108 @@ TBLPROPERTIES (
 'Table has been created.'
 ====
 ---- QUERY
+# Create external JDBC table with decimal type of columns.
+# The decimal precisions and scales of decimal columns equal to the decimal precisions
+# and scales of the corresponding decimal columns in the remote table.
+DROP TABLE IF EXISTS jdbc_decimal_tbl;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl (
+ d1 DECIMAL(9,0),
+ d2 DECIMAL(10,0),
+ d3 DECIMAL(20,10),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(10,5),
+ d6 DECIMAL(9,0))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="IMPALA",
+"jdbc.url"="jdbc:impala://$INTERNAL_LISTEN_HOST:21050/functional",
+"jdbc.auth"="AuthMech=0",
+"jdbc.driver"="com.cloudera.impala.jdbc.Driver",
+"driver.url"="$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/ImpalaJDBC42.jar",
+"dbcp.username"="impala",
+"dbcp.password"="cloudera",
+"table"="decimal_tbl");
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# Create external JDBC table with decimal type.
+# The decimal precisions and scales of decimal columns are greater than or equal to the
+# decimal precision and scales of the corresponding decimal columns in the remote table.
+DROP TABLE IF EXISTS jdbc_decimal_tbl_2;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl_2 (
+ d1 DECIMAL(10,1),
+ d2 DECIMAL(11,1),
+ d3 DECIMAL(21,11),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(11,6),
+ d6 DECIMAL(10,1))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="IMPALA",
+"jdbc.url"="jdbc:impala://$INTERNAL_LISTEN_HOST:21050/functional",
+"jdbc.auth"="AuthMech=0",
+"jdbc.driver"="com.cloudera.impala.jdbc.Driver",
+"driver.url"="$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/ImpalaJDBC42.jar",
+"dbcp.username"="impala",
+"dbcp.password"="cloudera",
+"table"="decimal_tbl");
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# Create external JDBC table with decimal type.
+# The decimal scales of two columns d3 and d5 are less than the decimal scales of the
+# corresponding decimal columns in the remote table, which cause truncation, hence
+# exception will be thrown.
+DROP TABLE IF EXISTS jdbc_decimal_tbl_3;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl_3 (
+ d1 DECIMAL(9,0),
+ d2 DECIMAL(10,0),
+ d3 DECIMAL(20,9),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(10,4),
+ d6 DECIMAL(9,0))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="IMPALA",
+"jdbc.url"="jdbc:impala://$INTERNAL_LISTEN_HOST:21050/functional",
+"jdbc.auth"="AuthMech=0",
+"jdbc.driver"="com.cloudera.impala.jdbc.Driver",
+"driver.url"="$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/ImpalaJDBC42.jar",
+"dbcp.username"="impala",
+"dbcp.password"="cloudera",
+"table"="decimal_tbl");
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# Create external JDBC table with decimal type.
+# The decimal scale of column d5 is larger than the decimal scale of the corresponding
+# decimal columns in the remote table, but their decimal precisions are same, which cause
+# truncation, hence exception will be thrown.
+DROP TABLE IF EXISTS jdbc_decimal_tbl_4;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl_4 (
+ d1 DECIMAL(9,0),
+ d2 DECIMAL(10,0),
+ d3 DECIMAL(20,10),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(10,6),
+ d6 DECIMAL(9,0))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="IMPALA",
+"jdbc.url"="jdbc:impala://$INTERNAL_LISTEN_HOST:21050/functional",
+"jdbc.auth"="AuthMech=0",
+"jdbc.driver"="com.cloudera.impala.jdbc.Driver",
+"driver.url"="$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/ImpalaJDBC42.jar",
+"dbcp.username"="impala",
+"dbcp.password"="cloudera",
+"table"="decimal_tbl");
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
 # Test the jdbc DataSource
 # count(*) with a predicate evaluated by Impala
 # Binary predicates are pushed to the external jdbc DataSource.
@@ -202,6 +304,38 @@ row_regex: .*RowsRead: 3 .*
 aggregation(SUM, RowsRead): 6
 ====
 ---- QUERY
+select * from jdbc_decimal_tbl;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900,1
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100,1
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000,1
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000,1
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889,1
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl_2;
+---- RESULTS
+1234.0,2222.0,1.23456789000,0.12345678900000000000000000000000000000,12345.789000,1.0
+2345.0,111.0,12.34567890000,0.12345678900000000000000000000000000000,3.141000,1.0
+12345.0,333.0,123.45678900000,0.12345678900000000000000000000000000000,11.220000,1.0
+12345.0,333.0,1234.56789000000,0.12345678900000000000000000000000000000,0.100000,1.0
+132842.0,333.0,12345.67890000000,0.12345678900000000000000000000000000000,0.778890,1.0
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl_3;
+---- CATCH
+row_regex: .*Invalid DECIMAL\(20, 9\) for column d3 since there is possible loss of precision when casting from DECIMAL\(11, 10\).*
+====
+---- QUERY
+select * from jdbc_decimal_tbl_4;
+---- CATCH
+row_regex: .*Invalid DECIMAL\(10, 6\) for column d5 since there is possible loss of precision when casting from DECIMAL\(10, 5\).*
+====
+---- QUERY
 # Drop table
 DROP TABLE alltypes_jdbc_datasource;
 ---- RESULTS
@@ -213,3 +347,27 @@ DROP TABLE alltypes_jdbc_datasource_2;
 ---- RESULTS
 'Table has been dropped.'
 ====
+---- QUERY
+# Drop table
+DROP TABLE jdbc_decimal_tbl;
+---- RESULTS
+'Table has been dropped.'
+====
+---- QUERY
+# Drop table
+DROP TABLE jdbc_decimal_tbl_2;
+---- RESULTS
+'Table has been dropped.'
+====
+---- QUERY
+# Drop table
+DROP TABLE jdbc_decimal_tbl_3;
+---- RESULTS
+'Table has been dropped.'
+====
+---- QUERY
+# Drop table
+DROP TABLE jdbc_decimal_tbl_4;
+---- RESULTS
+'Table has been dropped.'
+====
--- a/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/jdbc-data-source.test
@@ -1,6 +1,6 @@
 ====
 ---- QUERY
-# Create external JDBC DataSource table
+# Create external JDBC table
 DROP TABLE IF EXISTS alltypes_jdbc_datasource;
 CREATE EXTERNAL TABLE alltypes_jdbc_datasource (
 id INT,
@@ -28,7 +28,7 @@ TBLPROPERTIES (
 'Table has been created.'
 ====
 ---- QUERY
-# Create external JDBC DataSource table
+# Create external JDBC table
 DROP TABLE IF EXISTS alltypes_jdbc_datasource_2;
 CREATE EXTERNAL TABLE alltypes_jdbc_datasource_2 (
 id INT,
@@ -56,6 +56,27 @@ TBLPROPERTIES (
 'Table has been created.'
 ====
 ---- QUERY
+# Create external JDBC table with decimal type of columns
+DROP TABLE IF EXISTS jdbc_decimal_tbl;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl (
+ d1 DECIMAL(9,0),
+ d2 DECIMAL(10,0),
+ d3 DECIMAL(20,10),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(10,5))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="POSTGRES",
+"jdbc.url"="jdbc:postgresql://$INTERNAL_LISTEN_HOST:5432/functional",
+"jdbc.driver"="org.postgresql.Driver",
+"driver.url"="$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/postgresql-jdbc.jar",
+"dbcp.username"="hiveuser",
+"dbcp.password"="password",
+"table"="decimal_tbl");
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
 # Test the jdbc DataSource
 # count(*) with a predicate evaluated by Impala
 select count(*) from alltypes_jdbc_datasource
@@ -422,6 +443,75 @@ where timestamp_col > '2009-01-03 00:22:00' order by id limit 5;
 INT, BOOLEAN, SMALLINT, FLOAT, DOUBLE, DATE, TIMESTAMP
 ====
 ---- QUERY
+select * from jdbc_decimal_tbl;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 < 10.00;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 <= 123.45678900;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 = 123.45678900;
+---- RESULTS
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 != 123.45678900;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 >= 123.45678900;
+---- RESULTS
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 > 123.45678900;
+---- RESULTS
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 >= 123.45678900 and d5 < 10.0;
+---- RESULTS
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
 # Drop table
 DROP TABLE alltypes_jdbc_datasource;
 ---- RESULTS
@@ -434,6 +524,12 @@ DROP TABLE alltypes_jdbc_datasource_2;
 'Table has been dropped.'
 ====
 ---- QUERY
+# Drop table.
+DROP TABLE jdbc_decimal_tbl;
+---- RESULTS
+'Table has been dropped.'
+====
+---- QUERY
 # Create JDBC table with unknown database type.
 DROP TABLE IF EXISTS jdbc_test;
 CREATE EXTERNAL TABLE jdbc_test (
--- a/testdata/workloads/functional-query/queries/QueryTest/mysql-ext-jdbc-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/mysql-ext-jdbc-tables.test
@@ -1,6 +1,6 @@
 ====
 ---- QUERY
-# Create external JDBC DataSource table
+# Create external JDBC table
 DROP TABLE IF EXISTS alltypes_jdbc_datasource;
 CREATE EXTERNAL TABLE alltypes_jdbc_datasource (
 id INT,
@@ -28,7 +28,7 @@ TBLPROPERTIES (
 'Table has been created.'
 ====
 ---- QUERY
-# Create external JDBC DataSource table
+# Create external JDBC table
 DROP TABLE IF EXISTS alltypes_jdbc_datasource_2;
 CREATE EXTERNAL TABLE alltypes_jdbc_datasource_2 (
 id INT,
@@ -56,6 +56,27 @@ TBLPROPERTIES (
 'Table has been created.'
 ====
 ---- QUERY
+# Create external JDBC table with decimal type of columns
+DROP TABLE IF EXISTS jdbc_decimal_tbl;
+CREATE EXTERNAL TABLE jdbc_decimal_tbl (
+ d1 DECIMAL(9,0),
+ d2 DECIMAL(10,0),
+ d3 DECIMAL(20,10),
+ d4 DECIMAL(38,38),
+ d5 DECIMAL(10,5))
+STORED BY JDBC
+TBLPROPERTIES (
+"database.type"="MYSQL",
+"jdbc.url"="jdbc:mysql://localhost:3306/functional",
+"jdbc.driver"="com.mysql.cj.jdbc.Driver",
+"driver.url"="$FILESYSTEM_PREFIX/test-warehouse/data-sources/jdbc-drivers/mysql-jdbc.jar",
+"dbcp.username"="hiveuser",
+"dbcp.password"="password",
+"table"="decimal_tbl");
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
 # Test the jdbc DataSource
 # count(*) with a predicate evaluated by Impala
 select count(*) from alltypes_jdbc_datasource
@@ -422,6 +443,75 @@ where timestamp_col > '2009-01-03 00:22:00' order by id limit 5;
 INT, BOOLEAN, SMALLINT, FLOAT, DOUBLE, DATE, TIMESTAMP
 ====
 ---- QUERY
+select * from jdbc_decimal_tbl;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 < 10.00;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 <= 123.45678900;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 = 123.45678900;
+---- RESULTS
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 != 123.45678900;
+---- RESULTS
+1234,2222,1.2345678900,0.12345678900000000000000000000000000000,12345.78900
+2345,111,12.3456789000,0.12345678900000000000000000000000000000,3.14100
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 >= 123.45678900;
+---- RESULTS
+12345,333,123.4567890000,0.12345678900000000000000000000000000000,11.22000
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 > 123.45678900;
+---- RESULTS
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
+select * from jdbc_decimal_tbl where d3 >= 123.45678900 and d5 < 10.0;
+---- RESULTS
+12345,333,1234.5678900000,0.12345678900000000000000000000000000000,0.10000
+132842,333,12345.6789000000,0.12345678900000000000000000000000000000,0.77889
+---- TYPES
+DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
+---- QUERY
 # Drop table
 DROP TABLE alltypes_jdbc_datasource;
 ---- RESULTS
@@ -433,3 +523,9 @@ DROP TABLE alltypes_jdbc_datasource_2;
 ---- RESULTS
 'Table has been dropped.'
 ====
+---- QUERY
+# Drop table
+DROP TABLE jdbc_decimal_tbl;
+---- RESULTS
+'Table has been dropped.'
+====