IMPALA-6954: Fix problems with CTAS into Kudu with an expr rewrite

This patch fixes two problems:
- Previously a CTAS into a Kudu table where an expr rewrite occurred
  would create an unpartitioned table, due to the partition info being
  reset in TableDataLayout and then never reconstructed. Since the
  Kudu partition info is set by the parser and never changes, the
  solution is to not reset it.
- Previously a CTAS into a Kudu table with a range partition where an
  expr rewrite occurred would fail with an analysis exception due to
  a Precondition check in RangePartition.analyze that checked that
  the RangePartition wasn't already analyzed, as the analysis can't
  be done twice. Since the state in RangePartition never changes, it
  doesn't need to be reanalyzed and we can just return instead of
  failing on the check.

Testing:
- Added an e2e test that creates a partitioned Kudu table with a CTAS
  with a rewrite, and checks that the expected partitions are created.

Change-Id: I731743bd84cc695119e99342e1b155096147f0ed
Reviewed-on: http://gerrit.cloudera.org:8080/10251
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
Thomas Tauber-Marshall
2018-04-30 15:32:30 -07:00
committed by Impala Public Jenkins
parent 583dcc31b0
commit ba84ad03cb
5 changed files with 16 additions and 8 deletions

View File

@@ -253,6 +253,9 @@ public class CreateTableAsSelectStmt extends StatementBase {
public void reset() {
super.reset();
createStmt_.reset();
// This is populated for CTAS in analyze(), so it needs to be cleared here. For other
// types of CreateTableStmts it is set by the parser and should not be reset.
createStmt_.getPartitionColumnDefs().clear();
insertStmt_.reset();
}
}

View File

@@ -121,8 +121,9 @@ public class RangePartition implements ParseNode {
public void analyze(Analyzer analyzer, List<ColumnDef> partColDefs)
throws AnalysisException {
// Reanalyzing not supported because TIMESTAMPs are converted to BIGINT (unixtime
// micros) in place.
Preconditions.checkArgument(!isAnalyzed_);
// micros) in place. We can just return because none of the state will have changed
// since the first time we did the analysis.
if (isAnalyzed_) return;
analyzeBoundaryValues(lowerBound_, partColDefs, analyzer);
if (!isSingletonRange_) {
analyzeBoundaryValues(upperBound_, partColDefs, analyzer);

View File

@@ -52,9 +52,4 @@ class TableDataLayout {
List<ColumnDef> getPartitionColumnDefs() { return partitionColDefs_; }
List<KuduPartitionParam> getKuduPartitionParams() { return kuduPartitionParams_; }
public void reset() {
partitionColDefs_.clear();
kuduPartitionParams_.clear();
}
}

View File

@@ -161,7 +161,6 @@ class TableDef {
public void reset() {
primaryKeyColDefs_.clear();
dataLayout_.reset();
columnDefs_.clear();
isAnalyzed_ = false;
generatedKuduProperties_.clear();

View File

@@ -336,3 +336,13 @@ select * from ctas_decimal;
---- TYPES
DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL,DECIMAL
====
---- QUERY
# IMPALA-6954: CTAS with an expr rewrite.
create table ctas_rewrite primary key(id)
partition by range(id) (partition 0 <= values < 100) stored as kudu
as select id, tinyint_col from functional.alltypes
where id between 0 and 1;
show range partitions ctas_rewrite;
---- RESULTS
'0 <= VALUES < 100'
====