diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java index 916de2207..0f6cc6588 100644 --- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java @@ -1106,6 +1106,7 @@ public class Analyzer { if (pathType == PathType.SLOT_REF || pathType == PathType.STAR) { // Paths rooted at all of the unique registered tuple descriptors. for (TableRef tblRef: tableRefMap_.values()) { + if (tblRef.isHidden()) continue; candidates.add(new Path(tblRef.getDesc(), rawPath)); } } else { diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java index 8acdcb665..ef175eb68 100644 --- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java @@ -575,6 +575,7 @@ public class SelectStmt extends QueryStmt { } // expand in From clause order for (TableRef tableRef: fromClause_) { + if (tableRef.isHidden()) continue; if (analyzer_.isSemiJoined(tableRef.getId())) continue; Path resolvedPath = new Path(tableRef.getDesc(), Collections.emptyList()); diff --git a/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java b/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java index b8099ada8..3bc0f399e 100644 --- a/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java +++ b/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java @@ -1827,10 +1827,20 @@ public class StmtRewriter { analyzer.removeAlias(collTblRef.getUniqueAlias()); TableRef newCollTblRef = TableRef.newTableRef(analyzer, newCollPath, collTblRef.getUniqueAlias()); + // Set JOIN attributes. Please note that we cannot use TableRef.setJoinAttrs() + // because we only want to copy the ON clause and the plan hints. The col names + // in USING have been already converted to an ON clause. We let the analyzer/ + // planner to figure out the other attributes. + newCollTblRef.setOnClause(collTblRef.getOnClause()); + newCollTblRef.setJoinHints(collTblRef.getJoinHints()); + newCollTblRef.setTableHints(collTblRef.getTableHints()); // Substitute the old collection ref to 'newCollTblRef'. stmt.fromClause_.set(tableRefIdx, newCollTblRef); // Insert the base table ref in front of the collection ref. stmt.fromClause_.add(tableRefIdx, baseTblRef); + // The newly added table ref should be hidden, e.g. it shouldn't affect star + // expansion, neither column resolution. + baseTblRef.setHidden(true); } private List generatePathFrom(TableName tblName) { diff --git a/fe/src/main/java/org/apache/impala/analysis/TableRef.java b/fe/src/main/java/org/apache/impala/analysis/TableRef.java index 3a9b358f5..d6b312c51 100644 --- a/fe/src/main/java/org/apache/impala/analysis/TableRef.java +++ b/fe/src/main/java/org/apache/impala/analysis/TableRef.java @@ -150,6 +150,10 @@ public class TableRef extends StmtNode { // END: Members that need to be reset() ///////////////////////////////////////// + // true if this table ref is hidden, because e.g. it was generated during statement + // rewrite. + private boolean isHidden_ = false; + /** * Returns a new, resolved, and analyzed table ref. */ @@ -228,6 +232,7 @@ public class TableRef extends StmtNode { desc_ = other.desc_; exposeNestedColumnsByTableMaskView_ = other.exposeNestedColumnsByTableMaskView_; scalarColumns_ = new LinkedHashMap<>(other.scalarColumns_); + isHidden_ = other.isHidden_; } @Override @@ -343,6 +348,9 @@ public class TableRef extends StmtNode { return exposeNestedColumnsByTableMaskView_; } + public void setHidden(boolean isHidden) { isHidden_ = isHidden; } + public boolean isHidden() { return isHidden_; } + public void setJoinHints(List hints) { Preconditions.checkNotNull(hints); joinHints_ = hints; @@ -736,4 +744,4 @@ public class TableRef extends StmtNode { joinHints_ = new ArrayList<>(); tableHints_ = new ArrayList<>(); } -} \ No newline at end of file +} diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java index 7d6c8d177..59c002892 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java @@ -4550,7 +4550,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest { testNumberOfMembers(ValuesStmt.class, 0); // Also check TableRefs. - testNumberOfMembers(TableRef.class, 25); + testNumberOfMembers(TableRef.class, 26); testNumberOfMembers(BaseTableRef.class, 0); testNumberOfMembers(InlineViewRef.class, 9); } diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index 6fafd3fd2..e209ceed3 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -800,6 +800,20 @@ DELETE FROM {db_name}{db_suffix}.{table_name} WHERE id % 2 = 0; ---- DATASET functional ---- BASE_TABLE_NAME +pos_item_key_value_complextypestbl +---- COLUMNS +pos bigint +item int +key string +value int +int_array array +int_map map +---- DEPENDENT_LOAD_HIVE +INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} SELECT id, id, CAST(id AS STRING), CAST(id AS STRING), int_array, int_map FROM {db_name}{db_suffix}.complextypestbl; +==== +---- DATASET +functional +---- BASE_TABLE_NAME complextypestbl_non_transactional ---- COLUMNS id bigint diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv index 66854c80f..ad52f4fcd 100644 --- a/testdata/datasets/functional/schema_constraints.csv +++ b/testdata/datasets/functional/schema_constraints.csv @@ -92,6 +92,8 @@ table_name:complextypestbl_deleted_rows, constraint:restrict_to, table_format:or table_name:complextypestbl_medium, constraint:restrict_to, table_format:parquet/none/none table_name:complextypestbl_medium, constraint:restrict_to, table_format:orc/def/block table_name:complextypestbl_non_transactional, constraint:restrict_to, table_format:orc/def/block +table_name:pos_item_key_value_complextypestbl, constraint:restrict_to, table_format:orc/def/block +table_name:pos_item_key_value_complextypestbl, constraint:restrict_to, table_format:parquet/none/none table_name:alltypeserror, constraint:exclude, table_format:parquet/none/none table_name:alltypeserrornonulls, constraint:exclude, table_format:parquet/none/none diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-scanner-basic.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-scanner-basic.test index 3caca5b7d..266d90461 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-scanner-basic.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-scanner-basic.test @@ -234,3 +234,198 @@ select count(*) from complextypestbl.nested_struct.c.d ---- TYPES bigint ==== +---- QUERY +# IMPALA-10482: Select-star query on unrelative collection column of transactional +# table hits IllegalStateException +select * from complextypestbl.int_array; +---- RESULTS +1 +2 +3 +NULL +1 +2 +NULL +3 +NULL +-1 +---- TYPES +INT +==== +---- QUERY +# IMPALA-10482: Select-star query on unrelative collection column of transactional +# table hits IllegalStateException +select * from complextypestbl.int_array_array.item +---- RESULTS +-1 +-2 +1 +2 +3 +4 +NULL +1 +2 +NULL +3 +NULL +4 +5 +6 +---- TYPES +INT +==== +---- QUERY +# IMPALA-10482: Select-star query on unrelative collection column of transactional +# table hits IllegalStateException +select * from complextypestbl.nested_struct.c.d.item; +---- RESULTS +-1,'nonnullable' +10,'aaa' +-10,'bbb' +11,'c' +NULL,'NULL' +10,'aaa' +NULL,'NULL' +-10,'bbb' +NULL,'NULL' +11,'c' +NULL,'NULL' +NULL,'NULL' +---- TYPES +INT,STRING +==== +---- QUERY +# IMPALA-10482: Select-star query on unrelative collection column of transactional +# table hits IllegalStateException +select * from complextypestbl.int_array a1, complextypestbl.int_array a2 +order by 1,2 +limit 10; +---- RESULTS +-1,-1 +-1,1 +-1,1 +-1,2 +-1,2 +-1,3 +-1,3 +-1,NULL +-1,NULL +-1,NULL +---- TYPES +INT,INT +==== +---- QUERY +select * +from complextypestbl.int_array a1, complextypestbl.int_array a2 +where a1.item=a2.item and a1.item < 2; +---- RESULTS +1,1 +1,1 +1,1 +1,1 +-1,-1 +---- TYPES +INT,INT +==== +---- QUERY +# IMPALA-10493: Use ON clause to join full ACID collections +select * +from complextypestbl.int_array a1 join complextypestbl.int_array a2 +on a1.item=a2.item +where a1.item < 2; +---- RESULTS +1,1 +1,1 +1,1 +1,1 +-1,-1 +---- TYPES +INT,INT +==== +---- QUERY +# IMPALA-10493: Use USING clause to join full ACID collections +select a1.item, a2.item +from complextypestbl.int_array a1 join complextypestbl.int_array a2 +using (item) where a1.item != 1; +---- RESULTS +-1,-1 +2,2 +2,2 +3,3 +3,3 +2,2 +2,2 +3,3 +3,3 +---- TYPES +INT,INT +==== +---- QUERY +select count(*) from complextypestbl.int_array a1, complextypestbl.int_array a2; +---- RESULTS +100 +==== +---- QUERY +select count(*) from complextypestbl.int_array a1, complextypestbl.int_array a2 +where a1.item=a2.item and a1.item > 1; +---- RESULTS +8 +==== +---- QUERY +select pos, item from pos_item_key_value_complextypestbl; +---- RESULTS +1,1 +2,2 +3,3 +4,4 +5,5 +6,6 +7,7 +8,8 +---- TYPES +BIGINT,INT +==== +---- QUERY +select pos, item from pos_item_key_value_complextypestbl.int_array; +---- RESULTS +0,-1 +0,1 +1,2 +2,3 +0,NULL +1,1 +2,2 +3,NULL +4,3 +5,NULL +---- TYPES +BIGINT,INT +==== +---- QUERY +select key, value from pos_item_key_value_complextypestbl; +---- RESULTS +'1',1 +'2',2 +'3',3 +'4',4 +'5',5 +'6',6 +'7',7 +'8',8 +---- TYPES +STRING,INT +==== +---- QUERY +select key, value from pos_item_key_value_complextypestbl.int_map; +---- RESULTS +'k1',-1 +'k1',1 +'k2',100 +'k1',2 +'k2',NULL +'k1',NULL +'k3',NULL +---- TYPES +STRING,INT +====