diff --git a/be/src/codegen/codegen-anyval.cc b/be/src/codegen/codegen-anyval.cc index 92ec2d06b..b9fbc03e8 100644 --- a/be/src/codegen/codegen-anyval.cc +++ b/be/src/codegen/codegen-anyval.cc @@ -666,7 +666,8 @@ void CodegenAnyVal::StoreToNativePtr(llvm::Value* raw_val_ptr, llvm::Value* pool switch (type_.type) { case TYPE_STRING: case TYPE_VARCHAR: - case TYPE_ARRAY: { // CollectionVal has same memory layout as StringVal. + case TYPE_ARRAY: // CollectionVal has same memory layout as StringVal. + case TYPE_MAP: { // CollectionVal has same memory layout as StringVal. // Convert StringVal to StringValue llvm::Value* string_value = llvm::Constant::getNullValue(raw_type); llvm::Value* len = GetLen(); diff --git a/be/src/runtime/raw-value.cc b/be/src/runtime/raw-value.cc index 6acc789a6..eba2b9d40 100644 --- a/be/src/runtime/raw-value.cc +++ b/be/src/runtime/raw-value.cc @@ -411,10 +411,52 @@ template void RawValue::WritePrimitive(const void* value, Tuple* tuple, const SlotDescriptor* slot_desc, MemPool* pool, std::vector* string_values); -void RawValue::PrintArrayValue(const CollectionValue* array_val, - const TupleDescriptor* item_tuple_desc, int scale, stringstream *stream) { +bool PrintNestedValueIfNull(const SlotDescriptor& slot_desc, Tuple* item, + stringstream* stream) { + bool is_null = item->IsNull(slot_desc.null_indicator_offset()); + if (is_null) *stream << "NULL"; + return is_null; +} + +void PrintNonNullNestedCollection(const SlotDescriptor& slot_desc, Tuple* item, int scale, + stringstream* stream) { + const CollectionValue* nested_collection_val = + item->GetCollectionSlot(slot_desc.tuple_offset()); + DCHECK(nested_collection_val != nullptr); + const TupleDescriptor* child_item_tuple_desc = + slot_desc.children_tuple_descriptor(); + DCHECK(child_item_tuple_desc != nullptr); + RawValue::PrintCollectionValue(nested_collection_val, child_item_tuple_desc, scale, + stream, slot_desc.type().IsMapType()); +} + +void PrintNonNullNestedPrimitive(const SlotDescriptor& slot_desc, Tuple* item, int scale, + stringstream* stream) { + RawValue::PrintValue(item->GetSlot(slot_desc.tuple_offset()), slot_desc.type(), scale, + stream, true); +} + +void PrintNestedValue(const SlotDescriptor& slot_desc, Tuple* item, int scale, + stringstream* stream) { + bool is_null = PrintNestedValueIfNull(slot_desc, item, stream); + if (is_null) return; + + if (slot_desc.type().IsCollectionType()) { + // The item is also an array or a map, recurse deeper if not NULL. + PrintNonNullNestedCollection(slot_desc, item, scale, stream); + } else if (!slot_desc.type().IsComplexType()) { + // The item is a scalar, print it with the usual PrintValue. + PrintNonNullNestedPrimitive(slot_desc, item, scale, stream); + } else { + DCHECK(false); + } +} + +void RawValue::PrintCollectionValue(const CollectionValue* coll_val, + const TupleDescriptor* item_tuple_desc, int scale, stringstream *stream, + bool is_map) { DCHECK(item_tuple_desc != nullptr); - if (array_val == nullptr) { + if (coll_val == nullptr) { *stream << "NULL"; return; } @@ -422,42 +464,28 @@ void RawValue::PrintArrayValue(const CollectionValue* array_val, const vector& slot_descs = item_tuple_desc->slots(); // TODO: This has to be changed once structs are supported too. - DCHECK(slot_descs.size() == 1); - DCHECK(slot_descs[0] != nullptr); - - *stream << "["; - if (slot_descs[0]->type().IsArrayType()) { - // The item is also an array, recurse deeper if not NULL. - for (int i = 0; i < array_val->num_tuples; ++i) { - Tuple* item = reinterpret_cast(array_val->ptr + i * item_byte_size); - if (item->IsNull(slot_descs[0]->null_indicator_offset())) { - *stream << "NULL"; - } else { - const CollectionValue* nested_array_val = - item->GetCollectionSlot(slot_descs[0]->tuple_offset()); - const TupleDescriptor* child_item_tuple_desc = - slot_descs[0]->children_tuple_descriptor(); - DCHECK(child_item_tuple_desc != nullptr); - PrintArrayValue(nested_array_val, child_item_tuple_desc, scale, stream); - } - if (i < array_val->num_tuples - 1) *stream << ","; - } - } else if (!slot_descs[0]->type().IsComplexType()) { - // The item is a scalar, print it with the usual PrintValue. - for (int i = 0; i < array_val->num_tuples; ++i) { - Tuple* item = reinterpret_cast(array_val->ptr + i * item_byte_size); - if (item->IsNull(slot_descs[0]->null_indicator_offset())) { - *stream << "NULL"; - } else { - PrintValue(item->GetSlot(slot_descs[0]->tuple_offset()), slot_descs[0]->type(), - scale, stream, true); - } - if (i < array_val->num_tuples - 1) *stream << ","; - } + if (is_map) { + DCHECK(slot_descs.size() == 2); + DCHECK(slot_descs[0] != nullptr); + DCHECK(slot_descs[1] != nullptr); } else { - DCHECK(false); + DCHECK(slot_descs.size() == 1); + DCHECK(slot_descs[0] != nullptr); } - *stream << "]"; + + *stream << (is_map ? "{" : "["); + for (int i = 0; i < coll_val->num_tuples; ++i) { + Tuple* item = reinterpret_cast(coll_val->ptr + i * item_byte_size); + + PrintNestedValue(*slot_descs[0], item, scale, stream); + if (is_map) { + *stream << ":"; + PrintNestedValue(*slot_descs[1], item, scale, stream); + } + + if (i < coll_val->num_tuples - 1) *stream << ","; + } + *stream << (is_map ? "}" : "]"); } } diff --git a/be/src/runtime/raw-value.h b/be/src/runtime/raw-value.h index 78dfb15ad..8b99e7c25 100644 --- a/be/src/runtime/raw-value.h +++ b/be/src/runtime/raw-value.h @@ -67,10 +67,11 @@ class RawValue { return str; } - /// Similar to PrintValue() but works with array values. - /// Converts 'array_val' array into ascii and writes to 'stream'. - static void PrintArrayValue(const CollectionValue* array_val, - const TupleDescriptor* item_tuple_desc, int scale, std::stringstream *stream); + /// Similar to PrintValue() but works with collection values. + /// Converts 'coll_val' collection into ascii and writes to 'stream'. + static void PrintCollectionValue(const CollectionValue* coll_val, + const TupleDescriptor* item_tuple_desc, int scale, std::stringstream *stream, + bool is_map); /// Writes the byte representation of a value to a stringstream character-by-character static void PrintValueAsBytes(const void* value, const ColumnType& type, diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h index a8c68663e..3a4e95e86 100644 --- a/be/src/runtime/types.h +++ b/be/src/runtime/types.h @@ -247,6 +247,7 @@ struct ColumnType { } inline bool IsArrayType() const { return type == TYPE_ARRAY; } + inline bool IsMapType() const { return type == TYPE_MAP; } inline bool IsVarLenType() const { return IsVarLenStringType() || IsCollectionType(); diff --git a/be/src/service/hs2-util.cc b/be/src/service/hs2-util.cc index ce9178e9a..ebb059662 100644 --- a/be/src/service/hs2-util.cc +++ b/be/src/service/hs2-util.cc @@ -421,9 +421,10 @@ static void StructExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval, } } -static void ArrayExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval, +static void CollectionExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval, const TColumnType& type, RowBatch* batch, int start_idx, int num_rows, - uint32_t output_row_idx, apache::hive::service::cli::thrift::TColumn* column) { + uint32_t output_row_idx, apache::hive::service::cli::thrift::TColumn* column, + bool is_map) { DCHECK(type.types.size() > 1); ReserveSpace(num_rows, output_row_idx, &column->stringVal); FOREACH_ROW_LIMIT(batch, start_idx, num_rows, it) { @@ -439,7 +440,7 @@ static void ArrayExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval, CollectionValue value(coll_val); // TODO: use rapidjson as in for structs stringstream stream; - RawValue::PrintArrayValue(&value, item_tuple_desc, -1, &stream); + RawValue::PrintCollectionValue(&value, item_tuple_desc, -1, &stream, is_map); column->stringVal.values.emplace_back(stream.str()); } SetNullBit(output_row_idx, coll_val.is_null, &column->stringVal.nulls); @@ -461,8 +462,12 @@ void impala::ExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval, expr_eval, type, batch, start_idx, num_rows, output_row_idx, column); return; case TTypeNodeType::ARRAY: - ArrayExprValuesToHS2TColumn( - expr_eval, type, batch, start_idx, num_rows, output_row_idx, column); + CollectionExprValuesToHS2TColumn( + expr_eval, type, batch, start_idx, num_rows, output_row_idx, column, false); + return; + case TTypeNodeType::MAP: + CollectionExprValuesToHS2TColumn( + expr_eval, type, batch, start_idx, num_rows, output_row_idx, column, true); return; default: break; @@ -939,6 +944,7 @@ thrift::TTypeEntry impala::ColumnToHs2Type( } case TYPE_STRUCT: case TYPE_ARRAY: + case TYPE_MAP: type_entry.__set_type(thrift::TTypeId::STRING_TYPE); break; case TYPE_BINARY: diff --git a/be/src/service/impala-beeswax-server.cc b/be/src/service/impala-beeswax-server.cc index bd4a3b4ae..e73a41e11 100644 --- a/be/src/service/impala-beeswax-server.cc +++ b/be/src/service/impala-beeswax-server.cc @@ -206,7 +206,8 @@ string ImpalaServer::ColumnTypeToBeeswaxTypeString(const TColumnType& type) { DCHECK_EQ(TTypeNodeType::SCALAR, type.types[0].type); DCHECK(type.types[0].__isset.scalar_type); return TypeToOdbcString(type); - } else if (type.types[0].type == TTypeNodeType::ARRAY) { + } else if (type.types[0].type == TTypeNodeType::ARRAY + || type.types[0].type == TTypeNodeType::MAP) { DCHECK_GT(type.types.size(), 1); // TODO (IMPALA-11041): consider returning the real type return "string"; @@ -220,7 +221,7 @@ string ImpalaServer::ColumnTypeToBeeswaxTypeString(const TColumnType& type) { } } -// TODO: Handle complex types. +// TODO: Handle struct types. void ImpalaServer::get_results_metadata(ResultsMetadata& results_metadata, const beeswax::QueryHandle& beeswax_handle) { ScopedSessionState session_handle(this); diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h index bff9166d1..37f41546b 100644 --- a/be/src/service/impala-server.h +++ b/be/src/service/impala-server.h @@ -886,8 +886,8 @@ class ImpalaServer : public ImpalaServiceIf, /// Initializes the backend descriptor in 'be_desc' with the local backend information. void BuildLocalBackendDescriptorInternal(BackendDescriptorPB* be_desc); - /// Converts a type to a string. The only complex type supported is array which - /// is returned as string (see IMPALA-11041). + /// Converts a type to a string. The only complex types supported are array and map, + /// which are returned as strings (see IMPALA-11041). std::string ColumnTypeToBeeswaxTypeString(const TColumnType& type); /// Snapshot of a query's state, archived in the query log. Not mutated after diff --git a/be/src/service/query-result-set.cc b/be/src/service/query-result-set.cc index 19f2b73d7..c67f416eb 100644 --- a/be/src/service/query-result-set.cc +++ b/be/src/service/query-result-set.cc @@ -197,11 +197,9 @@ Status AsciiQueryResultSet::AddRows(const vector& expr_eva &out_stream); } else if (metadata_.columns[i].columnType.types.size() > 1) { ColumnType col_type = ColumnType::FromThrift(metadata_.columns[i].columnType); - // TODO: Implement map (IMPALA-10918) type. - DCHECK(col_type.IsArrayType()); - if (col_type.IsArrayType()) { - PrintArrayValue(expr_evals[i], it.Get(), scales[i], &out_stream); - } + DCHECK(col_type.IsArrayType() || col_type.IsMapType()); + PrintCollectionValue(expr_evals[i], it.Get(), scales[i], &out_stream, + col_type.IsMapType()); } else { DCHECK(false); } @@ -212,8 +210,8 @@ Status AsciiQueryResultSet::AddRows(const vector& expr_eva return Status::OK(); } -void QueryResultSet::PrintArrayValue(ScalarExprEvaluator* expr_eval, - const TupleRow* row, int scale, stringstream *stream) { +void QueryResultSet::PrintCollectionValue(ScalarExprEvaluator* expr_eval, + const TupleRow* row, int scale, stringstream *stream, bool is_map) { const ScalarExpr& scalar_expr = expr_eval->root(); // Currently scalar_expr can be only a slot ref as no functions return arrays. DCHECK(scalar_expr.IsSlotRef()); @@ -222,7 +220,7 @@ void QueryResultSet::PrintArrayValue(ScalarExprEvaluator* expr_eval, const CollectionValue* array_val = static_cast(expr_eval->GetValue(row)); - RawValue::PrintArrayValue(array_val, item_tuple_desc, scale, stream); + RawValue::PrintCollectionValue(array_val, item_tuple_desc, scale, stream, is_map); } Status AsciiQueryResultSet::AddOneRow(const TResultRow& row) { @@ -447,8 +445,9 @@ void HS2ColumnarResultSet::InitColumns() { DCHECK(type_nodes.size() > 0); ThriftTColumn col_output; if (type_nodes[0].type == TTypeNodeType::STRUCT - || type_nodes[0].type == TTypeNodeType::ARRAY) { - // Return structs and arrays as string. + || type_nodes[0].type == TTypeNodeType::ARRAY + || type_nodes[0].type == TTypeNodeType::MAP) { + // Return structs, arrays and maps as string. col_output.__isset.stringVal = true; } else { DCHECK(type_nodes.size() == 1); diff --git a/be/src/service/query-result-set.h b/be/src/service/query-result-set.h index 282f7f615..176e18c48 100644 --- a/be/src/service/query-result-set.h +++ b/be/src/service/query-result-set.h @@ -78,10 +78,10 @@ class QueryResultSet { apache::hive::service::cli::thrift::TRowSet* rowset); protected: - /// Wrapper to call RawValue::PrintArrayValue for a given collection column. + /// Wrapper to call RawValue::PrintCollectionValue for a given collection column. /// expr_eval must be a SlotRef on a collection slot. - static void PrintArrayValue(ScalarExprEvaluator* expr_eval, const TupleRow* row, - int scale, std::stringstream *stream); + static void PrintCollectionValue(ScalarExprEvaluator* expr_eval, const TupleRow* row, + int scale, std::stringstream *stream, bool is_map); }; } diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java index 25b880f1f..91f27b470 100644 --- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java @@ -1465,8 +1465,8 @@ public class Analyzer { SlotDescriptor existingSlotDesc = slotPathMap_.get(key); if (existingSlotDesc != null) return existingSlotDesc; - if (slotPath.destType().isArrayType()) { - SlotDescriptor result = registerArraySlotRef(slotPath); + if (slotPath.destType().isCollectionType()) { + SlotDescriptor result = registerCollectionSlotRef(slotPath); result.setPath(slotPath); slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), result); registerColumnPrivReq(result); @@ -1652,13 +1652,13 @@ public class Analyzer { } /** - * Registers an array and its descendants. + * Registers a collection and its descendants. * Creates a CollectionTableRef for all collections on the path. */ - private SlotDescriptor registerArraySlotRef(Path slotPath) + private SlotDescriptor registerCollectionSlotRef(Path slotPath) throws AnalysisException { Preconditions.checkState(slotPath.isResolved()); - Preconditions.checkState(slotPath.destType().isArrayType()); + Preconditions.checkState(slotPath.destType().isCollectionType()); List rawPath = slotPath.getRawPath(); List collectionTableRawPath = new ArrayList<>(); TupleDescriptor rootDesc = slotPath.getRootDesc(); @@ -1678,28 +1678,38 @@ public class Analyzer { SlotDescriptor desc = ((SlotRef) collTblRef.getCollectionExpr()).getDesc(); desc.setIsMaterializedRecursively(true); - // Resolve path - List rawPathToItem = new ArrayList(); - rawPathToItem.add(Path.ARRAY_ITEM_FIELD_NAME); + if (slotPath.destType().isArrayType()) { + // Resolve path + List rawPathToItem = Arrays.asList(Path.ARRAY_ITEM_FIELD_NAME); + resolveAndRegisterDescendantPath(collTblRef, rawPathToItem); + } else { + Preconditions.checkState(slotPath.destType().isMapType()); - Path resolvedPathToItem = new Path(collTblRef.getDesc(), rawPathToItem); - boolean isResolved = resolvedPathToItem.resolve(); + List rawPathToKey = Arrays.asList(Path.MAP_KEY_FIELD_NAME); + resolveAndRegisterDescendantPath(collTblRef, rawPathToKey); + + List rawPathToValue = Arrays.asList(Path.MAP_VALUE_FIELD_NAME); + resolveAndRegisterDescendantPath(collTblRef, rawPathToValue); + } + + return desc; + } + + private void resolveAndRegisterDescendantPath(CollectionTableRef collTblRef, + List rawPath) throws AnalysisException { + Path resolvedPath = new Path(collTblRef.getDesc(), rawPath); + boolean isResolved = resolvedPath.resolve(); Preconditions.checkState(isResolved); - if (resolvedPathToItem.destType().isStructType()) { + if (resolvedPath.destType().isStructType()) { throw new AnalysisException( "STRUCT type inside collection types is not supported."); } - if (resolvedPathToItem.destType().isMapType()) { - throw new AnalysisException( - "MAP type inside collection types is not supported."); - } - if (resolvedPathToItem.destType().isBinary()) { + if (resolvedPath.destType().isBinary()) { throw new AnalysisException( "Binary type inside collection types is not supported (IMPALA-11491)."); } - registerSlotRef(resolvedPathToItem, false); - return desc; + registerSlotRef(resolvedPath, false); } /** diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java index fb65f2a26..944348e5c 100644 --- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java +++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java @@ -304,7 +304,7 @@ public class InlineViewRef extends TableRef { analyzer.createAuxEqPredicate(new SlotRef(slotDesc), colExpr.clone()); } - if (colExpr.getType().isArrayType()) { + if (colExpr.getType().isCollectionType()) { // Calling registerSlotRef() above created a new SlotDescriptor + TupleDescriptor // hierarchy for Array types. Walk through this hiararchy and add all slot refs // to smap_ and baseTblSmap_. @@ -322,22 +322,26 @@ public class InlineViewRef extends TableRef { if (itemTupleDesc != null) { Preconditions.checkState(srcItemTupleDesc != null); Preconditions.checkState(baseTableItemTupleDesc != null); - // Assume that there is only a single slot as struct in nested in - // arrays is not yet supported in select lists. - Preconditions.checkState(itemTupleDesc.getSlots().size() == 1); - Preconditions.checkState(srcItemTupleDesc.getSlots().size() == 1); - Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() == 1); - SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(0); - SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(0); - SlotDescriptor baseTableItemSlotDesc = baseTableItemTupleDesc.getSlots().get(0); - SlotRef itemSlotRef = new SlotRef(itemSlotDesc); - SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc); - SlotRef beseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc); - smap_.put(itemSlotRef, srcItemSlotRef); - baseTblSmap_.put(itemSlotRef, beseTableItemSlotRef); - if (createAuxPredicate(colExpr)) { - analyzer.createAuxEqPredicate( - new SlotRef(itemSlotDesc), srcItemSlotRef.clone()); + + final int num_slots = itemTupleDesc.getSlots().size(); + Preconditions.checkState(srcItemTupleDesc.getSlots().size() == num_slots); + Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() == num_slots); + + // There is one slot for arrays and two for maps. When we add support for structs + // in collections in the select list, there may be more slots. + for (int i = 0; i < num_slots; i++) { + SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(i); + SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(i); + SlotDescriptor baseTableItemSlotDesc = baseTableItemTupleDesc.getSlots().get(i); + SlotRef itemSlotRef = new SlotRef(itemSlotDesc); + SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc); + SlotRef beseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc); + smap_.put(itemSlotRef, srcItemSlotRef); + baseTblSmap_.put(itemSlotRef, beseTableItemSlotRef); + if (createAuxPredicate(colExpr)) { + analyzer.createAuxEqPredicate( + new SlotRef(itemSlotDesc), srcItemSlotRef.clone()); + } } } } diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java index 8c2b4dcfc..8774fbb39 100644 --- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java @@ -39,6 +39,7 @@ import org.apache.impala.catalog.FeIcebergTable.Utils; import org.apache.impala.catalog.FeKuduTable; import org.apache.impala.catalog.FeTable; import org.apache.impala.catalog.FeView; +import org.apache.impala.catalog.MapType; import org.apache.impala.catalog.StructField; import org.apache.impala.catalog.StructType; import org.apache.impala.catalog.TableLoadingException; @@ -488,10 +489,12 @@ public class SelectStmt extends QueryStmt { expr.getType().toSql() + "' in '" + expr.toSql() + "'."); } } else if (expr.getType().isMapType()) { - throw new AnalysisException(String.format( - "Expr '%s' in select list returns a map type '%s'.\n" + - "Map type is not allowed in the select list.", - expr.toSql(), expr.getType().toSql())); + MapType mapType = (MapType) expr.getType(); + if (!mapType.getKeyType().isSupported() + || !mapType.getValueType().isSupported()) { + throw new AnalysisException("Unsupported type '" + + expr.getType().toSql() + "' in '" + expr.toSql() + "'."); + } } else if (expr.getType().isStructType()) { if (!analyzer_.getQueryCtx().client_request.query_options.disable_codegen) { throw new AnalysisException("Struct type in select list is not allowed " + diff --git a/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java b/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java index c8d6b943e..ccf81c30b 100644 --- a/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java @@ -627,7 +627,7 @@ public class SetOperationStmt extends QueryStmt { SlotDescriptor slotDesc = analyzer.addSlotDescriptor(tupleDesc); slotDesc.setLabel(getColLabels().get(i)); slotDesc.setType(expr.getType()); - if (expr.getType().isArrayType()) { + if (expr.getType().isCollectionType()) { slotDesc.setItemTupleDesc(((SlotRef)expr).getDesc().getItemTupleDesc()); } slotDesc.setStats(columnStats.get(i)); diff --git a/fe/src/main/java/org/apache/impala/catalog/Type.java b/fe/src/main/java/org/apache/impala/catalog/Type.java index 1068ab8d6..7e4c2fb6f 100644 --- a/fe/src/main/java/org/apache/impala/catalog/Type.java +++ b/fe/src/main/java/org/apache/impala/catalog/Type.java @@ -351,7 +351,7 @@ public abstract class Type { * casting between any two types other than both decimals. * * - * TODO: Support map and struct types. + * TODO: Support struct types. */ public static Type getAssignmentCompatibleType( Type t1, Type t2, boolean strict, boolean strictDecimal) { @@ -361,6 +361,9 @@ public abstract class Type { } else if (t1.isArrayType() && t2.isArrayType()) { // Only support exact match for array types. if (t1.equals(t2)) return t2; + } else if (t1.isMapType() && t2.isMapType()) { + // Only support exact match for map types. + if (t1.equals(t2)) return t2; } return ScalarType.INVALID; } diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java index 87df7da18..30dfb15be 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java @@ -2292,8 +2292,7 @@ public class AnalyzeDDLTest extends FrontendTestBase { "Cannot create table 't': Type CHAR(5) is not supported in Kudu"); AnalysisError("create table t primary key (id) partition by hash partitions 3" + " stored as kudu as select id, m from functional.complextypes_fileformat", - "Expr 'm' in select list returns a map type 'MAP'.\n" + - "Map type is not allowed in the select list."); + "Cannot create table 't': Type MAP is not supported in Kudu"); AnalysisError("create table t primary key (id) partition by hash partitions 3" + " stored as kudu as select id, a from functional.complextypes_fileformat", "Cannot create table 't': Type ARRAY is not supported in Kudu"); @@ -3092,13 +3091,9 @@ public class AnalyzeDDLTest extends FrontendTestBase { AnalyzesOk("create view functional.foo (a) as " + "select int_array_col " + "from functional.allcomplextypes"); - // View cannot have map typed columns because map-typed exprs are - // not supported in the select list. - AnalysisError("create view functional.foo (a) as " + + AnalyzesOk("create view functional.foo (a) as " + "select int_map_col " + - "from functional.allcomplextypes", - "Expr 'int_map_col' in select list returns a map type 'MAP'.\n" + - "Map type is not allowed in the select list."); + "from functional.allcomplextypes"); // It's allowed to do the same with struct as it is supported in the select list. AnalysisContext ctx = createAnalysisCtx(); // TODO: Turning Codegen OFF could be removed once the Codegen support is implemented diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java index 1dcabdc15..8ad03e6f7 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java @@ -120,10 +120,5 @@ public class AnalyzeUpsertStmtTest extends AnalyzerTest { "'b.int_array_col' correlated with an outer block as well as an " + "uncorrelated one 'functional.alltypestiny':\n" + "SELECT item FROM b.int_array_col, functional.alltypestiny"); - // Illegal map-typed expr - AnalysisError("upsert into functional_kudu.testtbl " + - "select int_map_col from functional.allcomplextypes", - "Expr 'int_map_col' in select list returns a map type 'MAP'.\n" + - "Map type is not allowed in the select list."); } } diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java index ad91500eb..0fcae9d30 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java @@ -303,15 +303,12 @@ public class AnalyzerTest extends FrontendTestBase { "select binary_member_col from functional_parquet.binary_in_complex_types", "Struct containing a BINARY type is not allowed in the select list " + "(IMPALA-11491)."); - // TODO: change error message once IMPALA-10918 is finished. AnalysisError( "select binary_key_col from functional_parquet.binary_in_complex_types", - "Expr 'binary_key_col' in select list returns a map type 'MAP'." + - "\nMap type is not allowed in the select list."); + "Binary type inside collection types is not supported (IMPALA-11491)."); AnalysisError( "select binary_value_col from functional_parquet.binary_in_complex_types", - "Expr 'binary_value_col' in select list returns a map type 'MAP'." + - "\nMap type is not allowed in the select list."); + "Binary type inside collection types is not supported (IMPALA-11491)."); for (ScalarType t: Type.getUnsupportedTypes()) { // Create/Alter table. diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java index 0bb4e39fd..b1b27ee2b 100644 --- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java +++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java @@ -718,6 +718,40 @@ public class AuthorizationStmtTest extends AuthorizationTestBase { .error(selectError("functional.allcomplextypes"), onColumn("functional", "allcomplextypes", new String[]{"array_array_col"}, TPrivilegeLevel.SELECT)); + // Select with map types in select list. + authorize("select int_map_col, array_map_col, map_map_col " + + "from functional.allcomplextypes") + .ok(onServer(TPrivilegeLevel.ALL)) + .ok(onServer(TPrivilegeLevel.OWNER)) + .ok(onServer(TPrivilegeLevel.SELECT)) + .ok(onDatabase("functional", TPrivilegeLevel.ALL)) + .ok(onDatabase("functional", TPrivilegeLevel.OWNER)) + .ok(onDatabase("functional", TPrivilegeLevel.SELECT)) + .ok(onTable("functional", "allcomplextypes", TPrivilegeLevel.ALL)) + .ok(onTable("functional", "allcomplextypes", TPrivilegeLevel.OWNER)) + .ok(onTable("functional", "allcomplextypes", TPrivilegeLevel.SELECT)) + .ok(onColumn("functional", "allcomplextypes", + new String[]{"int_map_col", "array_map_col", "map_map_col"}, + TPrivilegeLevel.SELECT)) + .error(selectError("functional.allcomplextypes")) + .error(selectError("functional.allcomplextypes"), onServer(allExcept( + TPrivilegeLevel.ALL, TPrivilegeLevel.OWNER, TPrivilegeLevel.SELECT))) + .error(selectError("functional.allcomplextypes"), onDatabase( + "functional", allExcept(TPrivilegeLevel.ALL, TPrivilegeLevel.OWNER, + TPrivilegeLevel.SELECT))) + .error(selectError("functional.allcomplextypes"), onTable("functional", + "allcomplextypes", allExcept(TPrivilegeLevel.ALL, TPrivilegeLevel.OWNER, + TPrivilegeLevel.SELECT))) + .error(selectError("functional.allcomplextypes"), onColumn("functional", + "allcomplextypes", new String[]{"int_map_col"}, TPrivilegeLevel.SELECT)) + .error(selectError("functional.allcomplextypes"), onColumn("functional", + "allcomplextypes", new String[]{"array_map_col"}, TPrivilegeLevel.SELECT)) + .error(selectError("functional.allcomplextypes"), onColumn("functional", + "allcomplextypes", new String[]{"map_map_col"}, TPrivilegeLevel.SELECT)) + .error(selectError("functional.allcomplextypes"), onColumn("functional", + "allcomplextypes", new String[]{"int_map_col", "map_map_col"}, + TPrivilegeLevel.SELECT)); + for (AuthzTest authzTest: new AuthzTest[]{ // Select with cross join. authorize("select * from functional.alltypes union all " + diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql index fdb5b6f48..4f1ff5cd0 100644 --- a/testdata/datasets/functional/functional_schema_template.sql +++ b/testdata/datasets/functional/functional_schema_template.sql @@ -3430,6 +3430,16 @@ AS SELECT id, int_array, int_array_array FROM {db_name}{db_suffix}.complextypest ---- DATASET functional ---- BASE_TABLE_NAME +complextypes_maps_view +---- CREATE +DROP VIEW IF EXISTS {db_name}{db_suffix}.{table_name}; +CREATE VIEW {db_name}{db_suffix}.{table_name} +AS SELECT id, int_map, int_map_array FROM {db_name}{db_suffix}.complextypestbl; +---- LOAD +==== +---- DATASET +functional +---- BASE_TABLE_NAME iceberg_v2_delete_positional ---- CREATE CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} @@ -3636,26 +3646,40 @@ select count(*) as mv_count from {db_name}{db_suffix}.{table_name}; ---- DATASET functional ---- BASE_TABLE_NAME -array_tbl +collection_tbl ---- COLUMNS id INT -int_1d ARRAY -int_2d ARRAY> -int_3d ARRAY>> -string_1d ARRAY -string_2d ARRAY> -string_3d ARRAY>> +arr_int_1d ARRAY +arr_int_2d ARRAY> +arr_int_3d ARRAY>> +arr_string_1d ARRAY +arr_string_2d ARRAY> +arr_string_3d ARRAY>> +map_1d MAP +map_2d MAP> +map_3d MAP>> +map_map_array MAP>> ---- DEPENDENT_LOAD_HIVE -- It would be nice to insert NULLs, but I couldn't find a way in Hive. INSERT INTO {db_name}{db_suffix}.{table_name} VALUES - (1, - array(1, 2, NULL), - array(array(1, 2, NULL), array(3)), - array(array(array(1, 2, NULL), array(3)), array(array(4))), - array("1", "2", NULL), - array(array("1", "2", NULL), array("3")), - array(array(array("1", "2", NULL), array("3")), array(array("4"))) - ); + (1, + array(1, 2, NULL), + array(array(1, 2, NULL), array(3)), + array(array(array(1, 2, NULL), array(3)), array(array(4))), + array("1", "2", NULL), + array(array("1", "2", NULL), array("3")), + array(array(array("1", "2", NULL), array("3")), array(array("4"))), + map(1, "first", 2, "second"), + map(1, map(10, "ten", 20, "twenty"), 2, map(30, "thirty", 40, "forty")), + map( + 1, map(10, map(100, "hundred", 200, "two hundred"), 20, map(300, "three hundred", 400, "four hundred")), + 2, map(30, map(500, "five hundred", 600, "six hundred"), 40, map(700, "seven hundred", 800, "eight hundred")) + ), + map( + 1, map(10, array(100, 200), 20, array(300, 400)), + 2, map(30, array(500, 600), 40, array(700, 800)) + ) + ); ---- LOAD ==== ---- DATASET diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv index c42602db0..7b3a265be 100644 --- a/testdata/datasets/functional/schema_constraints.csv +++ b/testdata/datasets/functional/schema_constraints.csv @@ -344,8 +344,11 @@ table_name:alltypessmall_bool_sorted, constraint:restrict_to, table_format:orc/d table_name:complextypes_arrays_only_view, constraint:restrict_to, table_format:parquet/none/none table_name:complextypes_arrays_only_view, constraint:restrict_to, table_format:orc/def/block -table_name:array_tbl, constraint:restrict_to, table_format:parquet/none/none -table_name:array_tbl, constraint:restrict_to, table_format:orc/def/block +table_name:collection_tbl, constraint:restrict_to, table_format:parquet/none/none +table_name:collection_tbl, constraint:restrict_to, table_format:orc/def/block + +table_name:complextypes_maps_view, constraint:restrict_to, table_format:parquet/none/none +table_name:complextypes_maps_view, constraint:restrict_to, table_format:orc/def/block # 'alltypestiny_negative' only used in ORC tests. table_name:alltypestiny_negative, constraint:restrict_to, table_format:orc/def/block diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test index 8904119d6..14c1c54d1 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test @@ -386,7 +386,7 @@ INT,INT,STRING ---- QUERY # Regression test for: # IMPALA-11434: "More than 1 2d arrays in select list causes analysis error" -select id, int_1d, int_2d, int_3d, string_1d, string_2d, string_3d from array_tbl; +select id, arr_int_1d, arr_int_2d, arr_int_3d, arr_string_1d, arr_string_2d, arr_string_3d from collection_tbl; ---- RESULTS 1,'[1,2,NULL]','[[1,2,NULL],[3]]','[[[1,2,NULL],[3]],[[4]]]','["1","2",NULL]','[["1","2",NULL],["3"]]','[[["1","2",NULL],["3"]],[["4"]]]' ---- TYPES diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test new file mode 100644 index 000000000..622bac4b1 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test @@ -0,0 +1,326 @@ +==== +---- QUERY +select id, int_map from complextypestbl +---- RESULTS +1,'{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}' +3,'{}' +4,'{}' +5,'{}' +6,'NULL' +7,'{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}' +---- TYPES +bigint,string +==== +---- QUERY +select id, int_map from complextypestbl where id=1 +---- RESULTS +1,'{"k1":1,"k2":100}' +---- TYPES +bigint,string +==== +---- QUERY +select id, int_map, int_map_array from complextypestbl +---- RESULTS +1,'{"k1":1,"k2":100}','[{"k1":1}]' +2,'{"k1":2,"k2":NULL}','[{"k3":NULL,"k1":1},NULL,{}]' +3,'{}','[NULL,NULL]' +4,'{}','[]' +5,'{}','NULL' +6,'NULL','NULL' +7,'{"k1":NULL,"k3":NULL}','NULL' +8,'{"k1":-1}','[{},{"k1":1},{},{}]' +---- TYPES +bigint,string,string +==== +---- QUERY +# Sorting is not supported yet for collections: IMPALA-10939 +select id, int_map_array, int_map from complextypestbl order by id +---- CATCH +IllegalStateException: Sorting is not supported if the select list contains collection columns. +==== +---- QUERY +# Same collection used twice in a select list. +select id, int_map, int_map from complextypestbl +---- RESULTS +1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}' +3,'{}','{}' +4,'{}','{}' +5,'{}','{}' +6,'NULL','NULL' +7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}','{"k1":-1}' +---- TYPES +bigint,string,string +==== +---- QUERY +# Same collection used from two versions of the same table/ +select t1.id, t1.int_map, t2.int_map + from complextypestbl t1 join complextypestbl t2 + on t1.id = t2.id +---- RESULTS +1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}' +3,'{}','{}' +4,'{}','{}' +5,'{}','{}' +6,'NULL','NULL' +7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}','{"k1":-1}' +---- TYPES +bigint,string,string +==== +---- QUERY +select id, int_map from complextypestbl union all select id, int_map from complextypestbl +---- RESULTS +1,'{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}' +3,'{}' +4,'{}' +5,'{}' +6,'NULL' +7,'{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}' +1,'{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}' +3,'{}' +4,'{}' +5,'{}' +6,'NULL' +7,'{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}' +---- TYPES +bigint,string +==== +---- QUERY +# TODO: only UNION ALL is supported. UNION needs several utility functions in the BE, so +# for now we reject it in the FE. +select id, int_map from complextypestbl union select id, int_map from complextypestbl; +---- CATCH +IllegalStateException: UNION, EXCEPT and INTERSECT are not supported for collection types +==== +---- QUERY +# Changing a column to a different type leads "non-pass-through" union that does a +# deepcopy on the tuple, which is not yet implemented in BE for arrays. This case is +# currently caught in the planner. +select id, int_map from complextypestbl + union all select cast(id as tinyint), int_map from complextypestbl +---- CATCH +IllegalStateException: only pass-through UNION ALL is supported for array columns +==== +---- QUERY +# Constants in the select list of unions also lead to "non-pass-through" union. +select 1, int_map from complextypestbl + union all select 2, int_map from complextypestbl; +---- CATCH +IllegalStateException: only pass-through UNION ALL is supported for array columns +==== +---- QUERY +select 1 from (select int_map from complextypestbl) s +---- RESULTS +1 +1 +1 +1 +1 +1 +1 +1 +---- TYPES +tinyint +==== +---- QUERY +select id, int_map from (select id, int_map from complextypestbl) s; +---- RESULTS +1,'{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}' +3,'{}' +4,'{}' +5,'{}' +6,'NULL' +7,'{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}' +---- TYPES +bigint,string +==== +---- QUERY +with s as (select id, t.int_map from complextypestbl t) +select id, int_map from s; +---- RESULTS +1,'{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}' +3,'{}' +4,'{}' +5,'{}' +6,'NULL' +7,'{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}' +---- TYPES +bigint,string +==== +---- QUERY +select id, int_map from complextypes_maps_view; +---- RESULTS +1,'{"k1":1,"k2":100}' +2,'{"k1":2,"k2":NULL}' +3,'{}' +4,'{}' +5,'{}' +6,'NULL' +7,'{"k1":NULL,"k3":NULL}' +8,'{"k1":-1}' +---- TYPES +bigint,string +==== +---- QUERY +# Unnesting map returned by view. +select id, m.key, m.value from complextypes_maps_view v, v.int_map m; +---- RESULTS +1,'k1',1 +1,'k2',100 +2,'k1',2 +2,'k2',NULL +7,'k1',NULL +7,'k3',NULL +8,'k1',-1 +---- TYPES +bigint,string,int +==== +---- QUERY +# Unnesting map returned from WITH clause and predicate in inner query. +with v as (select id, int_map from complextypestbl where id=1) +select v.id, a.key, a.value from v, v.int_map a; +---- RESULTS +1,'k1',1 +1,'k2',100 +---- TYPES +bigint,string,int +==== +---- QUERY +# Unnesting map returned from WITH clause and predicate in outer query. +with v as (select id, int_map from complextypestbl) +select v.id, a.key, a.value from v, v.int_map a where id=1; +---- RESULTS +1,'k1',1 +1,'k2',100 +---- TYPES +bigint,string,int +==== +---- QUERY +# Unnesting map returned from WITH clause on item. +with v as (select id, int_map from complextypestbl) +select v.id, a.key, a.value from v, v.int_map a where a.key='k1' +---- RESULTS +1,'k1',1 +2,'k1',2 +7,'k1',NULL +8,'k1',-1 +---- TYPES +bigint,string,int +==== +---- QUERY +# Unnesting map returned by view wrapped in inline view. +select v.id, a.key, a.value from + (select id, int_map from complextypes_maps_view) v, v.int_map a; +---- RESULTS +1,'k1',1 +1,'k2',100 +2,'k1',2 +2,'k2',NULL +7,'k1',NULL +7,'k3',NULL +8,'k1',-1 +---- TYPES +bigint,string,int +==== +---- QUERY +# Unnesting map returned by view wrapped in inline view + WITH clause. +with v2 as (select id, int_map from complextypes_maps_view) +select v.id, a.key, a.value from (select id, int_map from v2) v, v.int_map a; +---- RESULTS +1,'k1',1 +1,'k2',100 +2,'k1',2 +2,'k2',NULL +7,'k1',NULL +7,'k3',NULL +8,'k1',-1 +---- TYPES +bigint,string,int +==== +---- QUERY +# Unnesting map returned by view wrapped in inline view + WITH clause. +with v2 as (select id, int_map from complextypes_maps_view) +select v.id, a.key, a.value from + (select id, int_map from v2 where id=1) v, v.int_map a + where a.key='k1'; +---- RESULTS +1,'k1',1 +---- TYPES +bigint,string,int +==== +---- QUERY +select item from unnest(complextypestbl.int_map_array) +---- RESULTS +'{"k1":1}' +'{"k3":NULL,"k1":1}' +'NULL' +'{}' +'NULL' +'NULL' +'{}' +'{"k1":1}' +'{}' +'{}' +---- TYPES +string +==== +---- QUERY +select id, a.key, a.value from complextypes_maps_view t left join t.int_map a; +---- RESULTS +1,'k1',1 +1,'k2',100 +2,'k1',2 +2,'k2',NULL +3,'NULL',NULL +4,'NULL',NULL +5,'NULL',NULL +6,'NULL',NULL +7,'k1',NULL +7,'k3',NULL +8,'k1',-1 +---- TYPES +BIGINT,STRING,INT +==== +---- QUERY +select id, a2.key, a2.value from complextypes_maps_view v, v.int_map_array a1, a1.item a2; +---- RESULTS +1,'k1',1 +2,'k3',NULL +2,'k1',1 +8,'k1',1 +---- TYPES +BIGINT,STRING,INT +==== +---- QUERY +# Regression test for: +# IMPALA-11434: "More than 1 2d arrays in select list causes analysis error" +select id, map_1d, map_2d, map_3d, arr_int_3d, map_map_array from collection_tbl; +---- RESULTS +1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred"},20:{300:"three hundred",400:"four hundred"}},2:{30:{500:"five hundred",600:"six hundred"},40:{700:"seven hundred",800:"eight hundred"}}}','[[[1,2,NULL],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}' +---- TYPES +INT,STRING,STRING,STRING,STRING,STRING +===== +---- QUERY +select id, map_1d, map_2d, mma.value mma_value, ma.value ma_value +from collection_tbl c, c.map_map_array mma, mma.value ma; +---- RESULTS +1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{10:[100,200],20:[300,400]}','[100,200]' +1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{10:[100,200],20:[300,400]}','[300,400]' +1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{30:[500,600],40:[700,800]}','[500,600]' +1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{30:[500,600],40:[700,800]}','[700,800]' +---- TYPES +INT,STRING,STRING,STRING,STRING +===== diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test index 3f9fd0606..316f573b4 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test +++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test @@ -632,3 +632,16 @@ select id, a.item from complextypes_arrays_only_view v, v.int_array a; ---- TYPES BIGINT,INT ==== +---- QUERY +select id, a.key, a.value from complextypes_maps_view v, v.int_map a; +---- RESULTS +100,'k1',1 +100,'k2',100 +200,'k1',2 +200,'k2',NULL +700,'k1',NULL +700,'k3',NULL +800,'k1',-1 +---- TYPES +BIGINT,STRING,INT +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test index 1e3186ff4..573f7cc2d 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test +++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test @@ -46,4 +46,34 @@ NULL,'NULL' NULL,'[NULL,[5,6]]' ---- TYPES bigint,string -==== \ No newline at end of file +==== +---- QUERY +select id, int_map from functional_orc_def.complextypestbl +---- RESULTS +NULL,'{"k1":-1}' +NULL,'{"k1":1,"k2":100}' +NULL,'{"k1":2,"k2":NULL}' +NULL,'{}' +NULL,'{}' +NULL,'{}' +NULL,'NULL' +NULL,'{"k1":NULL,"k3":NULL}' +---- TYPES +bigint,string +==== +---- QUERY +# There is a column mask added for int_map_array, but it is ignored. +# It is not clear whether we should support this (RANGER-3525). +select id, int_map_array from functional_orc_def.complextypestbl +---- RESULTS +NULL,'[{},{"k1":1},{},{}]' +NULL,'[{"k1":1}]' +NULL,'[{"k3":NULL,"k1":1},NULL,{}]' +NULL,'[NULL,NULL]' +NULL,'[]' +NULL,'NULL' +NULL,'NULL' +NULL,'NULL' +---- TYPES +bigint,string +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test index deddd5159..f2b5ed498 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test +++ b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test @@ -642,3 +642,9 @@ select nested_struct.c.d from complextypestbl; ---- CATCH AnalysisException: STRUCT type inside collection types is not supported. ==== +---- QUERY +# Structs inside maps are not yet supported. +select nested_struct.g from complextypestbl; +---- CATCH +AnalysisException: STRUCT type inside collection types is not supported. +==== diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py index 9a989395c..23c5a1e73 100644 --- a/tests/authorization/test_ranger.py +++ b/tests/authorization/test_ranger.py @@ -1809,6 +1809,10 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite): unique_name + str(policy_cnt), user, "functional_orc_def", "complextypestbl", "int_array_array", "MASK_NULL") policy_cnt += 1 + TestRanger._add_column_masking_policy( + unique_name + str(policy_cnt), user, "functional_orc_def", + "complextypestbl", "int_array_map", "MASK_NULL") + policy_cnt += 1 self.execute_query_expect_success(admin_client, "refresh authorization", user=ADMIN) self.run_test_case("QueryTest/ranger_column_masking_struct_in_select_list", vector, diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py index 0097fa117..9b3ca59ca 100644 --- a/tests/query_test/test_nested_types.py +++ b/tests/query_test/test_nested_types.py @@ -152,7 +152,7 @@ class TestNestedStructsInSelectList(ImpalaTestSuite): self.run_test_case('QueryTest/nested-struct-in-select-list', new_vector) -class TestNestedTArraysInSelectList(ImpalaTestSuite): +class TestNestedCollectionsInSelectList(ImpalaTestSuite): """Functional tests for nested arrays provided in the select list.""" @classmethod def get_workload(self): @@ -160,7 +160,7 @@ class TestNestedTArraysInSelectList(ImpalaTestSuite): @classmethod def add_test_dimensions(cls): - super(TestNestedTArraysInSelectList, cls).add_test_dimensions() + super(TestNestedCollectionsInSelectList, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_constraint(lambda v: v.get_value('table_format').file_format in ['parquet', 'orc']) cls.ImpalaTestMatrix.add_dimension( @@ -176,6 +176,10 @@ class TestNestedTArraysInSelectList(ImpalaTestSuite): """Queries where an array column is in the select list""" self.run_test_case('QueryTest/nested-array-in-select-list', vector) + def test_map_in_select_list(self, vector, unique_database): + """Queries where a map column is in the select list""" + self.run_test_case('QueryTest/nested-map-in-select-list', vector) + # Moved this to a separate test class from TestNestedTypesInSelectList because this needs # a narrower test vector.