IMPALA-11719: Inconsistency in printing NULL values

NULL values are printed as "NULL" if they are top level or in
collections, but as "null" in structs. We should print collections and
structs in JSON form, so it should be "null" in collections, too. Hive
also follows the latter (correct) approach.

This commit changes the printing of NULL values to "null" in
collections.

Testing:
 - Modified the tests to expect "null" instead of "NULL" in collections.

Change-Id: Ie5e7f98df4014ea417ddf73ac0fb8ec01ef655ba
Reviewed-on: http://gerrit.cloudera.org:8080/19236
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Daniel Becker <daniel.becker@cloudera.com>

be/src/runtime/raw-value.cc

@@ -40,6 +40,17 @@ constexpr float RawValue::CANONICAL_FLOAT_NAN;
 constexpr double RawValue::CANONICAL_DOUBLE_ZERO;
 constexpr float RawValue::CANONICAL_FLOAT_ZERO;
 
+namespace {
+
+// Top level null values are printed as "NULL"; collections and structs are printed in
+// JSON format, which requires "null".
+constexpr const char* NullLiteral(bool top_level) {
+  if (top_level) return "NULL";
+  return "null";
+}
+
+}
+
 void RawValue::PrintValueAsBytes(const void* value, const ColumnType& type,
                                  stringstream* stream) {
   if (value == NULL) return;
@@ -93,7 +104,7 @@ void RawValue::PrintValueAsBytes(const void* value, const ColumnType& type,
 void RawValue::PrintValue(const void* value, const ColumnType& type, int scale,
                           string* str) {
   if (value == NULL) {
-    *str = "NULL";
+    *str = NullLiteral(true);
     return;
   }
 
@@ -283,7 +294,7 @@ void RawValue::PrintValue(
     const void* value, const ColumnType& type, int scale, std::stringstream* stream,
     bool quote_val) {
   if (value == NULL) {
-    *stream << "NULL";
+    *stream << NullLiteral(true);
     return;
   }
 
@@ -414,7 +425,7 @@ template void RawValue::WritePrimitive<false>(const void* value, Tuple* tuple,
 bool PrintNestedValueIfNull(const SlotDescriptor& slot_desc, Tuple* item,
     stringstream* stream) {
   bool is_null = item->IsNull(slot_desc.null_indicator_offset());
-  if (is_null) *stream << "NULL";
+  if (is_null) *stream << NullLiteral(false);
   return is_null;
 }
 
@@ -457,7 +468,9 @@ void RawValue::PrintCollectionValue(const CollectionValue* coll_val,
     bool is_map) {
   DCHECK(item_tuple_desc != nullptr);
   if (coll_val == nullptr) {
-    *stream << "NULL";
+    // We only reach this code path if this is a top level collection. Otherwise
+    // PrintNestedValue() handles the printing of the NULL literal.
+    *stream << NullLiteral(true);
     return;
   }
   int item_byte_size = item_tuple_desc->byte_size();

testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test

@@ -3,7 +3,7 @@
 select id, int_array from complextypestbl
 ---- RESULTS
 1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
 3,'[]'
 4,'NULL'
 5,'NULL'
@@ -25,12 +25,12 @@ select id, int_array, int_array_array from complextypestbl
 ---- RESULTS
 8,'[-1]','[[-1,-2],[]]'
 1,'[1,2,3]','[[1,2],[3,4]]'
-2,'[NULL,1,2,NULL,3,NULL]','[[NULL,1,2,NULL],[3,NULL,4],[],NULL]'
-3,'[]','[NULL]'
+2,'[null,1,2,null,3,null]','[[null,1,2,null],[3,null,4],[],null]'
+3,'[]','[null]'
 4,'NULL','[]'
 5,'NULL','NULL'
 6,'NULL','NULL'
-7,'NULL','[NULL,[5,6]]'
+7,'NULL','[null,[5,6]]'
 ---- TYPES
 bigint,string,string
 ====
@@ -45,7 +45,7 @@ IllegalStateException: Sorting is not supported if the select list contains coll
 select id, int_array, int_array from complextypestbl
 ---- RESULTS
 1,'[1,2,3]','[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]','[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]','[null,1,2,null,3,null]'
 3,'[]','[]'
 4,'NULL','NULL'
 5,'NULL','NULL'
@@ -66,7 +66,7 @@ select t1.id, t1.int_array, t2.int_array
 7,'NULL','NULL'
 8,'[-1]','[-1]'
 1,'[1,2,3]','[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]','[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]','[null,1,2,null,3,null]'
 4,'NULL','NULL'
 6,'NULL','NULL'
 ---- TYPES
@@ -76,7 +76,7 @@ bigint,string,string
 select id, int_array from complextypestbl union all select id, int_array from complextypestbl
 ---- RESULTS
 1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
 3,'[]'
 4,'NULL'
 5,'NULL'
@@ -84,7 +84,7 @@ select id, int_array from complextypestbl union all select id, int_array from co
 7,'NULL'
 8,'[-1]'
 1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
 3,'[]'
 4,'NULL'
 5,'NULL'
@@ -135,7 +135,7 @@ tinyint
 select id, int_array from (select id, int_array from complextypestbl) s;
 ---- RESULTS
 1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
 3,'[]'
 4,'NULL'
 5,'NULL'
@@ -150,7 +150,7 @@ with s as (select id, t.int_array from complextypestbl t)
 select id, int_array from s;
 ---- RESULTS
 1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
 3,'[]'
 4,'NULL'
 5,'NULL'
@@ -164,7 +164,7 @@ bigint,string
 select id, int_array from complextypes_arrays_only_view;
 ---- RESULTS
 1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
 3,'[]'
 4,'NULL'
 5,'NULL'
@@ -277,8 +277,8 @@ select item from unnest(complextypestbl.int_array_array)
 '[]'
 '[1,2]'
 '[3,4]'
-'[NULL,1,2,NULL]'
-'[3,NULL,4]'
+'[null,1,2,null]'
+'[3,null,4]'
 '[]'
 'NULL'
 'NULL'
@@ -388,7 +388,7 @@ INT,INT,STRING
 #   IMPALA-11434: "More than 1 2d arrays in select list causes analysis error"
 select id, arr_int_1d, arr_int_2d, arr_int_3d, arr_string_1d, arr_string_2d, arr_string_3d from collection_tbl;
 ---- RESULTS
-1,'[1,2,NULL]','[[1,2,NULL],[3]]','[[[1,2,NULL],[3]],[[4]]]','["1","2",NULL]','[["1","2",NULL],["3"]]','[[["1","2",NULL],["3"]],[["4"]]]'
+1,'[1,2,null]','[[1,2,null],[3]]','[[[1,2,null],[3]],[[4]]]','["1","2",null]','[["1","2",null],["3"]]','[[["1","2",null],["3"]],[["4"]]]'
 ---- TYPES
 INT,STRING,STRING,STRING,STRING,STRING,STRING
 ====
@@ -401,7 +401,7 @@ from alltypestiny a left outer join
 on a.id = b.id where a.id < 3;
 ---- RESULTS
 0,'NULL','NULL'
-1,'[1,2,NULL]','[[1,2,NULL],[3]]'
+1,'[1,2,null]','[[1,2,null],[3]]'
 2,'NULL','NULL'
 ---- TYPES
 INT,STRING,STRING

testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test

@@ -3,12 +3,12 @@
 select id, int_map from complextypestbl
 ---- RESULTS
 1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
 3,'{}'
 4,'{}'
 5,'{}'
 6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
 8,'{"k1":-1}'
 ---- TYPES
 bigint,string
@@ -24,12 +24,12 @@ bigint,string
 select id, int_map, int_map_array from complextypestbl
 ---- RESULTS
 1,'{"k1":1,"k2":100}','[{"k1":1}]'
-2,'{"k1":2,"k2":NULL}','[{"k3":NULL,"k1":1},NULL,{}]'
-3,'{}','[NULL,NULL]'
+2,'{"k1":2,"k2":null}','[{"k3":null,"k1":1},null,{}]'
+3,'{}','[null,null]'
 4,'{}','[]'
 5,'{}','NULL'
 6,'NULL','NULL'
-7,'{"k1":NULL,"k3":NULL}','NULL'
+7,'{"k1":null,"k3":null}','NULL'
 8,'{"k1":-1}','[{},{"k1":1},{},{}]'
 ---- TYPES
 bigint,string,string
@@ -45,12 +45,12 @@ IllegalStateException: Sorting is not supported if the select list contains coll
 select id, int_map, int_map from complextypestbl
 ---- RESULTS
 1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}','{"k1":2,"k2":null}'
 3,'{}','{}'
 4,'{}','{}'
 5,'{}','{}'
 6,'NULL','NULL'
-7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}','{"k1":null,"k3":null}'
 8,'{"k1":-1}','{"k1":-1}'
 ---- TYPES
 bigint,string,string
@@ -62,12 +62,12 @@ select t1.id, t1.int_map, t2.int_map
  on t1.id = t2.id
 ---- RESULTS
 1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}','{"k1":2,"k2":null}'
 3,'{}','{}'
 4,'{}','{}'
 5,'{}','{}'
 6,'NULL','NULL'
-7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}','{"k1":null,"k3":null}'
 8,'{"k1":-1}','{"k1":-1}'
 ---- TYPES
 bigint,string,string
@@ -76,20 +76,20 @@ bigint,string,string
 select id, int_map from complextypestbl union all select id, int_map from complextypestbl
 ---- RESULTS
 1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
 3,'{}'
 4,'{}'
 5,'{}'
 6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
 8,'{"k1":-1}'
 1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
 3,'{}'
 4,'{}'
 5,'{}'
 6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
 8,'{"k1":-1}'
 ---- TYPES
 bigint,string
@@ -135,12 +135,12 @@ tinyint
 select id, int_map from (select id, int_map from complextypestbl) s;
 ---- RESULTS
 1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
 3,'{}'
 4,'{}'
 5,'{}'
 6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
 8,'{"k1":-1}'
 ---- TYPES
 bigint,string
@@ -150,12 +150,12 @@ with s as (select id, t.int_map from complextypestbl t)
 select id, int_map from s;
 ---- RESULTS
 1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
 3,'{}'
 4,'{}'
 5,'{}'
 6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
 8,'{"k1":-1}'
 ---- TYPES
 bigint,string
@@ -164,12 +164,12 @@ bigint,string
 select id, int_map from complextypes_maps_view;
 ---- RESULTS
 1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
 3,'{}'
 4,'{}'
 5,'{}'
 6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
 8,'{"k1":-1}'
 ---- TYPES
 bigint,string
@@ -265,7 +265,7 @@ bigint,string,int
 select item from unnest(complextypestbl.int_map_array)
 ---- RESULTS
 '{"k1":1}'
-'{"k3":NULL,"k1":1}'
+'{"k3":null,"k1":1}'
 'NULL'
 '{}'
 'NULL'
@@ -309,7 +309,7 @@ BIGINT,STRING,INT
 #   IMPALA-11434: "More than 1 2d arrays in select list causes analysis error"
 select id, map_1d, map_2d, map_3d, arr_int_3d, map_map_array from collection_tbl;
 ---- RESULTS
-1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred"},20:{300:"three hundred",400:"four hundred"}},2:{30:{500:"five hundred",600:"six hundred"},40:{700:"seven hundred",800:"eight hundred"}}}','[[[1,2,NULL],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}'
+1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred"},20:{300:"three hundred",400:"four hundred"}},2:{30:{500:"five hundred",600:"six hundred"},40:{700:"seven hundred",800:"eight hundred"}}}','[[[1,2,null],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}'
 ---- TYPES
 INT,STRING,STRING,STRING,STRING,STRING
 =====

testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test

@@ -117,7 +117,7 @@ set EXPAND_COMPLEX_TYPES=true;
 select * from complextypes_structs s, complextypes_arrays a
 where s.id = a.id and s.id = 5
 ---- RESULTS
-5,'fifth item','NULL','{"b":false}','{"i":98765,"s":"abcde f"}',5,'[10,NULL,12]','["ten","eleven","twelve","thirteen"]'
+5,'fifth item','NULL','{"b":false}','{"i":98765,"s":"abcde f"}',5,'[10,null,12]','["ten","eleven","twelve","thirteen"]'
 ---- TYPES
 INT, STRING, STRING, STRING, STRING, INT, STRING, STRING
 ====
@@ -148,7 +148,7 @@ select s.*, a.*, m.*
 from complextypes_structs s, complextypes_arrays a, complextypes_maps_view m
 where s.id = a.id and s.id = m.id and s.id = 2
 ---- RESULTS
-2,'second item','{"ti":123,"si":4567,"i":1562322212,"bi":334333345342,"b":false,"f":NaN,"do":23233423.099,"da":null,"ts":"2020-06-11 12:10:04","s1":null,"s2":"NULL","c1":"a","c2":"ab ","vc":"varchar","de1":11223,"de2":null}','{"b":false}','{"i":19191,"s":"small_struct_str"}',2,'[1,NULL,3,4,5]','["one","two","three",NULL,"five"]',2,'{"k1":2,"k2":NULL}','[{"k3":NULL,"k1":1},NULL,{}]'
+2,'second item','{"ti":123,"si":4567,"i":1562322212,"bi":334333345342,"b":false,"f":NaN,"do":23233423.099,"da":null,"ts":"2020-06-11 12:10:04","s1":null,"s2":"NULL","c1":"a","c2":"ab ","vc":"varchar","de1":11223,"de2":null}','{"b":false}','{"i":19191,"s":"small_struct_str"}',2,'[1,null,3,4,5]','["one","two","three",null,"five"]',2,'{"k1":2,"k2":null}','[{"k3":null,"k1":1},null,{}]'
 ---- TYPES
 INT, STRING, STRING, STRING, STRING, INT, STRING, STRING, BIGINT, STRING, STRING
 ====

testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test

@@ -21,7 +21,7 @@ INT,STRING,STRING
 select id, int_array from functional_orc_def.complextypestbl
 ---- RESULTS
 NULL,'[1,2,3]'
-NULL,'[NULL,1,2,NULL,3,NULL]'
+NULL,'[null,1,2,null,3,null]'
 NULL,'[]'
 NULL,'NULL'
 NULL,'NULL'
@@ -38,12 +38,12 @@ select id, int_array_array from functional_orc_def.complextypestbl
 ---- RESULTS
 NULL,'[[-1,-2],[]]'
 NULL,'[[1,2],[3,4]]'
-NULL,'[[NULL,1,2,NULL],[3,NULL,4],[],NULL]'
-NULL,'[NULL]'
+NULL,'[[null,1,2,null],[3,null,4],[],null]'
+NULL,'[null]'
 NULL,'[]'
 NULL,'NULL'
 NULL,'NULL'
-NULL,'[NULL,[5,6]]'
+NULL,'[null,[5,6]]'
 ---- TYPES
 bigint,string
 ====
@@ -52,12 +52,12 @@ select id, int_map from functional_orc_def.complextypestbl
 ---- RESULTS
 NULL,'{"k1":-1}'
 NULL,'{"k1":1,"k2":100}'
-NULL,'{"k1":2,"k2":NULL}'
+NULL,'{"k1":2,"k2":null}'
 NULL,'{}'
 NULL,'{}'
 NULL,'{}'
 NULL,'NULL'
-NULL,'{"k1":NULL,"k3":NULL}'
+NULL,'{"k1":null,"k3":null}'
 ---- TYPES
 bigint,string
 ====
@@ -68,8 +68,8 @@ select id, int_map_array from functional_orc_def.complextypestbl
 ---- RESULTS
 NULL,'[{},{"k1":1},{},{}]'
 NULL,'[{"k1":1}]'
-NULL,'[{"k3":NULL,"k1":1},NULL,{}]'
-NULL,'[NULL,NULL]'
+NULL,'[{"k3":null,"k1":1},null,{}]'
+NULL,'[null,null]'
 NULL,'[]'
 NULL,'NULL'
 NULL,'NULL'

testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-generic.test

@@ -89,7 +89,7 @@ BIGINT, BIGINT
 select file__position, id, int_array from complextypestbl;
 ---- RESULTS
 0,1,'[1,2,3]'
-1,2,'[NULL,1,2,NULL,3,NULL]'
+1,2,'[null,1,2,null,3,null]'
 2,3,'[]'
 3,4,'NULL'
 4,5,'NULL'
@@ -121,8 +121,8 @@ select file__position, id, item from complextypestbl c, c.int_array_array;
 ---- RESULTS
 0,1,'[1,2]'
 0,1,'[3,4]'
-1,2,'[NULL,1,2,NULL]'
-1,2,'[3,NULL,4]'
+1,2,'[null,1,2,null]'
+1,2,'[3,null,4]'
 1,2,'[]'
 1,2,'NULL'
 2,3,'NULL'

testdata/workloads/functional-query/queries/QueryTest/zipping-unnest-in-select-list.test

@@ -198,8 +198,8 @@ select unnest(int_array_array) from complextypestbl;
 '[]'
 '[1,2]'
 '[3,4]'
-'[NULL,1,2,NULL]'
-'[3,NULL,4]'
+'[null,1,2,null]'
+'[3,null,4]'
 '[]'
 'NULL'
 'NULL'