You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by db...@apache.org on 2022/11/15 09:24:38 UTC
[impala] branch master updated: IMPALA-11719: Inconsistency in printing NULL values
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 03c465dac IMPALA-11719: Inconsistency in printing NULL values
03c465dac is described below
commit 03c465dac12e68628d4a623d3ae241d2f350fc35
Author: Daniel Becker <da...@cloudera.com>
AuthorDate: Fri Nov 11 14:44:26 2022 +0100
IMPALA-11719: Inconsistency in printing NULL values
NULL values are printed as "NULL" if they are top level or in
collections, but as "null" in structs. We should print collections and
structs in JSON form, so it should be "null" in collections, too. Hive
also follows the latter (correct) approach.
This commit changes the printing of NULL values to "null" in
collections.
Testing:
- Modified the tests to expect "null" instead of "NULL" in collections.
Change-Id: Ie5e7f98df4014ea417ddf73ac0fb8ec01ef655ba
Reviewed-on: http://gerrit.cloudera.org:8080/19236
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Daniel Becker <da...@cloudera.com>
---
be/src/runtime/raw-value.cc | 21 ++++++++---
.../QueryTest/nested-array-in-select-list.test | 30 ++++++++--------
.../QueryTest/nested-map-in-select-list.test | 42 +++++++++++-----------
.../QueryTest/nested-types-star-expansion.test | 4 +--
...anger_column_masking_struct_in_select_list.test | 16 ++++-----
.../virtual-column-file-position-generic.test | 6 ++--
.../QueryTest/zipping-unnest-in-select-list.test | 4 +--
7 files changed, 68 insertions(+), 55 deletions(-)
diff --git a/be/src/runtime/raw-value.cc b/be/src/runtime/raw-value.cc
index eba2b9d40..9a19753e9 100644
--- a/be/src/runtime/raw-value.cc
+++ b/be/src/runtime/raw-value.cc
@@ -40,6 +40,17 @@ constexpr float RawValue::CANONICAL_FLOAT_NAN;
constexpr double RawValue::CANONICAL_DOUBLE_ZERO;
constexpr float RawValue::CANONICAL_FLOAT_ZERO;
+namespace {
+
+// Top level null values are printed as "NULL"; collections and structs are printed in
+// JSON format, which requires "null".
+constexpr const char* NullLiteral(bool top_level) {
+ if (top_level) return "NULL";
+ return "null";
+}
+
+}
+
void RawValue::PrintValueAsBytes(const void* value, const ColumnType& type,
stringstream* stream) {
if (value == NULL) return;
@@ -93,7 +104,7 @@ void RawValue::PrintValueAsBytes(const void* value, const ColumnType& type,
void RawValue::PrintValue(const void* value, const ColumnType& type, int scale,
string* str) {
if (value == NULL) {
- *str = "NULL";
+ *str = NullLiteral(true);
return;
}
@@ -283,7 +294,7 @@ void RawValue::PrintValue(
const void* value, const ColumnType& type, int scale, std::stringstream* stream,
bool quote_val) {
if (value == NULL) {
- *stream << "NULL";
+ *stream << NullLiteral(true);
return;
}
@@ -414,7 +425,7 @@ template void RawValue::WritePrimitive<false>(const void* value, Tuple* tuple,
bool PrintNestedValueIfNull(const SlotDescriptor& slot_desc, Tuple* item,
stringstream* stream) {
bool is_null = item->IsNull(slot_desc.null_indicator_offset());
- if (is_null) *stream << "NULL";
+ if (is_null) *stream << NullLiteral(false);
return is_null;
}
@@ -457,7 +468,9 @@ void RawValue::PrintCollectionValue(const CollectionValue* coll_val,
bool is_map) {
DCHECK(item_tuple_desc != nullptr);
if (coll_val == nullptr) {
- *stream << "NULL";
+ // We only reach this code path if this is a top level collection. Otherwise
+ // PrintNestedValue() handles the printing of the NULL literal.
+ *stream << NullLiteral(true);
return;
}
int item_byte_size = item_tuple_desc->byte_size();
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test
index f5ef5612d..29d58b83e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test
@@ -3,7 +3,7 @@
select id, int_array from complextypestbl
---- RESULTS
1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
3,'[]'
4,'NULL'
5,'NULL'
@@ -25,12 +25,12 @@ select id, int_array, int_array_array from complextypestbl
---- RESULTS
8,'[-1]','[[-1,-2],[]]'
1,'[1,2,3]','[[1,2],[3,4]]'
-2,'[NULL,1,2,NULL,3,NULL]','[[NULL,1,2,NULL],[3,NULL,4],[],NULL]'
-3,'[]','[NULL]'
+2,'[null,1,2,null,3,null]','[[null,1,2,null],[3,null,4],[],null]'
+3,'[]','[null]'
4,'NULL','[]'
5,'NULL','NULL'
6,'NULL','NULL'
-7,'NULL','[NULL,[5,6]]'
+7,'NULL','[null,[5,6]]'
---- TYPES
bigint,string,string
====
@@ -45,7 +45,7 @@ IllegalStateException: Sorting is not supported if the select list contains coll
select id, int_array, int_array from complextypestbl
---- RESULTS
1,'[1,2,3]','[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]','[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]','[null,1,2,null,3,null]'
3,'[]','[]'
4,'NULL','NULL'
5,'NULL','NULL'
@@ -66,7 +66,7 @@ select t1.id, t1.int_array, t2.int_array
7,'NULL','NULL'
8,'[-1]','[-1]'
1,'[1,2,3]','[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]','[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]','[null,1,2,null,3,null]'
4,'NULL','NULL'
6,'NULL','NULL'
---- TYPES
@@ -76,7 +76,7 @@ bigint,string,string
select id, int_array from complextypestbl union all select id, int_array from complextypestbl
---- RESULTS
1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
3,'[]'
4,'NULL'
5,'NULL'
@@ -84,7 +84,7 @@ select id, int_array from complextypestbl union all select id, int_array from co
7,'NULL'
8,'[-1]'
1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
3,'[]'
4,'NULL'
5,'NULL'
@@ -135,7 +135,7 @@ tinyint
select id, int_array from (select id, int_array from complextypestbl) s;
---- RESULTS
1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
3,'[]'
4,'NULL'
5,'NULL'
@@ -150,7 +150,7 @@ with s as (select id, t.int_array from complextypestbl t)
select id, int_array from s;
---- RESULTS
1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
3,'[]'
4,'NULL'
5,'NULL'
@@ -164,7 +164,7 @@ bigint,string
select id, int_array from complextypes_arrays_only_view;
---- RESULTS
1,'[1,2,3]'
-2,'[NULL,1,2,NULL,3,NULL]'
+2,'[null,1,2,null,3,null]'
3,'[]'
4,'NULL'
5,'NULL'
@@ -277,8 +277,8 @@ select item from unnest(complextypestbl.int_array_array)
'[]'
'[1,2]'
'[3,4]'
-'[NULL,1,2,NULL]'
-'[3,NULL,4]'
+'[null,1,2,null]'
+'[3,null,4]'
'[]'
'NULL'
'NULL'
@@ -388,7 +388,7 @@ INT,INT,STRING
# IMPALA-11434: "More than 1 2d arrays in select list causes analysis error"
select id, arr_int_1d, arr_int_2d, arr_int_3d, arr_string_1d, arr_string_2d, arr_string_3d from collection_tbl;
---- RESULTS
-1,'[1,2,NULL]','[[1,2,NULL],[3]]','[[[1,2,NULL],[3]],[[4]]]','["1","2",NULL]','[["1","2",NULL],["3"]]','[[["1","2",NULL],["3"]],[["4"]]]'
+1,'[1,2,null]','[[1,2,null],[3]]','[[[1,2,null],[3]],[[4]]]','["1","2",null]','[["1","2",null],["3"]]','[[["1","2",null],["3"]],[["4"]]]'
---- TYPES
INT,STRING,STRING,STRING,STRING,STRING,STRING
====
@@ -401,7 +401,7 @@ from alltypestiny a left outer join
on a.id = b.id where a.id < 3;
---- RESULTS
0,'NULL','NULL'
-1,'[1,2,NULL]','[[1,2,NULL],[3]]'
+1,'[1,2,null]','[[1,2,null],[3]]'
2,'NULL','NULL'
---- TYPES
INT,STRING,STRING
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test
index 622bac4b1..326c6ae91 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test
@@ -3,12 +3,12 @@
select id, int_map from complextypestbl
---- RESULTS
1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
3,'{}'
4,'{}'
5,'{}'
6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
8,'{"k1":-1}'
---- TYPES
bigint,string
@@ -24,12 +24,12 @@ bigint,string
select id, int_map, int_map_array from complextypestbl
---- RESULTS
1,'{"k1":1,"k2":100}','[{"k1":1}]'
-2,'{"k1":2,"k2":NULL}','[{"k3":NULL,"k1":1},NULL,{}]'
-3,'{}','[NULL,NULL]'
+2,'{"k1":2,"k2":null}','[{"k3":null,"k1":1},null,{}]'
+3,'{}','[null,null]'
4,'{}','[]'
5,'{}','NULL'
6,'NULL','NULL'
-7,'{"k1":NULL,"k3":NULL}','NULL'
+7,'{"k1":null,"k3":null}','NULL'
8,'{"k1":-1}','[{},{"k1":1},{},{}]'
---- TYPES
bigint,string,string
@@ -45,12 +45,12 @@ IllegalStateException: Sorting is not supported if the select list contains coll
select id, int_map, int_map from complextypestbl
---- RESULTS
1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}','{"k1":2,"k2":null}'
3,'{}','{}'
4,'{}','{}'
5,'{}','{}'
6,'NULL','NULL'
-7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}','{"k1":null,"k3":null}'
8,'{"k1":-1}','{"k1":-1}'
---- TYPES
bigint,string,string
@@ -62,12 +62,12 @@ select t1.id, t1.int_map, t2.int_map
on t1.id = t2.id
---- RESULTS
1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}','{"k1":2,"k2":null}'
3,'{}','{}'
4,'{}','{}'
5,'{}','{}'
6,'NULL','NULL'
-7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}','{"k1":null,"k3":null}'
8,'{"k1":-1}','{"k1":-1}'
---- TYPES
bigint,string,string
@@ -76,20 +76,20 @@ bigint,string,string
select id, int_map from complextypestbl union all select id, int_map from complextypestbl
---- RESULTS
1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
3,'{}'
4,'{}'
5,'{}'
6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
8,'{"k1":-1}'
1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
3,'{}'
4,'{}'
5,'{}'
6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
8,'{"k1":-1}'
---- TYPES
bigint,string
@@ -135,12 +135,12 @@ tinyint
select id, int_map from (select id, int_map from complextypestbl) s;
---- RESULTS
1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
3,'{}'
4,'{}'
5,'{}'
6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
8,'{"k1":-1}'
---- TYPES
bigint,string
@@ -150,12 +150,12 @@ with s as (select id, t.int_map from complextypestbl t)
select id, int_map from s;
---- RESULTS
1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
3,'{}'
4,'{}'
5,'{}'
6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
8,'{"k1":-1}'
---- TYPES
bigint,string
@@ -164,12 +164,12 @@ bigint,string
select id, int_map from complextypes_maps_view;
---- RESULTS
1,'{"k1":1,"k2":100}'
-2,'{"k1":2,"k2":NULL}'
+2,'{"k1":2,"k2":null}'
3,'{}'
4,'{}'
5,'{}'
6,'NULL'
-7,'{"k1":NULL,"k3":NULL}'
+7,'{"k1":null,"k3":null}'
8,'{"k1":-1}'
---- TYPES
bigint,string
@@ -265,7 +265,7 @@ bigint,string,int
select item from unnest(complextypestbl.int_map_array)
---- RESULTS
'{"k1":1}'
-'{"k3":NULL,"k1":1}'
+'{"k3":null,"k1":1}'
'NULL'
'{}'
'NULL'
@@ -309,7 +309,7 @@ BIGINT,STRING,INT
# IMPALA-11434: "More than 1 2d arrays in select list causes analysis error"
select id, map_1d, map_2d, map_3d, arr_int_3d, map_map_array from collection_tbl;
---- RESULTS
-1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred"},20:{300:"three hundred",400:"four hundred"}},2:{30:{500:"five hundred",600:"six hundred"},40:{700:"seven hundred",800:"eight hundred"}}}','[[[1,2,NULL],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}'
+1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{1:{10:{100:"hundred",200:"two hundred"},20:{300:"three hundred",400:"four hundred"}},2:{30:{500:"five hundred",600:"six hundred"},40:{700:"seven hundred",800:"eight hundred"}}}','[[[1,2,null],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}'
---- TYPES
INT,STRING,STRING,STRING,STRING,STRING
=====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test
index 1bb8985fb..4579fb094 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-star-expansion.test
@@ -117,7 +117,7 @@ set EXPAND_COMPLEX_TYPES=true;
select * from complextypes_structs s, complextypes_arrays a
where s.id = a.id and s.id = 5
---- RESULTS
-5,'fifth item','NULL','{"b":false}','{"i":98765,"s":"abcde f"}',5,'[10,NULL,12]','["ten","eleven","twelve","thirteen"]'
+5,'fifth item','NULL','{"b":false}','{"i":98765,"s":"abcde f"}',5,'[10,null,12]','["ten","eleven","twelve","thirteen"]'
---- TYPES
INT, STRING, STRING, STRING, STRING, INT, STRING, STRING
====
@@ -148,7 +148,7 @@ select s.*, a.*, m.*
from complextypes_structs s, complextypes_arrays a, complextypes_maps_view m
where s.id = a.id and s.id = m.id and s.id = 2
---- RESULTS
-2,'second item','{"ti":123,"si":4567,"i":1562322212,"bi":334333345342,"b":false,"f":NaN,"do":23233423.099,"da":null,"ts":"2020-06-11 12:10:04","s1":null,"s2":"NULL","c1":"a","c2":"ab ","vc":"varchar","de1":11223,"de2":null}','{"b":false}','{"i":19191,"s":"small_struct_str"}',2,'[1,NULL,3,4,5]','["one","two","three",NULL,"five"]',2,'{"k1":2,"k2":NULL}','[{"k3":NULL,"k1":1},NULL,{}]'
+2,'second item','{"ti":123,"si":4567,"i":1562322212,"bi":334333345342,"b":false,"f":NaN,"do":23233423.099,"da":null,"ts":"2020-06-11 12:10:04","s1":null,"s2":"NULL","c1":"a","c2":"ab ","vc":"varchar","de1":11223,"de2":null}','{"b":false}','{"i":19191,"s":"small_struct_str"}',2,'[1,null,3,4,5]','["one","two","three",null,"five"]',2,'{"k1":2,"k2":null}','[{"k3":null,"k1":1},null,{}]'
---- TYPES
INT, STRING, STRING, STRING, STRING, INT, STRING, STRING, BIGINT, STRING, STRING
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
index 573f7cc2d..8a92439f9 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
@@ -21,7 +21,7 @@ INT,STRING,STRING
select id, int_array from functional_orc_def.complextypestbl
---- RESULTS
NULL,'[1,2,3]'
-NULL,'[NULL,1,2,NULL,3,NULL]'
+NULL,'[null,1,2,null,3,null]'
NULL,'[]'
NULL,'NULL'
NULL,'NULL'
@@ -38,12 +38,12 @@ select id, int_array_array from functional_orc_def.complextypestbl
---- RESULTS
NULL,'[[-1,-2],[]]'
NULL,'[[1,2],[3,4]]'
-NULL,'[[NULL,1,2,NULL],[3,NULL,4],[],NULL]'
-NULL,'[NULL]'
+NULL,'[[null,1,2,null],[3,null,4],[],null]'
+NULL,'[null]'
NULL,'[]'
NULL,'NULL'
NULL,'NULL'
-NULL,'[NULL,[5,6]]'
+NULL,'[null,[5,6]]'
---- TYPES
bigint,string
====
@@ -52,12 +52,12 @@ select id, int_map from functional_orc_def.complextypestbl
---- RESULTS
NULL,'{"k1":-1}'
NULL,'{"k1":1,"k2":100}'
-NULL,'{"k1":2,"k2":NULL}'
+NULL,'{"k1":2,"k2":null}'
NULL,'{}'
NULL,'{}'
NULL,'{}'
NULL,'NULL'
-NULL,'{"k1":NULL,"k3":NULL}'
+NULL,'{"k1":null,"k3":null}'
---- TYPES
bigint,string
====
@@ -68,8 +68,8 @@ select id, int_map_array from functional_orc_def.complextypestbl
---- RESULTS
NULL,'[{},{"k1":1},{},{}]'
NULL,'[{"k1":1}]'
-NULL,'[{"k3":NULL,"k1":1},NULL,{}]'
-NULL,'[NULL,NULL]'
+NULL,'[{"k3":null,"k1":1},null,{}]'
+NULL,'[null,null]'
NULL,'[]'
NULL,'NULL'
NULL,'NULL'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-generic.test b/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-generic.test
index bb06a353b..45100a434 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-generic.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/virtual-column-file-position-generic.test
@@ -89,7 +89,7 @@ BIGINT, BIGINT
select file__position, id, int_array from complextypestbl;
---- RESULTS
0,1,'[1,2,3]'
-1,2,'[NULL,1,2,NULL,3,NULL]'
+1,2,'[null,1,2,null,3,null]'
2,3,'[]'
3,4,'NULL'
4,5,'NULL'
@@ -121,8 +121,8 @@ select file__position, id, item from complextypestbl c, c.int_array_array;
---- RESULTS
0,1,'[1,2]'
0,1,'[3,4]'
-1,2,'[NULL,1,2,NULL]'
-1,2,'[3,NULL,4]'
+1,2,'[null,1,2,null]'
+1,2,'[3,null,4]'
1,2,'[]'
1,2,'NULL'
2,3,'NULL'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/zipping-unnest-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/zipping-unnest-in-select-list.test
index 2cbeea134..e36c05acf 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/zipping-unnest-in-select-list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/zipping-unnest-in-select-list.test
@@ -198,8 +198,8 @@ select unnest(int_array_array) from complextypestbl;
'[]'
'[1,2]'
'[3,4]'
-'[NULL,1,2,NULL]'
-'[3,NULL,4]'
+'[null,1,2,null]'
+'[3,null,4]'
'[]'
'NULL'
'NULL'