You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/12 15:35:37 UTC
[incubator-doris] branch master updated: [feature-wip](array-type) Support array type which doesn't contain null (#9809)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 415b6b8086 [feature-wip](array-type) Support array type which doesn't contain null (#9809)
415b6b8086 is described below
commit 415b6b8086b6fe8ff3b12c5c047f2fa4bcb07484
Author: Adonis Ling <ad...@gmail.com>
AuthorDate: Sun Jun 12 23:35:28 2022 +0800
[feature-wip](array-type) Support array type which doesn't contain null (#9809)
---
be/src/runtime/types.cpp | 6 +
be/src/runtime/types.h | 7 +-
be/src/vec/data_types/data_type_factory.cpp | 4 +-
be/test/runtime/array_test.cpp | 212 ++++++++++++++++++++-
fe/fe-core/src/main/cup/sql_parser.cup | 5 +-
.../org/apache/doris/analysis/ArrayLiteral.java | 17 +-
.../java/org/apache/doris/catalog/ArrayType.java | 50 +++--
.../main/java/org/apache/doris/catalog/Column.java | 5 +-
.../java/org/apache/doris/catalog/ColumnType.java | 4 +-
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../doris/analysis/RangePartitionPruneTest.java | 2 +-
.../org/apache/doris/catalog/CreateTableTest.java | 5 +
gensrc/proto/types.proto | 2 +
gensrc/thrift/Types.thrift | 3 +
14 files changed, 283 insertions(+), 40 deletions(-)
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index b59d5186d2..58261a216b 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -49,6 +49,9 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>& types, int* idx)
DCHECK(!node.__isset.scalar_type);
DCHECK_LT(*idx, types.size() - 1);
type = TYPE_ARRAY;
+ if (node.__isset.contains_null) {
+ contains_null = node.contains_null;
+ }
++(*idx);
children.push_back(TypeDescriptor(types, idx));
break;
@@ -165,6 +168,9 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
}
case TTypeNodeType::ARRAY: {
type = TYPE_ARRAY;
+ if (node.has_contains_null()) {
+ contains_null = node.contains_null();
+ }
++(*idx);
children.push_back(TypeDescriptor(types, idx));
break;
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index b85fdb2dc2..2987064962 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -57,12 +57,15 @@ struct TypeDescriptor {
/// The maximum precision representable by a 8-byte decimal (Decimal8Value)
static const int MAX_DECIMAL8_PRECISION = 18;
- /// Empty for scalar types
+ // Empty for scalar types
std::vector<TypeDescriptor> children;
- /// Only set if type == TYPE_STRUCT. The field name of each child.
+ // Only set if type == TYPE_STRUCT. The field name of each child.
std::vector<std::string> field_names;
+ // Used for complex types only.
+ bool contains_null = true;
+
TypeDescriptor() : type(INVALID_TYPE), len(-1), precision(-1), scale(-1) {}
// explicit TypeDescriptor(PrimitiveType type) :
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index 3658270ec7..43897e1f44 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -108,8 +108,8 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
break;
case TYPE_ARRAY:
DCHECK(col_desc.children.size() == 1);
- nested =
- std::make_shared<vectorized::DataTypeArray>(create_data_type(col_desc.children[0]));
+ nested = std::make_shared<vectorized::DataTypeArray>(
+ create_data_type(col_desc.children[0], col_desc.contains_null));
break;
case INVALID_TYPE:
default:
diff --git a/be/test/runtime/array_test.cpp b/be/test/runtime/array_test.cpp
index 2582a975f5..162157852d 100644
--- a/be/test/runtime/array_test.cpp
+++ b/be/test/runtime/array_test.cpp
@@ -52,9 +52,10 @@ namespace doris {
template <typename... Ts>
ColumnPB create_column_pb(const std::string& type, const Ts&... sub_column_types) {
ColumnPB column;
- column.set_type(type);
+ auto prefix = "NOT_NULL_";
+ column.set_is_nullable(type.compare(0, strlen(prefix), prefix) != 0);
+ column.set_type(column.is_nullable() ? type : type.substr(strlen(prefix)));
column.set_aggregation("NONE");
- column.set_is_nullable(true);
if (type == "ARRAY") {
column.set_length(OLAP_ARRAY_MAX_BYTES);
}
@@ -326,7 +327,7 @@ private:
meta->set_encoding(item_encoding);
}
meta->set_compression(segment_v2::LZ4F);
- meta->set_is_nullable(true);
+ meta->set_is_nullable(column.is_nullable());
for (uint32_t i = 0; i < column.get_subtype_count(); ++i) {
init_column_meta<array_encoding, item_encoding>(meta->add_children_columns(), column_id,
column.get_sub_column(i));
@@ -465,6 +466,36 @@ TEST_F(ArrayTest, TestBoolean) {
test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
}
+TEST_F(ArrayTest, TestNotNullBoolean) {
+ // depth 1
+ auto column_pb = create_column_pb("ARRAY", "NOT_NULL_BOOLEAN");
+ std::vector<std::string> literal_arrays = {
+ "[]",
+ "[true, false, false]",
+ };
+ test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
+
+ // depth 2
+ column_pb = create_column_pb("ARRAY", "ARRAY", "NOT_NULL_BOOLEAN");
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[false, true, false]]",
+ "[[false, true, false], [true, false, true]]",
+ };
+ test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
+
+ // depth 3
+ column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", "NOT_NULL_BOOLEAN");
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[[]]]",
+ "[[[]], [[false], [true, false]], [[false, true, false]]]",
+ };
+ test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
+}
+
void test_integer(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
@@ -512,6 +543,44 @@ TEST_F(ArrayTest, TestInteger) {
test_integer("LARGEINT", *this);
}
+void test_not_null_integer(const std::string& type, ArrayTest& test_suite) {
+ // depth 1
+ auto column_pb = create_column_pb("ARRAY", type);
+ std::vector<std::string> literal_arrays = {
+ "[]",
+ "[1, 2, 3]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+
+ // depth 2
+ column_pb = create_column_pb("ARRAY", "ARRAY", type);
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[1, 2, 3]]",
+ "[[1, 2, 3], [4, 5, 6]]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+
+ // depth 3
+ column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
+ literal_arrays = {
+ "[]", "[[]]", "[[[]]]", "[[[1, 2, 3]]]", "[[[]], [[1], [2, 3]], [[4, 5, 6]]]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+}
+
+TEST_F(ArrayTest, TestNotNullInteger) {
+ test_not_null_integer("NOT_NULL_TINYINT", *this);
+ test_not_null_integer("NOT_NULL_SMALLINT", *this);
+ test_not_null_integer("NOT_NULL_INT", *this);
+ test_not_null_integer("NOT_NULL_BIGINT", *this);
+ test_not_null_integer("NOT_NULL_LARGEINT", *this);
+}
+
void test_float(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
@@ -555,6 +624,40 @@ TEST_F(ArrayTest, TestFloat) {
test_float("DOUBLE", *this);
}
+void test_not_null_float(const std::string& type, ArrayTest& test_suite) {
+ // depth 1
+ auto column_pb = create_column_pb("ARRAY", type);
+ std::vector<std::string> literal_arrays = {
+ "[]",
+ "[1.5, 2.5, 3.5]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+ // depth 2
+ column_pb = create_column_pb("ARRAY", "ARRAY", type);
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[1.5, 2.5, 3.5]]",
+ "[[1.5, 2.5, 3.5], [4.5, 5.5, 6.5]]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+
+ // depth 3
+ column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
+ literal_arrays = {
+ "[]", "[[]]", "[[[]]]", "[[[1.5]]]", "[[[]], [[1.5], [2.5, 3.5]], [[4.5, 5.5, 6.5]]]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+}
+
+TEST_F(ArrayTest, TestNotNullFloat) {
+ test_not_null_float("NOT_NULL_FLOAT", *this);
+ test_not_null_float("NOT_NULL_DOUBLE", *this);
+}
+
void test_string(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
@@ -590,6 +693,35 @@ TEST_F(ArrayTest, TestString) {
test_string("STRING", *this);
}
+void test_not_null_string(const std::string& type, ArrayTest& test_suite) {
+ // depth 1
+ auto column_pb = create_column_pb("ARRAY", type);
+ std::vector<std::string> literal_arrays = {
+ "[]",
+ "[\"a\", \"b\", \"c\"]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::DICT_ENCODING>(column_pb,
+ literal_arrays);
+
+ // more depths
+ column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[[]]]",
+ "[[[\"a\", \"b\", \"c\"]]]",
+ "[[[\"a\", \"c\"], [\"d\", \"e\", \"f\"]], [[\"g\"]]]",
+ };
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::DICT_ENCODING>(column_pb,
+ literal_arrays);
+}
+
+TEST_F(ArrayTest, TestNotNullString) {
+ test_not_null_string("NOT_NULL_CHAR", *this);
+ test_not_null_string("NOT_NULL_VARCHAR", *this);
+ test_not_null_string("NOT_NULL_STRING", *this);
+}
+
void test_datetime(const std::string& type, ArrayTest& test_suite) {
auto column_pb = create_column_pb("ARRAY", type);
std::vector<std::string> literal_arrays;
@@ -681,9 +813,83 @@ TEST_F(ArrayTest, TestDateTime) {
test_datetime("DATETIME", *this);
}
+void test_not_null_datetime(const std::string& type, ArrayTest& test_suite) {
+ auto column_pb = create_column_pb("ARRAY", type);
+ std::vector<std::string> literal_arrays;
+ if (type == "DATE") {
+ literal_arrays = {
+ "[]",
+ "[\"2022-04-01\", \"2022-04-02\", \"2022-04-03\"]",
+ };
+ } else {
+ literal_arrays = {
+ "[]",
+ "[\"2022-04-01 19:30:40\", \"2022-04-02 19:30:40 \", \"2022-04-03 19:30:40\"]",
+ };
+ }
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+ // depth 2
+ column_pb = create_column_pb("ARRAY", "ARRAY", type);
+ if (type == "DATE") {
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[\"2022-04-01\", \"2022-04-02\", \"2022-04-03\"], [\"2022-04-04\", "
+ "\"2022-04-05\", "
+ "\"2022-04-06\"]]",
+ };
+ } else {
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[\"2022-04-01 19:30:40\", \"2022-04-02 19:30:40\", \"2022-04-03 19:30:40\"], "
+ "[\"2022-04-04 19:30:40\", "
+ "\"2022-04-05\", "
+ "\"2022-04-06\"]]",
+ };
+ }
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+
+ // depth 3
+ column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
+ if (type == "DATE") {
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[[]]]",
+ "[[[\"2022-04-01\"]]]",
+ "[[[]], [[\"2022-04-01\"], [\"2022-04-02\", \"2022-04-03\"]], "
+ "[[\"2022-04-04\", "
+ "\"2022-04-05\", \"2022-04-06\"]]]",
+ };
+ } else {
+ literal_arrays = {
+ "[]",
+ "[[]]",
+ "[[[]]]",
+ "[[[\"2022-04-01 19:30:40\"]]]",
+ "[[[]], [[\"2022-04-01 19:30:40\"], [\"2022-04-02 19:30:40\", \"2022-04-03 "
+ "19:30:40\"]], "
+ "[[\"2022-04-04 19:30:40\", "
+ "\"2022-04-05 19:30:40\", \"2022-04-06 19:30:40\"]]]",
+ };
+ }
+ test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
+ literal_arrays);
+}
+
+TEST_F(ArrayTest, TestNotNullDateTime) {
+ test_not_null_datetime("NOT_NULL_DATE", *this);
+ test_not_null_datetime("NOT_NULL_DATETIME", *this);
+}
+
TEST_F(ArrayTest, TestDecimal) {
test_integer("DECIMAL", *this);
+ test_not_null_integer("NOT_NULL_DECIMAL", *this);
test_float("DECIMAL", *this);
+ test_not_null_float("NOT_NULL_DECIMAL", *this);
}
} // namespace doris
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 2bdb818ed8..281c4c884f 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -276,7 +276,8 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_A
KW_UNCOMMITTED, KW_UNBOUNDED, KW_UNION, KW_UNIQUE, KW_UNLOCK, KW_UNSIGNED, KW_USE, KW_USER, KW_USING, KW_UNINSTALL,
KW_VALUE, KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VERBOSE, KW_VIEW,
KW_WARNINGS, KW_WEEK, KW_WHEN, KW_WHITELIST, KW_WHERE, KW_WITH, KW_WORK, KW_WRITE,
- KW_YEAR;
+ KW_YEAR,
+ KW_NOT_NULL;
terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT;
terminal BITAND, BITOR, BITXOR, BITNOT;
@@ -4628,6 +4629,8 @@ type ::=
{: RESULT = ScalarType.createVarcharType(-1); :}
| KW_ARRAY LESSTHAN type:value_type GREATERTHAN
{: RESULT = new ArrayType(value_type); :}
+ | KW_ARRAY LESSTHAN KW_NOT_NULL LPAREN type:value_type RPAREN GREATERTHAN
+ {: RESULT = new ArrayType(value_type, false); :}
| KW_MAP LESSTHAN type:key_type COMMA type:value_type GREATERTHAN
{: RESULT = new MapType(key_type,value_type); :}
| KW_STRUCT LESSTHAN struct_field_list:fields GREATERTHAN
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
index d9024de807..60be1f6001 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
@@ -35,17 +35,22 @@ import java.util.List;
public class ArrayLiteral extends LiteralExpr {
public ArrayLiteral() {
- this.type = new ArrayType(Type.NULL);
+ type = new ArrayType(Type.NULL, false);
children = new ArrayList<>();
}
public ArrayLiteral(LiteralExpr... v) {
- if (v.length < 1) {
- this.type = new ArrayType(Type.NULL);
- return;
+ Type itemType = Type.NULL;
+ boolean containsNull = false;
+ for (LiteralExpr expr : v) {
+ if (itemType == Type.NULL || expr.type.getSlotSize() > itemType.getSlotSize()) {
+ itemType = expr.type;
+ }
+ if (expr.isNullable()) {
+ containsNull = true;
+ }
}
-
- this.type = new ArrayType(v[0].type);
+ type = new ArrayType(itemType, containsNull);
children = new ArrayList<>(v.length);
children.addAll(Arrays.asList(v));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
index 028cb760fd..4748d4cad9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -37,22 +37,31 @@ public class ArrayType extends Type {
@SerializedName(value = "itemType")
private Type itemType;
+ @SerializedName(value = "containsNull")
+ private boolean containsNull;
+
public ArrayType() {
- this.itemType = NULL;
+ itemType = NULL;
+ containsNull = false;
}
public ArrayType(Type itemType) {
- this.itemType = itemType;
+ this(itemType, true);
}
- public void setItemType(Type itemType) {
+ public ArrayType(Type itemType, boolean containsNull) {
this.itemType = itemType;
+ this.containsNull = containsNull;
}
public Type getItemType() {
return itemType;
}
+ public boolean getContainsNull() {
+ return containsNull;
+ }
+
@Override
public PrimitiveType getPrimitiveType() {
return PrimitiveType.ARRAY;
@@ -69,32 +78,33 @@ public class ArrayType extends Type {
}
// Array(Null) is a virtual Array type, can match any Array(...) type
- if (itemType.isNull()) {
- return true;
- }
- if (((ArrayType) t).getItemType().isNull()) {
+ if (itemType.isNull() || ((ArrayType) t).getItemType().isNull()) {
return true;
}
-
- return itemType.matchesType(((ArrayType) t).itemType);
+ return itemType.matchesType(((ArrayType) t).itemType)
+ && ((ArrayType) t).containsNull == containsNull;
}
public static ArrayType create() {
return new ArrayType();
}
- public static ArrayType create(Type type) {
- return new ArrayType(type);
+ public static ArrayType create(Type type, boolean containsNull) {
+ return new ArrayType(type, containsNull);
}
@Override
public String toSql(int depth) {
- return String.format("ARRAY<%s>", itemType.toSql(depth + 1));
+ if (!containsNull) {
+ return "ARRAY<NOT_NULL(" + itemType.toSql(depth + 1) + ")>";
+ } else {
+ return "ARRAY<" + itemType.toSql(depth + 1) + ">";
+ }
}
@Override
public int hashCode() {
- return Objects.hash(itemType);
+ return Objects.hash(itemType, containsNull);
}
@Override
@@ -103,10 +113,13 @@ public class ArrayType extends Type {
return false;
}
ArrayType otherArrayType = (ArrayType) other;
- return otherArrayType.itemType.equals(itemType);
+ return otherArrayType.itemType.equals(itemType) && otherArrayType.containsNull == containsNull;
}
public static boolean canCastTo(ArrayType type, ArrayType targetType) {
+ if (!targetType.containsNull && type.containsNull) {
+ return false;
+ }
if (targetType.getItemType().isStringType() && type.getItemType().isStringType()) {
return true;
}
@@ -119,6 +132,7 @@ public class ArrayType extends Type {
container.types.add(node);
Preconditions.checkNotNull(itemType);
node.setType(TTypeNodeType.ARRAY);
+ node.setContainsNull(containsNull);
itemType.toThrift(container);
}
@@ -130,8 +144,7 @@ public class ArrayType extends Type {
}
// Pass in the padding to make sure nested fields are aligned properly,
// even if we then strip the top-level padding.
- String structStr = itemType.prettyPrint(lpad);
- structStr = structStr.substring(lpad);
+ String structStr = itemType.prettyPrint(lpad).substring(lpad);
return String.format("%sARRAY<%s>", leftPadding, structStr);
}
@@ -162,10 +175,7 @@ public class ArrayType extends Type {
@Override
public boolean supportsTablePartitioning() {
- if (!isSupported() || isComplexType()) {
- return false;
- }
- return true;
+ return isSupported() && !isComplexType();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 28e7a24303..7f75392090 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -166,10 +166,7 @@ public class Column implements Writable {
public void createChildrenColumn(Type type, Column column) {
if (type.isArrayType()) {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
- // TODO We always set the item type in array nullable.
- // We may provide an alternative to configure this property of
- // the item type in array in future.
- c.setIsAllowNull(true);
+ c.setIsAllowNull(((ArrayType) type).getContainsNull());
column.addChildrenColumn(c);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
index 41575518b4..81bd0e3238 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
@@ -134,6 +134,7 @@ public abstract class ColumnType {
ArrayType arrayType = (ArrayType) type;
Text.writeString(out, arrayType.getPrimitiveType().name());
write(out, arrayType.getItemType());
+ out.writeBoolean(arrayType.getContainsNull());
}
}
@@ -141,7 +142,8 @@ public abstract class ColumnType {
PrimitiveType primitiveType = PrimitiveType.valueOf(Text.readString(in));
if (primitiveType == PrimitiveType.ARRAY) {
Type itermType = read(in);
- return ArrayType.create(itermType);
+ boolean containsNull = in.readBoolean();
+ return ArrayType.create(itermType, containsNull);
} else {
int scale = in.readInt();
int precision = in.readInt();
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex
index fceb550129..d57c6e8519 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -432,6 +432,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("year", new Integer(SqlParserSymbols.KW_YEAR));
keywordMap.put("||", new Integer(SqlParserSymbols.KW_PIPE));
keywordMap.put("current_timestamp", new Integer(SqlParserSymbols.KW_CURRENT_TIMESTAMP));
+ keywordMap.put("not_null", new Integer(SqlParserSymbols.KW_NOT_NULL));
}
// map from token id to token description
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
index 52b52eb74f..fbc28fe6d5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
@@ -128,7 +128,7 @@ public class RangePartitionPruneTest extends PartitionPruneTestBase {
addCase("select * from test.t1 where dt in (20211124, 20211126, 20211122)", "partitions=3/8", "partitions=3/8");
// is null
addCase("select * from test.t1 where dt is null", "partitions=1/8", "partitions=1/8");
- addCase("select * from test.not_null where dt is null", "partitions=0/7", "partitions=0/7");
+ addCase("select * from test.`not_null` where dt is null", "partitions=0/7", "partitions=0/7");
// not equal to
addCase("select * from test.t1 where dt!=20211122", "partitions=8/8", "partitions=8/8");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 71dd3e399d..17ddd906c0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -539,5 +539,10 @@ public class CreateTableTest {
createTable("create table test.table2(k1 INT, k2 Array<Array<int>>) duplicate key (k1) "
+ "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
});
+
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.table3(k1 INT, k2 Array<not_null(int)>) duplicate key (k1) "
+ + "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
+ });
}
}
diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto
index a3ac3dc190..7a95288517 100644
--- a/gensrc/proto/types.proto
+++ b/gensrc/proto/types.proto
@@ -48,6 +48,8 @@ message PTypeNode {
optional PScalarType scalar_type = 2;
// only used for structs; has struct_fields.size() corresponding child types
repeated PStructField struct_fields = 3;
+ // only used for complex types, such as array, map and etc.
+ optional bool contains_null = 4;
};
// A flattened representation of a tree of column types obtained by depth-first
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index d75376443f..098307c98e 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -126,6 +126,9 @@ struct TTypeNode {
// only used for structs; has struct_fields.size() corresponding child types
3: optional list<TStructField> struct_fields
+
+ // only used for complex types, such as array, map and etc.
+ 4: optional bool contains_null
}
// A flattened representation of a tree of column types obtained by depth-first
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org