You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/03/01 03:57:28 UTC
[incubator-doris] 01/02: [feature-wip](array-type) Create table with nested array type. (#8003)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch array-type
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 37becad1ff454d2c1f2a78a4fd95de71dbe1a572
Author: Adonis Ling <ad...@gmail.com>
AuthorDate: Wed Feb 16 12:44:30 2022 +0800
[feature-wip](array-type) Create table with nested array type. (#8003)
```
create table array_type_table(k1 INT, k2 Array<Array<int>>) duplicate key (k1)
distributed by hash(k1) buckets 1 properties('replication_num' = '1');
```
---
be/src/olap/tablet_meta.cpp | 9 ++++-----
.../org/apache/doris/analysis/CreateTableStmt.java | 16 ++++------------
.../main/java/org/apache/doris/analysis/TypeDef.java | 10 ++++++++--
.../java/org/apache/doris/catalog/ArrayType.java | 9 +--------
.../main/java/org/apache/doris/catalog/Column.java | 10 ++++------
.../src/main/java/org/apache/doris/catalog/Type.java | 20 +++++++-------------
.../org/apache/doris/catalog/CreateTableTest.java | 13 +++++++++++++
7 files changed, 41 insertions(+), 46 deletions(-)
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 6e6195d..73632a1 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -128,11 +128,6 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
}
}
}
-
- if (tcolumn.column_type.type == TPrimitiveType::ARRAY) {
- ColumnPB* children_column = column->add_children_columns();
- _init_column_from_tcolumn(0, tcolumn.children_column[0], children_column);
- }
}
schema->set_next_column_unique_id(next_unique_id);
@@ -210,6 +205,10 @@ void TabletMeta::_init_column_from_tcolumn(uint32_t unique_id, const TColumn& tc
if (tcolumn.__isset.is_bloom_filter_column) {
column->set_is_bf_column(tcolumn.is_bloom_filter_column);
}
+ if (tcolumn.column_type.type == TPrimitiveType::ARRAY) {
+ ColumnPB* children_column = column->add_children_columns();
+ _init_column_from_tcolumn(0, tcolumn.children_column[0], children_column);
+ }
}
OLAPStatus TabletMeta::create_from_file(const string& file_path) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index ed2b689..5fb937e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -17,8 +17,11 @@
package org.apache.doris.analysis;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.catalog.AggregateType;
-import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Index;
@@ -35,12 +38,6 @@ import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -372,11 +369,6 @@ public class CreateTableStmt extends DdlStmt {
columnDef.analyze(engineName.equals("olap"));
if (columnDef.getType().isArrayType()) {
- ArrayType tp = (ArrayType) columnDef.getType();
- if (!tp.getItemType().getPrimitiveType().isIntegerType() &&
- !tp.getItemType().getPrimitiveType().isCharFamily()) {
- throw new AnalysisException("Array column just support INT/VARCHAR sub-type");
- }
if (columnDef.getAggregateType() != null && columnDef.getAggregateType() != AggregateType.NONE) {
throw new AnalysisException("Array column can't support aggregation " + columnDef.getAggregateType());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
index 6bce9b5..d81d633 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -92,8 +92,10 @@ public class TypeDef implements ParseNode {
throw new AnalysisException("Unsupported data type: " + type.toSql());
}
if (type.isArrayType()) {
- ScalarType itemType = (ScalarType) ((ArrayType) type).getItemType();
- analyzeNestedType(itemType);
+ Type itemType = ((ArrayType) type).getItemType();
+ if (itemType instanceof ScalarType) {
+ analyzeNestedType((ScalarType) itemType);
+ }
}
if (type.isMapType()) {
ScalarType keyType = (ScalarType) ((MapType) type).getKeyType();
@@ -115,6 +117,10 @@ public class TypeDef implements ParseNode {
if (type.isNull()) {
throw new AnalysisException("Unsupported data type: " + type.toSql());
}
+ if (!type.getPrimitiveType().isIntegerType() &&
+ !type.getPrimitiveType().isCharFamily()) {
+ throw new AnalysisException("Array column just support INT/VARCHAR sub-type");
+ }
if (type.getPrimitiveType().isStringType()
&& !type.isAssignedStrLenInColDefinition()) {
type.setLength(1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
index e4c16a1..fff4f05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -83,9 +83,6 @@ public class ArrayType extends Type {
@Override
public String toSql(int depth) {
- if (depth >= MAX_NESTING_DEPTH) {
- return "ARRAY<...>";
- }
return String.format("ARRAY<%s>", itemType.toSql(depth + 1));
}
@@ -125,11 +122,7 @@ public class ArrayType extends Type {
if (!Config.enable_complex_type_support) {
return false;
}
-
- if (itemType.isNull()) {
- return false;
- }
- return true;
+ return !itemType.isNull();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 522d6c0..278055f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -17,6 +17,9 @@
package org.apache.doris.catalog;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
@@ -30,11 +33,6 @@ import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TColumnType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.gson.annotations.SerializedName;
-
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -341,7 +339,6 @@ public class Column implements Writable {
tColumn.setIsAllowNull(this.isAllowNull);
tColumn.setDefaultValue(this.defaultValue);
tColumn.setVisible(visible);
- tColumn.setChildrenColumn(new ArrayList<>());
toChildrenThrift(this, tColumn);
// ATTN:
@@ -370,6 +367,7 @@ public class Column implements Writable {
childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
childrenTColumn.setColumnType(childrenTColumnType);
+ tColumn.setChildrenColumn(new ArrayList<>());
tColumn.children_column.add(childrenTColumn);
toChildrenThrift(children, childrenTColumn);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
index efee561..8d51a1f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
@@ -46,9 +46,10 @@ import java.util.List;
public abstract class Type {
private static final Logger LOG = LogManager.getLogger(Type.class);
- // Maximum nesting depth of a type. This limit was determined experimentally byorg.apache.doris.rewrite.FoldConstantsRule.apply
- // generating and scanning deeply nested Parquet and Avro files. In those experiments,
- // we exceeded the stack space in the scanner (which uses recursion for dealing with
+ // Maximum nesting depth of a type. This limit was determined experimentally by
+ // org.apache.doris.rewrite.FoldConstantsRule.apply generating and scanning
+ // deeply nested Parquet and Avro files. In those experiments, we exceeded
+ // the stack space in the scanner (which uses recursion for dealing with
// nested types) at a nesting depth between 200 and 300 (200 worked, 300 crashed).
public static int MAX_NESTING_DEPTH = 2;
@@ -469,20 +470,13 @@ public abstract class Type {
}
}
} else if (isArrayType()) {
- ArrayType arrayType = (ArrayType) this;
- if (arrayType.getItemType().exceedsMaxNestingDepth(d + 1)) {
- return true;
- }
+ return false;
} else if (isMultiRowType()) {
MultiRowType multiRowType = (MultiRowType) this;
- if (multiRowType.getItemType().exceedsMaxNestingDepth(d + 1)) {
- return true;
- }
+ return multiRowType.getItemType().exceedsMaxNestingDepth(d + 1);
} else if (isMapType()) {
MapType mapType = (MapType) this;
- if (mapType.getValueType().exceedsMaxNestingDepth(d + 1)) {
- return true;
- }
+ return mapType.getValueType().exceedsMaxNestingDepth(d + 1);
} else {
Preconditions.checkState(isScalarType());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 24b3c89..67686a7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -519,4 +519,17 @@ public class CreateTableTest {
" 'data_sort.sort_type' = 'zorder'," +
" 'data_sort.col_num' = '');"));
}
+
+ @Test
+ public void testCreateTableWithArrayType() throws Exception {
+ Config.enable_complex_type_support = true;
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.table1(k1 INT, k2 Array<int>) duplicate key (k1) " +
+ "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
+ });
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.table2(k1 INT, k2 Array<Array<int>>) duplicate key (k1) " +
+ "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
+ });
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org