You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2024/09/19 02:28:53 UTC
(doris) branch master updated: [Featrue](default value) Support `bitmap_empty` default value (#40364)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 979cb012f88 [Featrue](default value) Support `bitmap_empty` default value (#40364)
979cb012f88 is described below
commit 979cb012f8857712caa3fcf28096b8d91054f085
Author: abmdocrt <Yu...@gmail.com>
AuthorDate: Thu Sep 19 10:28:45 2024 +0800
[Featrue](default value) Support `bitmap_empty` default value (#40364)
---
be/src/olap/partial_update_info.cpp | 6 +
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 3 +-
.../java/org/apache/doris/analysis/ColumnDef.java | 6 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 2 +
.../plans/commands/info/ColumnDefinition.java | 8 +-
.../trees/plans/commands/info/DefaultValue.java | 3 +-
.../correctness_p0/test_default_bitmap_empty.out | 43 +++++++
.../test_default_bitmap_empty_streamload.csv | 2 +
.../test_default_bitmap_empty.groovy | 137 +++++++++++++++++++++
10 files changed, 204 insertions(+), 7 deletions(-)
diff --git a/be/src/olap/partial_update_info.cpp b/be/src/olap/partial_update_info.cpp
index a576096fcd4..b8e528e99e1 100644
--- a/be/src/olap/partial_update_info.cpp
+++ b/be/src/olap/partial_update_info.cpp
@@ -26,6 +26,7 @@
#include "olap/tablet_meta.h"
#include "olap/tablet_schema.h"
#include "olap/utils.h"
+#include "util/bitmap_value.h"
#include "vec/common/assert_cast.h"
#include "vec/core/block.h"
@@ -180,6 +181,11 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids(
DateV2Value<DateV2ValueType> dv;
dv.from_unixtime(timestamp_ms / 1000, timezone);
default_value = dv.debug_string();
+ } else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT &&
+ to_lower(column.default_value()).find(to_lower("BITMAP_EMPTY")) !=
+ std::string::npos)) {
+ BitmapValue v = BitmapValue {};
+ default_value = v.to_string();
} else {
default_value = column.default_value();
}
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 9695c34ebbd..ac6958894d1 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -114,6 +114,7 @@ BINARY: 'BINARY';
BINLOG: 'BINLOG';
BITAND: 'BITAND';
BITMAP: 'BITMAP';
+BITMAP_EMPTY: 'BITMAP_EMPTY';
BITMAP_UNION: 'BITMAP_UNION';
BITOR: 'BITOR';
BITXOR: 'BITXOR';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 8faef475892..b22295dc5c8 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -1323,7 +1323,7 @@ columnDef
((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)?
((NOT)? nullable=NULL)?
(AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)?
- (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | E | stringValue=STRING_LITERAL
+ (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | E | BITMAP_EMPTY | stringValue=STRING_LITERAL
| CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))?
(ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)?
(COMMENT comment=STRING_LITERAL)?
@@ -1787,6 +1787,7 @@ nonReserved
| BIN
| BITAND
| BITMAP
+ | BITMAP_EMPTY
| BITMAP_UNION
| BITOR
| BITXOR
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 625a3b3b131..9bd34e1410b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -427,8 +427,10 @@ public class ColumnDef {
}
if (type.getPrimitiveType() == PrimitiveType.BITMAP) {
- if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE) {
- throw new AnalysisException("Bitmap type column can not set default value");
+ if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE
+ && !defaultValue.value.equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value)) {
+ throw new AnalysisException("Bitmap type column default value only support null or "
+ + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value);
}
defaultValue = DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 55e2369c8c9..3e5ba15b0f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2793,6 +2793,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
defaultValue = Optional.of(DefaultValue.PI_DEFAULT_VALUE);
} else if (ctx.E() != null) {
defaultValue = Optional.of(DefaultValue.E_NUM_DEFAULT_VALUE);
+ } else if (ctx.BITMAP_EMPTY() != null) {
+ defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
}
}
if (ctx.UPDATE() != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 0b2694cc311..184b6d2fa55 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -318,13 +318,15 @@ public class ColumnDefinition {
}
defaultValue = Optional.of(DefaultValue.HLL_EMPTY_DEFAULT_VALUE);
} else if (type.isBitmapType()) {
- if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) {
- throw new AnalysisException("Bitmap type column can not set default value");
+ if (defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE
+ && !defaultValue.get().getValue().equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.getValue())) {
+ throw new AnalysisException("Bitmap type column default value only support "
+ + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
}
defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
} else if (type.isArrayType() && defaultValue.isPresent() && isOlap
&& defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get()
- .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
+ .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
throw new AnalysisException("Array type column default value only support null or "
+ DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE);
} else if (type.isMapType()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java
index 0d9de31157d..e3f388d272d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java
@@ -31,6 +31,7 @@ public class DefaultValue {
public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
public static String NOW = "now";
public static String HLL_EMPTY = "HLL_EMPTY";
+ public static String BITMAP_EMPTY = "BITMAP_EMPTY";
public static DefaultValue CURRENT_DATE_DEFAULT_VALUE = new DefaultValue(CURRENT_DATE, CURRENT_DATE.toLowerCase());
public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new DefaultValue(CURRENT_TIMESTAMP, NOW);
// default null
@@ -40,7 +41,7 @@ public class DefaultValue {
// default "value", "0" means empty hll
public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, HLL_EMPTY);
// default "value", "0" means empty bitmap
- public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO);
+ public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, BITMAP_EMPTY);
// default "value", "[]" means empty array
public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue("[]");
// Lets use the const value from math pacakge.
diff --git a/regression-test/data/correctness_p0/test_default_bitmap_empty.out b/regression-test/data/correctness_p0/test_default_bitmap_empty.out
new file mode 100644
index 00000000000..33cc825e767
--- /dev/null
+++ b/regression-test/data/correctness_p0/test_default_bitmap_empty.out
@@ -0,0 +1,43 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !insert_into1 --
+0
+0
+0
+0
+
+-- !stream_load_csv1 --
+0
+0
+0
+0
+0
+0
+
+-- !select_2 --
+0
+0
+0
+0
+
+-- !stream_load_csv2 --
+0
+0
+0
+0
+0
+0
+
+-- !insert_into3 --
+0
+0
+0
+0
+
+-- !stream_load_csv3 --
+0
+0
+0
+0
+0
+0
+
diff --git a/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv b/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv
new file mode 100644
index 00000000000..f4ec2d7748a
--- /dev/null
+++ b/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv
@@ -0,0 +1,2 @@
+5,5
+6,6
\ No newline at end of file
diff --git a/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy b/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy
new file mode 100644
index 00000000000..60b6ff73cb3
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_default_bitmap_empty") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ def tableName = "test_default_bitmap_empty"
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName}
+ (
+ k TINYINT,
+ v1 bitmap NOT NULL DEFAULT bitmap_empty,
+ v2 INT
+ )
+ UNIQUE KEY(K)
+ DISTRIBUTED BY HASH(k)
+ PROPERTIES("replication_num" = "1");
+ """
+
+ // test insert into.
+ sql " insert into ${tableName} (k, v2) values (1, 1); "
+ sql " insert into ${tableName} (k, v2) values (2, 2); "
+ sql " insert into ${tableName} (k, v2) values (3, 3); "
+ sql " insert into ${tableName} (k, v2) values (4, 4); "
+ sql "sync"
+ qt_insert_into1 """ select bitmap_count(v1) from ${tableName}; """
+
+ // test csv stream load.
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'k, v1=bitmap_empty(), v2'
+
+ file 'test_default_bitmap_empty_streamload.csv'
+
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_stream_load_csv1 """ select bitmap_count(v1) from ${tableName}; """
+
+ // test partial update
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName}
+ (
+ k TINYINT,
+ v1 bitmap NOT NULL DEFAULT bitmap_empty,
+ v2 INT
+ )
+ UNIQUE KEY(K)
+ DISTRIBUTED BY HASH(k)
+ PROPERTIES("replication_num" = "1");
+ """
+
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+
+ sql " insert into ${tableName} (k, v2) values (1, 1); "
+ sql " insert into ${tableName} (k, v2) values (2, 2); "
+ sql " insert into ${tableName} (k, v2) values (3, 3); "
+ sql " insert into ${tableName} (k, v2) values (4, 4); "
+ sql "sync"
+
+ qt_select_2 "select bitmap_count(v1) from ${tableName};"
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'partial_columns', 'true'
+ set 'column_separator', ','
+ set 'columns', 'k, v2'
+
+ file 'test_default_bitmap_empty_streamload.csv'
+
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_stream_load_csv2 """ select bitmap_count(v1) from ${tableName}; """
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName}
+ (
+ k TINYINT,
+ v1 bitmap BITMAP_UNION default BITMAP_EMPTY,
+ v2 INT replace_if_not_null
+ )
+ aggregate KEY(K)
+ DISTRIBUTED BY HASH(k)
+ PROPERTIES("replication_num" = "1");
+ """
+
+ // test insert into.
+ sql " insert into ${tableName} (k, v2) values (1, 1); "
+ sql " insert into ${tableName} (k, v2) values (2, 2); "
+ sql " insert into ${tableName} (k, v2) values (3, 3); "
+ sql " insert into ${tableName} (k, v2) values (4, 4); "
+ sql "sync"
+ qt_insert_into3 """ select bitmap_count(v1) from ${tableName}; """
+
+ // test csv stream load.
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'k, v1=bitmap_empty(), v2'
+
+ file 'test_default_bitmap_empty_streamload.csv'
+
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ qt_stream_load_csv3 """ select bitmap_count(v1) from ${tableName}; """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org