You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/10 14:56:31 UTC
[doris] branch branch-1.2-lts updated (97775be474 -> 0b842e5aa2)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 97775be474 [cherry-pick](udaf) fix java-udaf can't exectue add function (#20663)
new 46effbfad0 [fix] (clone) fix drop biggest version replica during reblance step (#20107)
new f01f8784cb [fix](merge-on-write) fix that set_txn_related_delete_bitmap may coredump (#20300)
new a8141babcb [fix](dynamic partition) partition create failed after alter distributed column (#20239)
new 9b684d52e4 [fix](sequence) value predicates shouldn't be push down when has sequence column (#20408)
new 3554187d16 [Fix] (tablet) fix tablet queryable set (#20413) (#20414)
new 0f1020ca51 [bug](jdbc) fix trino date/datetime filter (#20443)
new 82a0d918eb [fix](case expr) fix coredump of case for null value (#20564)
new ccdc63634d [fix](replay) fix truncate partition name need case insensitive (#20098)
new 7b7a2619a5 [Fix](2PC) fix timeout config is not avaible for commit phase in 2pc (#20423)
new 0b842e5aa2 [branch1.2](fix) BE code style
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/olap/rowset/beta_rowset_reader.cpp | 6 +-
be/src/olap/txn_manager.cpp | 8 +-
be/src/vec/exec/vset_operation_node.cpp | 3 +-
be/src/vec/functions/function_case.h | 8 +-
.../doris/analysis/HashDistributionDesc.java | 4 +-
.../main/java/org/apache/doris/catalog/Column.java | 22 +++
.../main/java/org/apache/doris/catalog/Env.java | 25 ++-
.../apache/doris/catalog/HashDistributionInfo.java | 14 +-
.../main/java/org/apache/doris/catalog/Tablet.java | 9 ++
.../org/apache/doris/clone/TabletScheduler.java | 23 ++-
.../apache/doris/datasource/InternalCatalog.java | 15 +-
.../org/apache/doris/planner/OdbcScanNode.java | 19 +++
.../apache/doris/service/FrontendServiceImpl.java | 3 +-
.../unique/test_unique_value_predicate.out} | 60 ++-----
.../sql_functions/case_function/case_null0.csv | 3 +
.../sql_functions/case_function/case_null1.csv | 11 ++
.../case_function/test_case_function_null.out} | 6 +-
regression-test/pipeline/p0/conf/fe.conf | 2 +
regression-test/pipeline/p1/conf/fe.conf | 2 +
.../unique/test_unique_value_predicate.groovy | 111 +++++++++++++
.../suites/ddl_p0/test_truncate_table.groovy | 63 ++++++++
.../test_dynamic_partition_with_alter.groovy | 56 +++++++
.../test_dynamic_partition_with_rename.groovy | 60 +++++++
.../case_function/test_case_function_null.groovy | 176 +++++++++++++++++++++
24 files changed, 624 insertions(+), 85 deletions(-)
copy regression-test/data/{correctness_p0/test_outer_join_with_subquery.out => data_model_p0/unique/test_unique_value_predicate.out} (53%)
create mode 100644 regression-test/data/query_p0/sql_functions/case_function/case_null0.csv
create mode 100644 regression-test/data/query_p0/sql_functions/case_function/case_null1.csv
copy regression-test/data/{brown_p2/sql/dataTransferRate.out => query_p0/sql_functions/case_function/test_case_function_null.out} (58%)
create mode 100644 regression-test/suites/data_model_p0/unique/test_unique_value_predicate.groovy
create mode 100644 regression-test/suites/ddl_p0/test_truncate_table.groovy
create mode 100644 regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy
create mode 100644 regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_rename.groovy
create mode 100644 regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/10: [bug](jdbc) fix trino date/datetime filter (#20443)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0f1020ca515b7ab2b3f2950301e0e581a56bc43b
Author: zy-kkk <zh...@gmail.com>
AuthorDate: Tue Jun 6 11:20:42 2023 +0800
[bug](jdbc) fix trino date/datetime filter (#20443)
When querying Trino's JDBC catalog, if our WHERE filter condition is k1 >= '2022-01-01', this format is incorrect.
In Trino, the correct format should be k1 >= date '2022-01-01' or k1 >= timestamp '2022-01-01 00:00:00'.
Therefore, the date string in the WHERE condition needs to be converted to the date or timestamp format supported by Trino.
---
.../java/org/apache/doris/planner/OdbcScanNode.java | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
index f47be29d38..d606a1ae93 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
@@ -82,6 +82,25 @@ public class OdbcScanNode extends ScanNode {
return filter;
}
}
+ if (tableType.equals(TOdbcTableType.TRINO) && expr.contains(DateLiteral.class)
+ && (expr instanceof BinaryPredicate)) {
+ ArrayList<Expr> children = expr.getChildren();
+ if (children.get(1).isConstant() && (children.get(1).getType().isDate()) || children
+ .get(1).getType().isDateV2()) {
+ String filter = children.get(0).toSql();
+ filter += ((BinaryPredicate) expr).getOp().toString();
+ filter += "date '" + children.get(1).getStringValue() + "'";
+ return filter;
+ }
+ if (children.get(1).isConstant() && (children.get(1).getType().isDatetime() || children
+ .get(1).getType().isDatetimeV2())) {
+ String filter = children.get(0).toSql();
+ filter += ((BinaryPredicate) expr).getOp().toString();
+ filter += "timestamp '" + children.get(1).getStringValue() + "'";
+ return filter;
+ }
+ }
+
return expr.toMySql();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/10: [fix] (clone) fix drop biggest version replica during reblance step (#20107)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 46effbfad0308ab601118ac680b1a93738715bf5
Author: yujun <yu...@gmail.com>
AuthorDate: Mon May 29 09:00:51 2023 +0800
[fix] (clone) fix drop biggest version replica during reblance step (#20107)
* add check for rebalancer choose deleted replica
* impr a compare
---
.../java/org/apache/doris/clone/TabletScheduler.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
index c672976b92..8023fe2be5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -991,11 +991,22 @@ public class TabletScheduler extends MasterDaemon {
return false;
}
Replica chosenReplica = tabletCtx.getTablet().getReplicaById(id);
- if (chosenReplica != null) {
- deleteReplicaInternal(tabletCtx, chosenReplica, "src replica of rebalance", force);
- return true;
+ if (chosenReplica == null) {
+ return false;
}
- return false;
+ List<Replica> replicas = tabletCtx.getTablet().getReplicas();
+ int eqOrNewVersionCount = 0;
+ for (Replica replica : replicas) {
+ if (replica.getVersion() >= chosenReplica.getVersion()) {
+ eqOrNewVersionCount++;
+ }
+ }
+ if (eqOrNewVersionCount == 1) {
+ return false;
+ }
+ deleteReplicaInternal(tabletCtx, chosenReplica, "src replica of rebalance", force);
+
+ return true;
}
private boolean deleteReplicaOnHighLoadBackend(TabletSchedCtx tabletCtx, boolean force) throws SchedException {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 08/10: [fix](replay) fix truncate partition name need case insensitive (#20098)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ccdc63634dca6b9c0376a11d2a4f3cf5737bb732
Author: xueweizhang <zx...@163.com>
AuthorDate: Fri Jun 9 09:34:55 2023 +0800
[fix](replay) fix truncate partition name need case insensitive (#20098)
truncate table with partition name need case insensitive
---
.../apache/doris/datasource/InternalCatalog.java | 2 +-
.../suites/ddl_p0/test_truncate_table.groovy | 63 ++++++++++++++++++++++
2 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index ce760a38bf..beaf2c4794 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2535,7 +2535,7 @@ public class InternalCatalog implements CatalogIf<Database> {
TableName dbTbl = tblRef.getName();
// check, and save some info which need to be checked again later
- Map<String, Long> origPartitions = Maps.newHashMap();
+ Map<String, Long> origPartitions = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
Map<Long, DistributionInfo> partitionsDistributionInfo = Maps.newHashMap();
OlapTable copiedTbl;
diff --git a/regression-test/suites/ddl_p0/test_truncate_table.groovy b/regression-test/suites/ddl_p0/test_truncate_table.groovy
new file mode 100644
index 0000000000..e983915f22
--- /dev/null
+++ b/regression-test/suites/ddl_p0/test_truncate_table.groovy
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_truncate_table") {
+ def testTable = "test_truncate_table"
+
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ sql """
+ CREATE TABLE ${testTable}
+ (
+ k1 DATE,
+ k2 DECIMAL(10, 2) DEFAULT "10.5",
+ k3 CHAR(10) COMMENT "string column",
+ k4 INT NOT NULL DEFAULT "1" COMMENT "int column"
+ )
+ PARTITION BY RANGE(k1)
+ (
+ PARTITION p1 VALUES LESS THAN ("2020-02-01"),
+ PARTITION p2 VALUES LESS THAN ("2020-03-01"),
+ PARTITION p3 VALUES LESS THAN ("2020-04-01")
+ )
+ DISTRIBUTED BY HASH(k2) BUCKETS 32
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ List<List<Object>> result = sql "show partitions from ${testTable}"
+ logger.info("${result}")
+ assertEquals(result.size(), 3)
+ assertEquals(result.get(0).get(1), "p1")
+
+ sql """truncate table ${testTable};"""
+ result = sql "show partitions from ${testTable}"
+ logger.info("${result}")
+ assertEquals(result.size(), 3)
+ assertEquals(result.get(0).get(1), "p1")
+
+ sql """truncate table ${testTable} partitions (p1, P1);"""
+
+ result = sql "show partitions from ${testTable}"
+ logger.info("${result}")
+ assertEquals(result.size(), 3)
+ assertEquals(result.get(0).get(1), "p1")
+
+ sql "DROP TABLE IF EXISTS ${testTable}"
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/10: [fix](sequence) value predicates shouldn't be push down when has sequence column (#20408)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9b684d52e4c31136309f45da2648e53ab8fe069d
Author: Xin Liao <li...@126.com>
AuthorDate: Mon Jun 5 19:18:34 2023 +0800
[fix](sequence) value predicates shouldn't be push down when has sequence column (#20408)
* (fix)[sequence] value predicates shouldn't be push down when has sequence column
* add case
---
be/src/olap/rowset/beta_rowset_reader.cpp | 6 +-
.../unique/test_unique_value_predicate.out | 45 +++++++++
.../unique/test_unique_value_predicate.groovy | 111 +++++++++++++++++++++
3 files changed, 160 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index fa1f965ea2..24c1265049 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -384,10 +384,12 @@ Status BetaRowsetReader::next_block_view(vectorized::BlockView* block_view) {
bool BetaRowsetReader::_should_push_down_value_predicates() const {
// if unique table with rowset [0-x] or [0-1] [2-y] [...],
- // value column predicates can be pushdown on rowset [0-x] or [2-y], [2-y] must be compaction and not overlapping
+ // value column predicates can be pushdown on rowset [0-x] or [2-y], [2-y]
+ // must be compaction, not overlapping and don't have sequence column
return _rowset->keys_type() == UNIQUE_KEYS &&
(((_rowset->start_version() == 0 || _rowset->start_version() == 2) &&
- !_rowset->_rowset_meta->is_segments_overlapping()) ||
+ !_rowset->_rowset_meta->is_segments_overlapping() &&
+ _context->sequence_id_idx == -1) ||
_context->enable_unique_key_merge_on_write);
}
diff --git a/regression-test/data/data_model_p0/unique/test_unique_value_predicate.out b/regression-test/data/data_model_p0/unique/test_unique_value_predicate.out
new file mode 100644
index 0000000000..0fe421460f
--- /dev/null
+++ b/regression-test/data/data_model_p0/unique/test_unique_value_predicate.out
@@ -0,0 +1,45 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+0 1 test char 2000-01-01
+
+-- !select --
+0 1 test char 2000-01-01
+
+-- !select --
+
+-- !select --
+0 1 test char 2000-01-01
+
+-- !select --
+0 2 test int 2000-02-02
+
+-- !select --
+0 2 test int 2000-02-02
+
+-- !select --
+0 2 test int 2000-02-02
+
+-- !select --
+
+-- !select --
+0 1 test char 2000-01-01
+
+-- !select --
+0 1 test char 2000-01-01
+
+-- !select --
+
+-- !select --
+0 1 test char 2000-01-01
+
+-- !select --
+0 2 test int 2000-02-02
+
+-- !select --
+0 2 test int 2000-02-02
+
+-- !select --
+0 2 test int 2000-02-02
+
+-- !select --
+
diff --git a/regression-test/suites/data_model_p0/unique/test_unique_value_predicate.groovy b/regression-test/suites/data_model_p0/unique/test_unique_value_predicate.groovy
new file mode 100644
index 0000000000..c5a1e5b4ca
--- /dev/null
+++ b/regression-test/suites/data_model_p0/unique/test_unique_value_predicate.groovy
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_unique_value_predicate") {
+ // test uniq table
+ def tbName = "test_uniq_value_predicate"
+ // mor without seq
+ sql "DROP TABLE IF EXISTS ${tbName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName} (
+ k int,
+ int_value int,
+ char_value char(10),
+ date_value date
+ )
+ UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1",
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "false");
+ """
+ sql "insert into ${tbName} values(0, 2, 'test int', '2000-02-02')"
+ sql "insert into ${tbName} values(0, 1, 'test char', '2000-01-01')"
+ qt_select "select * from ${tbName}"
+ qt_select "select * from ${tbName} where k = 0"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 2"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 1"
+
+ // mor with seq
+ sql "DROP TABLE IF EXISTS ${tbName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName} (
+ k int,
+ int_value int,
+ char_value char(10),
+ date_value date
+ )
+ UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1",
+ "function_column.sequence_col" = "int_value",
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "false");
+ """
+ sql "insert into ${tbName} values(0, 2, 'test int', '2000-02-02')"
+ sql "insert into ${tbName} values(0, 1, 'test char', '2000-01-01')"
+ qt_select "select * from ${tbName}"
+ qt_select "select * from ${tbName} where k = 0"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 2"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 1"
+
+ // mow without seq
+ sql "DROP TABLE IF EXISTS ${tbName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName} (
+ k int,
+ int_value int,
+ char_value char(10),
+ date_value date
+ )
+ UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1",
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "true");
+ """
+ sql "insert into ${tbName} values(0, 2, 'test int', '2000-02-02')"
+ sql "insert into ${tbName} values(0, 1, 'test char', '2000-01-01')"
+ qt_select "select * from ${tbName}"
+ qt_select "select * from ${tbName} where k = 0"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 2"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 1"
+
+ // mow with seq
+ sql "DROP TABLE IF EXISTS ${tbName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName} (
+ k int,
+ int_value int,
+ char_value char(10),
+ date_value date
+ )
+ UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1",
+ "function_column.sequence_col" = "int_value",
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "true");
+ """
+ sql "insert into ${tbName} values(0, 2, 'test int', '2000-02-02')"
+ sql "insert into ${tbName} values(0, 1, 'test char', '2000-01-01')"
+ qt_select "select * from ${tbName}"
+ qt_select "select * from ${tbName} where k = 0"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 2"
+ qt_select "select * from ${tbName} where k = 0 && int_value = 1"
+ sql "DROP TABLE ${tbName}"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/10: [fix](case expr) fix coredump of case for null value (#20564)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 82a0d918eb9706c815c5d9582ce7f03527e681b0
Author: TengJianPing <18...@users.noreply.github.com>
AuthorDate: Thu Jun 8 20:05:23 2023 +0800
[fix](case expr) fix coredump of case for null value (#20564)
be coredump when when expr is null:
---
be/src/vec/functions/function_case.h | 8 +-
.../sql_functions/case_function/case_null0.csv | 3 +
.../sql_functions/case_function/case_null1.csv | 11 ++
.../case_function/test_case_function_null.out | 6 +
.../case_function/test_case_function_null.groovy | 176 +++++++++++++++++++++
5 files changed, 200 insertions(+), 4 deletions(-)
diff --git a/be/src/vec/functions/function_case.h b/be/src/vec/functions/function_case.h
index 5c2b39d7f8..a4049f7840 100644
--- a/be/src/vec/functions/function_case.h
+++ b/be/src/vec/functions/function_case.h
@@ -301,13 +301,13 @@ public:
bool then_null = false;
for (int i = 1 + has_case; i < arguments.size() - has_else; i += 2) {
auto then_column_ptr = block.get_by_position(arguments[i]).column;
- if (then_column_ptr->is_nullable()) {
+ if (then_column_ptr->is_nullable() || then_column_ptr->only_null()) {
then_null = true;
}
}
if constexpr (has_else) {
auto else_column_ptr = block.get_by_position(arguments[arguments.size() - 1]).column;
- if (else_column_ptr->is_nullable()) {
+ if (else_column_ptr->is_nullable() || else_column_ptr->only_null()) {
then_null = true;
}
} else {
@@ -333,13 +333,13 @@ public:
bool when_null = false;
if constexpr (has_case) {
auto case_column_ptr = block.get_by_position(arguments[0]).column;
- if (case_column_ptr->is_nullable()) {
+ if (case_column_ptr->is_nullable() || case_column_ptr->only_null()) {
when_null = true;
}
}
for (int i = has_case; i < arguments.size() - has_else; i += 2) {
auto when_column_ptr = block.get_by_position(arguments[i]).column;
- if (when_column_ptr->is_nullable()) {
+ if (when_column_ptr->is_nullable() || when_column_ptr->only_null()) {
when_null = true;
}
}
diff --git a/regression-test/data/query_p0/sql_functions/case_function/case_null0.csv b/regression-test/data/query_p0/sql_functions/case_function/case_null0.csv
new file mode 100644
index 0000000000..5275e77971
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/case_function/case_null0.csv
@@ -0,0 +1,3 @@
+1.0 1 1970-11-13
+0.7 0 1970-01-04
+\N 0 1970-10-29
diff --git a/regression-test/data/query_p0/sql_functions/case_function/case_null1.csv b/regression-test/data/query_p0/sql_functions/case_function/case_null1.csv
new file mode 100644
index 0000000000..3b211b2d1f
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/case_function/case_null1.csv
@@ -0,0 +1,11 @@
+
+kejc8^H
+
+
+
+
+)ꪁ
+
+
+
+kꪁNhR
diff --git a/regression-test/data/query_p0/sql_functions/case_function/test_case_function_null.out b/regression-test/data/query_p0/sql_functions/case_function/test_case_function_null.out
new file mode 100644
index 0000000000..e671ae0ee4
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/case_function/test_case_function_null.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql_case0 --
+1970-01-04 false
+1970-10-29 false
+1970-11-13 true
+
diff --git a/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
new file mode 100644
index 0000000000..e625023525
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_case_function_null", "query,p0") {
+ sql """ drop table if exists case_null0 """
+ sql """ create table case_null0 (
+ `c0` decimalv3(17, 1) NULL,
+ `c1` boolean NOT NULL,
+ `c2` date NOT NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`c0`, `c1`, `c2`)
+ DISTRIBUTED BY RANDOM BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """ drop table if exists `case_null1` """
+ sql """
+ CREATE TABLE `case_null1` (
+ `c0` varchar(7) NOT NULL DEFAULT ""
+ ) ENGINE=OLAP
+ AGGREGATE KEY(`c0`)
+ DISTRIBUTED BY RANDOM BUCKETS 24
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ def tables = ["case_null0", "case_null1"]
+
+ for (String tableName in tables) {
+ streamLoad {
+ // a default db 'regression_test' is specified in
+ // ${DORIS_HOME}/conf/regression-conf.groovy
+ table tableName
+
+ // default label is UUID:
+ // set 'label' UUID.randomUUID().toString()
+
+ // default column_separator is specify in doris fe config, usually is '\t'.
+ // this line change to ','
+ set 'column_separator', '\t'
+ // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+ // also, you can stream load a http stream, e.g. http://xxx/some.csv
+ file """${tableName}.csv"""
+
+ time 10000 // limit inflight 10s
+
+ // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+
+ // if declared a check callback, the default check condition will ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+ }
+ }
+ }
+
+ qt_sql_case0 """
+ SELECT * FROM (
+ SELECT
+ case_null0.c2 c2,
+ case_null0.c1 c1
+ FROM
+ case_null0,
+ case_null1
+ WHERE
+ CASE
+ (
+ CAST(TIMESTAMP '1970-10-27 14:44:27' AS DATE) BETWEEN DATE '1970-08-05'
+ AND DATE '1970-07-12'
+ )
+ WHEN CAST(NULL AS BOOLEAN) THEN (NULL IN (NULL))
+ WHEN (('쬊o') !=('Sf^M7mir')) THEN (
+ ('0.24865') LIKE (
+ CASE
+ false
+ WHEN case_null0.c1 THEN case_null1.c0
+ WHEN case_null0.c1 THEN case_null1.c0
+ END
+ )
+ )
+ END
+ GROUP BY
+ case_null0.c2,
+ case_null0.c1,
+ case_null1.c0
+ UNION
+ SELECT
+ case_null0.c2 c2,
+ case_null0.c1 c1
+ FROM
+ case_null0,
+ case_null1
+ WHERE
+ (
+ NOT CASE
+ (
+ CAST(TIMESTAMP '1970-10-27 14:44:27' AS DATE) BETWEEN DATE '1970-08-05'
+ AND DATE '1970-07-12'
+ )
+ WHEN CAST(NULL AS BOOLEAN) THEN (NULL IN (NULL))
+ WHEN (('쬊o') !=('Sf^M7mir')) THEN (
+ ('0.24865') LIKE (
+ CASE
+ false
+ WHEN case_null0.c1 THEN case_null1.c0
+ WHEN case_null0.c1 THEN case_null1.c0
+ END
+ )
+ )
+ END
+ )
+ GROUP BY
+ case_null0.c2,
+ case_null0.c1,
+ case_null1.c0
+ UNION
+ SELECT
+ case_null0.c2 c2,
+ case_null0.c1 c1
+ FROM
+ case_null0,
+ case_null1
+ WHERE
+ (
+ (
+ CASE
+ (
+ CAST(TIMESTAMP '1970-10-27 14:44:27' AS DATE) BETWEEN DATE '1970-08-05'
+ AND DATE '1970-07-12'
+ )
+ WHEN CAST(NULL AS BOOLEAN) THEN (NULL IN (NULL))
+ WHEN (('쬊o') !=('Sf^M7mir')) THEN (
+ ('0.24865') like (
+ CASE
+ false
+ WHEN case_null0.c1 THEN case_null1.c0
+ WHEN case_null0.c1 THEN case_null1.c0
+ END
+ )
+ )
+ END
+ ) IS NULL
+ )
+ GROUP BY
+ case_null0.c2,
+ case_null0.c1,
+ case_null1.c0
+ ) a
+ order BY
+ c2,
+ c1;
+ """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/10: [fix](merge-on-write) fix that set_txn_related_delete_bitmap may coredump (#20300)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f01f8784cbc18dcdcefb75cdfe4583a7303397d3
Author: Xin Liao <li...@126.com>
AuthorDate: Tue Jun 6 17:49:01 2023 +0800
[fix](merge-on-write) fix that set_txn_related_delete_bitmap may coredump (#20300)
---
be/src/olap/txn_manager.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index 945302fa18..8f9f5b643c 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -184,14 +184,18 @@ void TxnManager::set_txn_related_delete_bitmap(TPartitionId partition_id,
std::lock_guard<std::shared_mutex> wrlock(_get_txn_map_lock(transaction_id));
txn_tablet_map_t& txn_tablet_map = _get_txn_tablet_map(transaction_id);
auto it = txn_tablet_map.find(key);
- DCHECK(it != txn_tablet_map.end());
if (it == txn_tablet_map.end()) {
LOG(WARNING) << "transaction_id: " << transaction_id
<< " partition_id: " << partition_id << " may be cleared";
return;
}
auto load_itr = it->second.find(tablet_info);
- DCHECK(load_itr != it->second.end());
+ if (load_itr == it->second.end()) {
+ LOG(WARNING) << "transaction_id: " << transaction_id
+ << " partition_id: " << partition_id << " tablet_id: " << tablet_id
+ << " may be cleared";
+ return;
+ }
TabletTxnInfo& load_info = load_itr->second;
load_info.unique_key_merge_on_write = unique_key_merge_on_write;
load_info.delete_bitmap = delete_bitmap;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 09/10: [Fix](2PC) fix timeout config is not avaible for commit phase in 2pc (#20423)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7b7a2619a56f2eba80996c4c70dd337432047126
Author: GoGoWen <82...@users.noreply.github.com>
AuthorDate: Fri Jun 9 09:33:58 2023 +0800
[Fix](2PC) fix timeout config is not avaible for commit phase in 2pc (#20423)
fix config::txn_commit_rpc_timeout_ms is not available for commit phase in 2pc.
---
.../src/main/java/org/apache/doris/service/FrontendServiceImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 4be9605815..638f172660 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -786,8 +786,9 @@ public class FrontendServiceImpl implements FrontendService.Iface {
}
if (txnOperation.equalsIgnoreCase("commit")) {
+ long timeoutMs = request.isSetThriftRpcTimeoutMs() ? request.getThriftRpcTimeoutMs() / 2 : 5000;
Env.getCurrentGlobalTransactionMgr()
- .commitTransaction2PC(database, tableList, request.getTxnId(), 5000);
+ .commitTransaction2PC(database, tableList, request.getTxnId(), timeoutMs);
} else if (txnOperation.equalsIgnoreCase("abort")) {
Env.getCurrentGlobalTransactionMgr().abortTransaction2PC(database.getId(), request.getTxnId(), tableList);
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 10/10: [branch1.2](fix) BE code style
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0b842e5aa235bdacf704482f516e8b8c6394b097
Author: morningman <mo...@163.com>
AuthorDate: Sat Jun 10 22:56:20 2023 +0800
[branch1.2](fix) BE code style
---
be/src/vec/exec/vset_operation_node.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/be/src/vec/exec/vset_operation_node.cpp b/be/src/vec/exec/vset_operation_node.cpp
index 418aa06d85..3bd1fe9d82 100644
--- a/be/src/vec/exec/vset_operation_node.cpp
+++ b/be/src/vec/exec/vset_operation_node.cpp
@@ -26,8 +26,7 @@ namespace vectorized {
template <class HashTableContext>
struct HashTableBuild {
HashTableBuild(int rows, Block& acquired_block, ColumnRawPtrs& build_raw_ptrs,
- VSetOperationNode* operation_node, uint8_t offset,
- RuntimeState* state)
+ VSetOperationNode* operation_node, uint8_t offset, RuntimeState* state)
: _rows(rows),
_offset(offset),
_acquired_block(acquired_block),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/10: [fix](dynamic partition) partition create failed after alter distributed column (#20239)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a8141babcb69a77a56e18b4cb487464db30091f6
Author: camby <zh...@baidu.com>
AuthorDate: Mon Jun 5 12:20:50 2023 +0800
[fix](dynamic partition) partition create failed after alter distributed column (#20239)
This pr fix following two problems:
Problem1: Alter column comment make add dynamic partition failed inside issue #10811
create table with dynamic partition policy;
restart FE;
alter distribution column comment;
alter dynamic_partition.end to trigger add new partition by dynamic partition scheduler;
Then we got the error log, and the new partition create failed.
dynamic add partition failed: errCode = 2, detailMessage = Cannot assign hash distribution with different distribution cols. default is: [id int(11) NULL COMMENT 'new_comment_of_id'], db: default_cluster:example_db, table: test_2
Problem2: rename distributed column, make old partition insert failed. inside #20405
The key point of the reproduce steps is restart FE.
It seems all versions will be affected, include master and lts-1.1 and so on.
---
.../doris/analysis/HashDistributionDesc.java | 4 +-
.../main/java/org/apache/doris/catalog/Column.java | 22 ++++++++
.../main/java/org/apache/doris/catalog/Env.java | 25 ++++++---
.../apache/doris/catalog/HashDistributionInfo.java | 14 ++++-
.../apache/doris/datasource/InternalCatalog.java | 13 ++---
regression-test/pipeline/p0/conf/fe.conf | 2 +
regression-test/pipeline/p1/conf/fe.conf | 2 +
.../test_dynamic_partition_with_alter.groovy | 56 ++++++++++++++++++++
.../test_dynamic_partition_with_rename.groovy | 60 ++++++++++++++++++++++
9 files changed, 181 insertions(+), 17 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java
index 8b312a6ba7..4298db6939 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java
@@ -125,7 +125,9 @@ public class HashDistributionDesc extends DistributionDesc {
+ column.getName() + "].");
}
- distributionColumns.add(column);
+ // distribution info and base columns persist seperately inside OlapTable, so we need deep copy
+ // to avoid modify table columns also modify columns inside distribution info.
+ distributionColumns.add(new Column(column));
find = true;
break;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 03b8c99b24..3f174b8362 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -659,6 +659,28 @@ public class Column implements Writable, GsonPostProcessable {
&& Objects.equals(realDefaultValue, other.realDefaultValue);
}
+ // distribution column compare only care about attrs which affect data,
+ // do not care about attrs, such as comment
+ public boolean equalsForDistribution(Column other) {
+ if (other == this) {
+ return true;
+ }
+
+ return name.equalsIgnoreCase(other.name)
+ && Objects.equals(getDefaultValue(), other.getDefaultValue())
+ && Objects.equals(aggregationType, other.aggregationType)
+ && isAggregationTypeImplicit == other.isAggregationTypeImplicit
+ && isKey == other.isKey
+ && isAllowNull == other.isAllowNull
+ && getDataType().equals(other.getDataType())
+ && getStrLen() == other.getStrLen()
+ && getPrecision() == other.getPrecision()
+ && getScale() == other.getScale()
+ && visible == other.visible
+ && Objects.equals(children, other.children)
+ && Objects.equals(realDefaultValue, other.realDefaultValue);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
String json = GsonUtils.GSON.toJson(this);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 306a52e39f..d6ce141ca0 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4162,6 +4162,7 @@ public class Env {
// 4. modify distribution info
DistributionInfo distributionInfo = table.getDefaultDistributionInfo();
if (distributionInfo.getType() == DistributionInfoType.HASH) {
+ // modify default distribution info
List<Column> distributionColumns = ((HashDistributionInfo) distributionInfo).getDistributionColumns();
for (Column column : distributionColumns) {
if (column.getName().equalsIgnoreCase(colName)) {
@@ -4169,6 +4170,19 @@ public class Env {
break;
}
}
+ // modify distribution info inside partitions
+ for (Partition p : table.getPartitions()) {
+ DistributionInfo partDistInfo = p.getDistributionInfo();
+ if (partDistInfo.getType() != DistributionInfoType.HASH) {
+ continue;
+ }
+ List<Column> partDistColumns = ((HashDistributionInfo) partDistInfo).getDistributionColumns();
+ for (Column column : partDistColumns) {
+ if (column.getName().equalsIgnoreCase(colName)) {
+ column.setName(newColName);
+ }
+ }
+ }
}
// 5. modify sequence map col
@@ -4398,13 +4412,10 @@ public class Env {
}
if (distributionInfo.getType() == DistributionInfoType.HASH) {
HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo;
- List<Column> newDistriCols = hashDistributionInfo.getDistributionColumns();
- List<Column> defaultDistriCols
- = ((HashDistributionInfo) defaultDistributionInfo).getDistributionColumns();
- if (!newDistriCols.equals(defaultDistriCols)) {
- throw new DdlException(
- "Cannot assign hash distribution with different distribution cols. " + "default is: "
- + defaultDistriCols);
+ if (!hashDistributionInfo.sameDistributionColumns((HashDistributionInfo) defaultDistributionInfo)) {
+ throw new DdlException("Cannot assign hash distribution with different distribution cols. "
+ + "new is: " + hashDistributionInfo.getDistributionColumns() + " default is: "
+ + ((HashDistributionInfo) distributionInfo).getDistributionColumns());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
index d746bd355f..5f30bc2098 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
@@ -95,6 +95,18 @@ public class HashDistributionInfo extends DistributionInfo {
return distributionInfo;
}
+ public boolean sameDistributionColumns(HashDistributionInfo other) {
+ if (distributionColumns.size() != other.distributionColumns.size()) {
+ return false;
+ }
+ for (int i = 0; i < distributionColumns.size(); ++i) {
+ if (!distributionColumns.get(i).equalsForDistribution(other.distributionColumns.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -107,7 +119,7 @@ public class HashDistributionInfo extends DistributionInfo {
return false;
}
HashDistributionInfo that = (HashDistributionInfo) o;
- return bucketNum == that.bucketNum && Objects.equals(distributionColumns, that.distributionColumns);
+ return bucketNum == that.bucketNum && sameDistributionColumns(that);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index c25f2f1044..ce760a38bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1383,17 +1383,14 @@ public class InternalCatalog implements CatalogIf<Database> {
if (distributionInfo.getType() == DistributionInfoType.HASH) {
HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo;
- List<Column> newDistriCols = hashDistributionInfo.getDistributionColumns();
- List<Column> defaultDistriCols = ((HashDistributionInfo) defaultDistributionInfo)
- .getDistributionColumns();
- if (!newDistriCols.equals(defaultDistriCols)) {
- throw new DdlException(
- "Cannot assign hash distribution with different distribution cols. " + "default is: "
- + defaultDistriCols);
- }
if (hashDistributionInfo.getBucketNum() <= 0) {
throw new DdlException("Cannot assign hash distribution buckets less than 1");
}
+ if (!hashDistributionInfo.sameDistributionColumns((HashDistributionInfo) defaultDistributionInfo)) {
+ throw new DdlException("Cannot assign hash distribution with different distribution cols. "
+ + "new is: " + hashDistributionInfo.getDistributionColumns() + " default is: "
+ + ((HashDistributionInfo) distributionInfo).getDistributionColumns());
+ }
} else if (distributionInfo.getType() == DistributionInfoType.RANDOM) {
RandomDistributionInfo randomDistributionInfo = (RandomDistributionInfo) distributionInfo;
if (randomDistributionInfo.getBucketNum() <= 0) {
diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf
index fe7637add4..8d24d97aca 100644
--- a/regression-test/pipeline/p0/conf/fe.conf
+++ b/regression-test/pipeline/p0/conf/fe.conf
@@ -72,3 +72,5 @@ enable_outfile_to_local = true
tablet_create_timeout_second=20
remote_fragment_exec_timeout_ms=15000
use_fuzzy_session_variable=true
+dynamic_partition_check_interval_seconds=5
+
diff --git a/regression-test/pipeline/p1/conf/fe.conf b/regression-test/pipeline/p1/conf/fe.conf
index 6dfd804972..46ae8f860d 100644
--- a/regression-test/pipeline/p1/conf/fe.conf
+++ b/regression-test/pipeline/p1/conf/fe.conf
@@ -73,3 +73,5 @@ enable_outfile_to_local = true
tablet_create_timeout_second=20
remote_fragment_exec_timeout_ms=15000
use_fuzzy_session_variable=true
+dynamic_partition_check_interval_seconds=5
+
diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy
new file mode 100644
index 0000000000..32cfc742a7
--- /dev/null
+++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_dynamic_partition_with_alter") {
+ def tbl = "test_dynamic_partition_with_alter"
+ sql "drop table if exists ${tbl}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbl}
+ ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
+ AGGREGATE KEY(k1,k2)
+ PARTITION BY RANGE(k1) ( )
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "dynamic_partition.enable"="true",
+ "dynamic_partition.end"="3",
+ "dynamic_partition.buckets"="1",
+ "dynamic_partition.start"="-3",
+ "dynamic_partition.prefix"="p",
+ "dynamic_partition.time_unit"="DAY",
+ "dynamic_partition.create_history_partition"="true",
+ "dynamic_partition.replication_allocation" = "tag.location.default: 1")
+ """
+ result = sql "show partitions from ${tbl}"
+ assertEquals(7, result.size())
+
+ // modify distributed column comment, then try to add too more dynamic partition
+ sql """ alter table ${tbl} modify column k1 comment 'new_comment_for_k1' """
+ sql """ ADMIN SET FRONTEND CONFIG ('dynamic_partition_check_interval_seconds' = '1') """
+ sql """ alter table ${tbl} set('dynamic_partition.end'='5') """
+ result = sql "show partitions from ${tbl}"
+ for (def retry = 0; retry < 15; retry++) {
+ if (result.size() == 9) {
+ break;
+ }
+ logger.info("wait dynamic partition scheduler, sleep 1s")
+ sleep(1000);
+ result = sql "show partitions from ${tbl}"
+ }
+ assertEquals(9, result.size())
+
+ sql "drop table ${tbl}"
+}
diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_rename.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_rename.groovy
new file mode 100644
index 0000000000..b07a2f1a63
--- /dev/null
+++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_rename.groovy
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_dynamic_partition_with_rename") {
+ def tbl = "test_dynamic_partition_with_rename"
+ sql "drop table if exists ${tbl}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbl}
+ ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
+ AGGREGATE KEY(k1,k2)
+ PARTITION BY RANGE(k1) ( )
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "dynamic_partition.enable"="true",
+ "dynamic_partition.end"="3",
+ "dynamic_partition.buckets"="1",
+ "dynamic_partition.start"="-3",
+ "dynamic_partition.prefix"="p",
+ "dynamic_partition.time_unit"="DAY",
+ "dynamic_partition.create_history_partition"="true",
+ "dynamic_partition.replication_allocation" = "tag.location.default: 1")
+ """
+ result = sql "show partitions from ${tbl}"
+ assertEquals(7, result.size())
+
+ // rename distributed column, then try to add too more dynamic partition
+ sql "alter table ${tbl} rename column k1 renamed_k1"
+ sql """ ADMIN SET FRONTEND CONFIG ('dynamic_partition_check_interval_seconds' = '1') """
+ sql """ alter table ${tbl} set('dynamic_partition.end'='5') """
+ result = sql "show partitions from ${tbl}"
+ for (def retry = 0; retry < 15; retry++) {
+ if (result.size() == 9) {
+ break;
+ }
+ logger.info("wait dynamic partition scheduler, sleep 1s")
+ sleep(1000);
+ result = sql "show partitions from ${tbl}"
+ }
+ assertEquals(9, result.size())
+ for (def line = 0; line < result.size(); line++) {
+ // XXX: DistributionKey at pos(7), next maybe impl by sql meta
+ assertEquals("renamed_k1", result.get(line).get(7))
+ }
+
+ sql "drop table ${tbl}"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/10: [Fix] (tablet) fix tablet queryable set (#20413) (#20414)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3554187d16c9ac6c3741c35414c38e260a33863c
Author: yujun <yu...@gmail.com>
AuthorDate: Tue Jun 6 15:38:01 2023 +0800
[Fix] (tablet) fix tablet queryable set (#20413) (#20414)
---
fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java | 9 +++++++++
.../src/main/java/org/apache/doris/clone/TabletScheduler.java | 4 ++--
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
index fbe60b74c8..003097baa2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -216,6 +216,7 @@ public class Tablet extends MetaObject implements Writable {
// for query
public List<Replica> getQueryableReplicas(long visibleVersion) {
List<Replica> allQueryableReplica = Lists.newArrayListWithCapacity(replicas.size());
+ List<Replica> auxiliaryReplica = Lists.newArrayListWithCapacity(replicas.size());
for (Replica replica : replicas) {
if (replica.isBad()) {
continue;
@@ -231,9 +232,17 @@ public class Tablet extends MetaObject implements Writable {
if (replica.checkVersionCatchUp(visibleVersion, false)) {
allQueryableReplica.add(replica);
}
+ } else if (state == ReplicaState.DECOMMISSION) {
+ if (replica.checkVersionCatchUp(visibleVersion, false)) {
+ auxiliaryReplica.add(replica);
+ }
}
}
+ if (allQueryableReplica.isEmpty()) {
+ allQueryableReplica = auxiliaryReplica;
+ }
+
if (Config.skip_compaction_slower_replica && allQueryableReplica.size() > 1) {
long minVersionCount = Long.MAX_VALUE;
for (Replica replica : allQueryableReplica) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 8023fe2be5..30d6c23d8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -1123,8 +1123,8 @@ public class TabletScheduler extends MasterDaemon {
replica.setState(ReplicaState.DECOMMISSION);
// set priority to normal because it may wait for a long time. Remain it as VERY_HIGH may block other task.
tabletCtx.setOrigPriority(Priority.NORMAL);
- LOG.debug("set replica {} on backend {} of tablet {} state to DECOMMISSION",
- replica.getId(), replica.getBackendId(), tabletCtx.getTabletId());
+ LOG.info("set replica {} on backend {} of tablet {} state to DECOMMISSION due to reason {}",
+ replica.getId(), replica.getBackendId(), tabletCtx.getTabletId(), reason);
throw new SchedException(Status.SCHEDULE_FAILED, "set watermark txn " + nextTxnId);
} else if (replica.getState() == ReplicaState.DECOMMISSION && replica.getWatermarkTxnId() != -1) {
long watermarkTxnId = replica.getWatermarkTxnId();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org