You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/03/17 02:51:15 UTC
(doris) branch branch-4.1 updated (5993dbd9148 -> 1ce5fbac5c4)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from 5993dbd9148 branch-4.1: [chore](thirdparty) Integrate paimon-cpp into thirdparty build system #60296 (#61369)
new 13eccc4251b branch-4.0: [test](doc) Add regression test for ASOF JOIN documentation examples #61351 (#61353)
new 9ede6bcb384 branch-4.0: [enhance](load) exclude version-gap replicas from success counting in quorum success #60953 (#61359)
new c6d944981d0 branch-4.0: [fix](es-catalog) Fix query error when ES keyword field contains array data #61236 (#61361)
new 70c4f0d85a5 branch-4.0: [opt](cloud) Support fdb locality aware load balancing #61312 (#61375)
new 07daf38b8ca branch-4.0: [chore](fe) upgrade okhttp to 5.3.2 #61263 (#61370)
new 55b8e03682d branch-4.0: [fix](test) fix case salt_join make it disable join reorder #59035 (#61394)
new a3013183f1a branch-4.0: [Fix](mv) Fix ConcurrentModificationException in PartitionCompensator #61145 (#61402)
new 284ecc53114 [fix](filecache) add check and exception handle for empty block file (#59646) (#61203)
new 1ce5fbac5c4 branch-4.0: [enhance](auth) introduction of ldaps support via configuration property #60275 (#61406)
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/es/es_scroll_parser.cpp | 20 +-
be/src/exec/tablet_info.cpp | 8 +
be/src/exec/tablet_info.h | 2 +
be/src/io/cache/file_block.cpp | 17 +-
be/src/io/cache/file_block.h | 1 +
be/src/vec/sink/writer/vtablet_writer.cpp | 13 +-
be/src/vec/sink/writer/vtablet_writer.h | 4 +
be/src/vec/sink/writer/vtablet_writer_v2.cpp | 38 ++-
be/src/vec/sink/writer/vtablet_writer_v2.h | 3 +
be/test/io/cache/block_file_cache_test.cpp | 130 ++++++++
cloud/src/common/config.h | 5 +
cloud/src/meta-store/txn_kv.cpp | 57 +++-
conf/ldap.conf | 3 +
.../scripts/data/test_keyword_array.json | 3 +
.../elasticsearch/scripts/es_init.sh | 18 +-
.../scripts/index/es7_test_keyword_array.json | 28 ++
.../java/org/apache/doris/common/LdapConfig.java | 18 ++
fe/fe-core/pom.xml | 22 +-
.../java/org/apache/doris/catalog/OlapTable.java | 22 ++
.../doris/mysql/authenticate/ldap/LdapClient.java | 9 +-
.../rules/exploration/mv/PartitionCompensator.java | 19 +-
.../org/apache/doris/planner/OlapTableSink.java | 8 +
.../mysql/authenticate/ldap/LdapClientTest.java | 27 ++
.../exploration/mv/PartitionCompensatorTest.java | 123 +++++++
fe/pom.xml | 28 +-
gensrc/thrift/Descriptors.thrift | 3 +
.../data/doc/query-data/asof-join.md.out | 65 ++++
.../es/test_es_keyword_array_type.out | 33 ++
.../data/nereids_rules_p0/salt_join/salt_join.out | 300 ++++++++---------
.../suites/doc/query-data/asof-join.md.groovy | 355 +++++++++++++++++++++
.../es/test_es_keyword_array_type.groovy | 115 +++++++
.../nereids_rules_p0/salt_join/salt_join.groovy | 84 ++---
32 files changed, 1349 insertions(+), 232 deletions(-)
create mode 100644 docker/thirdparties/docker-compose/elasticsearch/scripts/data/test_keyword_array.json
create mode 100644 docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test_keyword_array.json
create mode 100644 regression-test/data/doc/query-data/asof-join.md.out
create mode 100644 regression-test/data/external_table_p0/es/test_es_keyword_array_type.out
create mode 100644 regression-test/suites/doc/query-data/asof-join.md.groovy
create mode 100644 regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/09: branch-4.0: [enhance](auth) introduction of ldaps support via configuration property #60275 (#61406)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1ce5fbac5c4f570d6f49c71ca793834e39aaf7b1
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 10:42:06 2026 +0800
branch-4.0: [enhance](auth) introduction of ldaps support via configuration property #60275 (#61406)
Cherry-picked from #60275
Co-authored-by: iaorekhov-1980 <i....@gmail.com>
---
conf/ldap.conf | 3 +++
.../java/org/apache/doris/common/LdapConfig.java | 18 +++++++++++++++
.../doris/mysql/authenticate/ldap/LdapClient.java | 9 ++++----
.../mysql/authenticate/ldap/LdapClientTest.java | 27 ++++++++++++++++++++++
4 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/conf/ldap.conf b/conf/ldap.conf
index deb1a06a1d0..9388ae7ee50 100644
--- a/conf/ldap.conf
+++ b/conf/ldap.conf
@@ -44,6 +44,9 @@ ldap_group_basedn = ou=group,dc=domain,dc=com
# ldap_user_cache_timeout_s = 5 * 60;
+## ldap_use_ssl - use secured connection to LDAP server if required (disabled by default). Note: When enabling SSL, ensure ldap_port is set appropriately (typically 636 for LDAPS instead of 389 for LDAP).
+# ldap_use_ssl = false
+
# LDAP pool configuration
# https://docs.spring.io/spring-ldap/docs/2.3.3.RELEASE/reference/#pool-configuration
# ldap_pool_max_active = 8
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java b/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java
index 9499fcc2a1b..881840696dc 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java
@@ -157,4 +157,22 @@ public class LdapConfig extends ConfigBase {
*/
@ConfigBase.ConfField
public static boolean ldap_pool_test_while_idle = true;
+
+ /**
+ * Flag to enable usage of LDAPS.
+ */
+ @ConfigBase.ConfField
+ public static boolean ldap_use_ssl = false;
+
+ /**
+ * The method constructs the correct URL connection string for the specified host and port depending on
+ * the value of the {@code ldap_use_ssl} property.
+ * If {@code ldap_use_ssl} is true, LDAPS is used as the protocol.
+ * If {@code ldap_use_ssl} is false or not specified, LDAP is used as the protocol.
+ * @param hostPortInAccessibleFormat the host and port in accessible format (for example, "host:port")
+ * @return the LDAP or LDAPS connection URL string
+ */
+ public static String getConnectionURL(String hostPortInAccessibleFormat) {
+ return ((LdapConfig.ldap_use_ssl ? "ldaps" : "ldap") + "://" + hostPortInAccessibleFormat);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java
index d5641ac6c09..79248ab0212 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java
@@ -65,8 +65,8 @@ public class LdapClient {
private void setLdapTemplateNoPool(String ldapPassword) {
LdapContextSource contextSource = new LdapContextSource();
- String url = "ldap://" + NetUtils
- .getHostPortInAccessibleFormat(LdapConfig.ldap_host, LdapConfig.ldap_port);
+ String url = LdapConfig.getConnectionURL(
+ NetUtils.getHostPortInAccessibleFormat(LdapConfig.ldap_host, LdapConfig.ldap_port));
contextSource.setUrl(url);
contextSource.setUserDn(LdapConfig.ldap_admin_name);
@@ -78,8 +78,8 @@ public class LdapClient {
private void setLdapTemplatePool(String ldapPassword) {
LdapContextSource contextSource = new LdapContextSource();
- String url = "ldap://" + NetUtils
- .getHostPortInAccessibleFormat(LdapConfig.ldap_host, LdapConfig.ldap_port);
+ String url = LdapConfig.getConnectionURL(
+ NetUtils.getHostPortInAccessibleFormat(LdapConfig.ldap_host, LdapConfig.ldap_port));
contextSource.setUrl(url);
contextSource.setUserDn(LdapConfig.ldap_admin_name);
@@ -108,6 +108,7 @@ public class LdapClient {
public boolean checkUpdate(String ldapPassword) {
return this.ldapPassword == null || !this.ldapPassword.equals(ldapPassword);
}
+
}
private void init() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/ldap/LdapClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/ldap/LdapClientTest.java
index 866a84e7528..c0d6c36f83b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/ldap/LdapClientTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/ldap/LdapClientTest.java
@@ -19,9 +19,11 @@ package org.apache.doris.mysql.authenticate.ldap;
import org.apache.doris.common.Config;
import org.apache.doris.common.LdapConfig;
+import org.apache.doris.common.util.NetUtils;
import mockit.Expectations;
import mockit.Tested;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -43,6 +45,7 @@ public class LdapClientTest {
LdapConfig.ldap_user_basedn = "dc=baidu,dc=com";
LdapConfig.ldap_group_basedn = "ou=group,dc=baidu,dc=com";
LdapConfig.ldap_user_filter = "(&(uid={login}))";
+ LdapConfig.ldap_use_ssl = false;
}
@Test
@@ -95,4 +98,28 @@ public class LdapClientTest {
};
Assert.assertEquals(1, ldapClient.getGroups("zhangsan").size());
}
+
+ @Test
+ public void testSecuredProtocolIsUsed() {
+ //testing default case with not specified property ldap_use_ssl or it is specified as false
+ String insecureUrl = LdapConfig.getConnectionURL(
+ NetUtils.getHostPortInAccessibleFormat(LdapConfig.ldap_host, LdapConfig.ldap_port));
+
+ Assert.assertNotNull("connection URL should not be null", insecureUrl);
+ Assert.assertTrue("with ldap_use_ssl = false or not specified URL should start with ldap, but received: " + insecureUrl,
+ insecureUrl.startsWith("ldap://"));
+
+ //testing new case with specified property ldap_use_ssl as true
+ LdapConfig.ldap_use_ssl = true;
+ String secureUrl = LdapConfig.getConnectionURL(
+ NetUtils.getHostPortInAccessibleFormat(LdapConfig.ldap_host, LdapConfig.ldap_port));
+ Assert.assertNotNull("connection URL should not be null", secureUrl);
+ Assert.assertTrue("with ldap_use_ssl = true URL should start with ldaps, but received: " + secureUrl,
+ secureUrl.startsWith("ldaps://"));
+ }
+
+ @After
+ public void tearDown() {
+ LdapConfig.ldap_use_ssl = false; // restoring default value for other tests
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/09: branch-4.0: [chore](fe) upgrade okhttp to 5.3.2 #61263 (#61370)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 07daf38b8ca9cae57087e7cbb791a986aa7c8a82
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:08:27 2026 +0800
branch-4.0: [chore](fe) upgrade okhttp to 5.3.2 #61263 (#61370)
Cherry-picked from #61263
Co-authored-by: Calvin Kirs <gu...@selectdb.com>
---
fe/fe-core/pom.xml | 22 +++++++++++++++++++++-
fe/pom.xml | 28 ++++++++++++++++++----------
2 files changed, 39 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 8c3bd0e6685..749343410c6 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -349,11 +349,31 @@ under the License.
<dependency>
<groupId>com.tencentcloudapi</groupId>
<artifactId>tencentcloud-sdk-java-sts</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.squareup.okio</groupId>
+ <artifactId>okio</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- huawei cloud sts -->
<dependency>
<groupId>com.huaweicloud.sdk</groupId>
<artifactId>huaweicloud-sdk-iam</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.squareup.okio</groupId>
+ <artifactId>okio</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- ICU4J for unicode normalization -->
<dependency>
@@ -605,7 +625,7 @@ under the License.
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
- <artifactId>okhttp</artifactId>
+ <artifactId>okhttp-jvm</artifactId>
</dependency>
<dependency>
diff --git a/fe/pom.xml b/fe/pom.xml
index 6c3cfe4773d..d85576e246a 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -305,9 +305,7 @@ under the License.
<generex.version>1.0.1</generex.version>
<jetty-servlet.version>11.0.26</jetty-servlet.version>
<jetty-websocket.version>9.4.58.v20250814</jetty-websocket.version>
- <logging-interceptor.version>4.12.0</logging-interceptor.version>
- <okhttp.version>4.9.3</okhttp.version>
- <okio.version>3.4.0</okio.version>
+ <okhttp.version>5.3.2</okhttp.version>
<snakeyaml.version>2.0</snakeyaml.version>
<validation-api.version>2.0.1.Final</validation-api.version>
<zjsonpatch.version>0.2.3</zjsonpatch.version>
@@ -1096,15 +1094,9 @@ under the License.
<!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
- <artifactId>okhttp</artifactId>
+ <artifactId>okhttp-jvm</artifactId>
<version>${okhttp.version}</version>
</dependency>
- <!-- https://mvnrepository.com/artifact/com.squareup.okio/okio -->
- <dependency>
- <groupId>com.squareup.okio</groupId>
- <artifactId>okio</artifactId>
- <version>${okio.version}</version>
- </dependency>
<!-- https://mvnrepository.com/artifact/org.yaml/snakeyaml -->
<dependency>
<groupId>org.yaml</groupId>
@@ -1338,6 +1330,14 @@ under the License.
<groupId>org.ini4j</groupId>
<artifactId>ini4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.squareup.okio</groupId>
+ <artifactId>okio</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<!-- For Iceberg, must be consistent with Iceberg version -->
@@ -1709,6 +1709,14 @@ under the License.
<artifactId>re2j</artifactId>
<groupId>io.trino</groupId>
</exclusion>
+ <exclusion>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.squareup.okio</groupId>
+ <artifactId>okio</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/09: branch-4.0: [test](doc) Add regression test for ASOF JOIN documentation examples #61351 (#61353)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 13eccc4251b79dfca04bae8e1f1021dc69f53df8
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:04:20 2026 +0800
branch-4.0: [test](doc) Add regression test for ASOF JOIN documentation examples #61351 (#61353)
Cherry-picked from #61351
Co-authored-by: zclllyybb <zh...@selectdb.com>
---
.../data/doc/query-data/asof-join.md.out | 65 ++++
.../suites/doc/query-data/asof-join.md.groovy | 355 +++++++++++++++++++++
2 files changed, 420 insertions(+)
diff --git a/regression-test/data/doc/query-data/asof-join.md.out b/regression-test/data/doc/query-data/asof-join.md.out
new file mode 100644
index 00000000000..92d04858d56
--- /dev/null
+++ b/regression-test/data/doc/query-data/asof-join.md.out
@@ -0,0 +1,65 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !example1 --
+1 AAPL 2024-01-01T10:00:05 150.50 1 2024-01-01T10:00 150.00 150.10
+2 AAPL 2024-01-01T10:00:15 151.00 2 2024-01-01T10:00:10 150.40 150.60
+3 AAPL 2024-01-01T10:00:25 150.75 3 2024-01-01T10:00:20 150.90 151.10
+4 GOOG 2024-01-01T10:00:10 2800.00 4 2024-01-01T10:00:05 2795.00 2800.00
+5 GOOG 2024-01-01T10:00:20 2805.00 5 2024-01-01T10:00:15 2802.00 2808.00
+6 MSFT 2024-01-01T10:00:08 380.00 6 2024-01-01T10:00 378.00 380.00
+
+-- !example2 --
+1 AAPL 2024-01-01T10:00:05 150.50 2 2024-01-01T10:00:10 150.40
+2 AAPL 2024-01-01T10:00:15 151.00 3 2024-01-01T10:00:20 150.90
+3 AAPL 2024-01-01T10:00:25 150.75 \N \N \N
+4 GOOG 2024-01-01T10:00:10 2800.00 5 2024-01-01T10:00:15 2802.00
+5 GOOG 2024-01-01T10:00:20 2805.00 \N \N \N
+6 MSFT 2024-01-01T10:00:08 380.00 7 2024-01-01T10:00:10 379.50
+
+-- !example3 --
+1 AAPL 2024-01-01T10:00:05 150.50 1 2024-01-01T10:00 150.00
+2 AAPL 2024-01-01T10:00:15 151.00 2 2024-01-01T10:00:10 150.40
+3 AAPL 2024-01-01T10:00:25 150.75 3 2024-01-01T10:00:20 150.90
+4 GOOG 2024-01-01T10:00:10 2800.00 4 2024-01-01T10:00:05 2795.00
+5 GOOG 2024-01-01T10:00:20 2805.00 5 2024-01-01T10:00:15 2802.00
+6 MSFT 2024-01-01T10:00:08 380.00 6 2024-01-01T10:00 378.00
+
+-- !example4 --
+1 101 US 2024-01-01T10:00:10 100.00 2024-01-01T10:00
+2 101 US 2024-01-01T10:00:30 102.00 2024-01-01T10:00:20
+3 102 EU 2024-01-01T10:00:15 200.00 2024-01-01T10:00
+4 102 EU 2024-01-01T10:00:45 205.00 2024-01-01T10:00:30
+
+-- !example5 --
+1 2024-01-01T12:00 1 2024-01-01T10:00 snapshot_1
+2 2024-01-01T14:00 3 2024-01-01T13:00 snapshot_3
+3 2024-01-01T13:00 4 2024-01-01T11:00 snapshot_4
+
+-- !example6 --
+1 2024-01-01T10:00:10 100.00 2024-01-01T10:00 500 2024-01-01T10:00
+2 2024-01-01T10:00:30 102.00 2024-01-01T10:00:20 480 2024-01-01T10:00:20
+3 2024-01-01T10:00:15 200.00 2024-01-01T10:00 300 2024-01-01T10:00
+4 2024-01-01T10:00:45 205.00 2024-01-01T10:00:30 290 2024-01-01T10:00:30
+
+-- !example6_mixed --
+1 Widget A 2024-01-01T10:00:10 100.00
+2 Widget A 2024-01-01T10:00:30 102.00
+3 Widget B 2024-01-01T10:00:15 200.00
+4 Widget B 2024-01-01T10:00:45 205.00
+
+-- !example7 --
+AAPL 3 150.4333
+GOOG 2 2798.5000
+MSFT 1 378.0000
+
+-- !example8 --
+1 2024-01-01T10:00:10 100.00 2024-01-01T10:00 102.00 2024-01-01T10:00:20
+2 2024-01-01T10:00:30 102.00 2024-01-01T10:00:20 \N \N
+3 2024-01-01T10:00:15 200.00 2024-01-01T10:00 205.00 2024-01-01T10:00:30
+4 2024-01-01T10:00:45 205.00 2024-01-01T10:00:30 \N \N
+
+-- !example9_ge --
+1 2024-01-01T10:00:06 1 2024-01-01T10:00
+
+-- !example9_le --
+1 2024-01-01T10:00:06 2 2024-01-01T10:00:08
+
diff --git a/regression-test/suites/doc/query-data/asof-join.md.groovy b/regression-test/suites/doc/query-data/asof-join.md.groovy
new file mode 100644
index 00000000000..bfabe281c13
--- /dev/null
+++ b/regression-test/suites/doc/query-data/asof-join.md.groovy
@@ -0,0 +1,355 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.jupiter.api.Assertions;
+
+suite("docs/query-data/asof-join.md") {
+ try {
+ // Preparation: Create trades and quotes tables
+ sql "DROP TABLE IF EXISTS trades"
+ sql """
+ CREATE TABLE trades (
+ trade_id INT,
+ symbol VARCHAR(10),
+ trade_time DATETIME,
+ price DECIMAL(10, 2),
+ quantity INT
+ ) DISTRIBUTED BY HASH(trade_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql "DROP TABLE IF EXISTS quotes"
+ sql """
+ CREATE TABLE quotes (
+ quote_id INT,
+ symbol VARCHAR(10),
+ quote_time DATETIME,
+ bid_price DECIMAL(10, 2),
+ ask_price DECIMAL(10, 2)
+ ) DISTRIBUTED BY HASH(quote_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO trades VALUES
+ (1, 'AAPL', '2024-01-01 10:00:05', 150.50, 100),
+ (2, 'AAPL', '2024-01-01 10:00:15', 151.00, 200),
+ (3, 'AAPL', '2024-01-01 10:00:25', 150.75, 150),
+ (4, 'GOOG', '2024-01-01 10:00:10', 2800.00, 50),
+ (5, 'GOOG', '2024-01-01 10:00:20', 2805.00, 75),
+ (6, 'MSFT', '2024-01-01 10:00:08', 380.00, 120)
+ """
+
+ sql """
+ INSERT INTO quotes VALUES
+ (1, 'AAPL', '2024-01-01 10:00:00', 150.00, 150.10),
+ (2, 'AAPL', '2024-01-01 10:00:10', 150.40, 150.60),
+ (3, 'AAPL', '2024-01-01 10:00:20', 150.90, 151.10),
+ (4, 'GOOG', '2024-01-01 10:00:05', 2795.00, 2800.00),
+ (5, 'GOOG', '2024-01-01 10:00:15', 2802.00, 2808.00),
+ (6, 'MSFT', '2024-01-01 10:00:00', 378.00, 380.00),
+ (7, 'MSFT', '2024-01-01 10:00:10', 379.50, 381.00)
+ """
+
+ // Preparation: Create orders and prices tables for examples 4, 6, 8
+ sql "DROP TABLE IF EXISTS orders"
+ sql """
+ CREATE TABLE orders (
+ order_id INT,
+ product_id INT,
+ region VARCHAR(20),
+ order_time DATETIME
+ ) DISTRIBUTED BY HASH(order_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql "DROP TABLE IF EXISTS prices"
+ sql """
+ CREATE TABLE prices (
+ price_id INT,
+ product_id INT,
+ region VARCHAR(20),
+ effective_time DATETIME,
+ price DECIMAL(10, 2)
+ ) DISTRIBUTED BY HASH(price_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO orders VALUES
+ (1, 101, 'US', '2024-01-01 10:00:10'),
+ (2, 101, 'US', '2024-01-01 10:00:30'),
+ (3, 102, 'EU', '2024-01-01 10:00:15'),
+ (4, 102, 'EU', '2024-01-01 10:00:45')
+ """
+
+ sql """
+ INSERT INTO prices VALUES
+ (1, 101, 'US', '2024-01-01 10:00:00', 100.00),
+ (2, 101, 'US', '2024-01-01 10:00:20', 102.00),
+ (3, 102, 'EU', '2024-01-01 10:00:00', 200.00),
+ (4, 102, 'EU', '2024-01-01 10:00:30', 205.00)
+ """
+
+ // Preparation: Create inventory and products tables for example 6
+ sql "DROP TABLE IF EXISTS inventory"
+ sql """
+ CREATE TABLE inventory (
+ inv_id INT,
+ product_id INT,
+ region VARCHAR(20),
+ snapshot_time DATETIME,
+ stock_level INT
+ ) DISTRIBUTED BY HASH(inv_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO inventory VALUES
+ (1, 101, 'US', '2024-01-01 10:00:00', 500),
+ (2, 101, 'US', '2024-01-01 10:00:20', 480),
+ (3, 102, 'EU', '2024-01-01 10:00:00', 300),
+ (4, 102, 'EU', '2024-01-01 10:00:30', 290)
+ """
+
+ sql "DROP TABLE IF EXISTS products"
+ sql """
+ CREATE TABLE products (
+ product_id INT,
+ product_name VARCHAR(50)
+ ) DISTRIBUTED BY HASH(product_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO products VALUES
+ (101, 'Widget A'),
+ (102, 'Widget B')
+ """
+
+ // Preparation: Create left_table and right_table for example 5
+ sql "DROP TABLE IF EXISTS left_table"
+ sql """
+ CREATE TABLE left_table (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ val VARCHAR(50)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql "DROP TABLE IF EXISTS right_table"
+ sql """
+ CREATE TABLE right_table (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(50)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO left_table VALUES
+ (1, 1, '2024-01-01 12:00:00', 'event_a'),
+ (2, 1, '2024-01-01 14:00:00', 'event_b'),
+ (3, 2, '2024-01-01 13:00:00', 'event_c')
+ """
+
+ sql """
+ INSERT INTO right_table VALUES
+ (1, 1, '2024-01-01 10:00:00', 'snapshot_1'),
+ (2, 1, '2024-01-01 12:00:00', 'snapshot_2'),
+ (3, 1, '2024-01-01 13:00:00', 'snapshot_3'),
+ (4, 2, '2024-01-01 11:00:00', 'snapshot_4')
+ """
+
+ // Example 1: Find the Most Recent Quote for Each Trade (>=)
+ order_qt_example1 """
+ SELECT t.trade_id, t.symbol, t.trade_time, t.price,
+ q.quote_id, q.quote_time, q.bid_price, q.ask_price
+ FROM trades t
+ ASOF LEFT JOIN quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // Example 2: Find the Next Quote After Each Trade (<=)
+ order_qt_example2 """
+ SELECT t.trade_id, t.symbol, t.trade_time, t.price,
+ q.quote_id, q.quote_time, q.bid_price
+ FROM trades t
+ ASOF LEFT JOIN quotes q
+ MATCH_CONDITION(t.trade_time <= q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // Example 3: ASOF INNER JOIN — Exclude Unmatched Rows
+ order_qt_example3 """
+ SELECT t.trade_id, t.symbol, t.trade_time, t.price,
+ q.quote_id, q.quote_time, q.bid_price
+ FROM trades t
+ ASOF INNER JOIN quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // Example 4: Multiple Equality Conditions
+ order_qt_example4 """
+ SELECT o.order_id, o.product_id, o.region, o.order_time,
+ p.price, p.effective_time
+ FROM orders o
+ ASOF LEFT JOIN prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ORDER BY o.order_id
+ """
+
+ // Example 5: Expression in MATCH_CONDITION (at least 1 hour gap)
+ order_qt_example5 """
+ SELECT l.id, l.ts, r.id AS rid, r.ts AS rts, r.data
+ FROM left_table l
+ ASOF LEFT JOIN right_table r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Example 6: Multi-level ASOF JOIN
+ order_qt_example6 """
+ SELECT o.order_id, o.order_time,
+ p.price, p.effective_time AS price_time,
+ i.stock_level, i.snapshot_time AS inv_time
+ FROM orders o
+ ASOF LEFT JOIN prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ASOF LEFT JOIN inventory i
+ MATCH_CONDITION(o.order_time >= i.snapshot_time)
+ ON o.product_id = i.product_id AND o.region = i.region
+ ORDER BY o.order_id
+ """
+
+ // Example 6 (continued): Mixing ASOF JOIN with regular JOIN
+ order_qt_example6_mixed """
+ SELECT o.order_id, prod.product_name,
+ o.order_time, p.price
+ FROM orders o
+ INNER JOIN products prod ON o.product_id = prod.product_id
+ ASOF LEFT JOIN prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ORDER BY o.order_id
+ """
+
+ // Example 7: ASOF JOIN with Aggregation
+ order_qt_example7 """
+ SELECT t.symbol,
+ COUNT(*) AS trade_count,
+ AVG(q.bid_price) AS avg_bid
+ FROM trades t
+ ASOF LEFT JOIN quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ GROUP BY t.symbol
+ ORDER BY t.symbol
+ """
+
+ // Example 8: Bidirectional ASOF JOIN — Finding Surrounding Records
+ order_qt_example8 """
+ SELECT o.order_id, o.order_time,
+ p_before.price AS price_before,
+ p_before.effective_time AS time_before,
+ p_after.price AS price_after,
+ p_after.effective_time AS time_after
+ FROM orders o
+ ASOF LEFT JOIN prices p_before
+ MATCH_CONDITION(o.order_time >= p_before.effective_time)
+ ON o.product_id = p_before.product_id AND o.region = p_before.region
+ ASOF LEFT JOIN prices p_after
+ MATCH_CONDITION(o.order_time <= p_after.effective_time)
+ ON o.product_id = p_after.product_id AND o.region = p_after.region
+ ORDER BY o.order_id
+ """
+
+ // Example 9: Directional Matching, Not Absolute Nearest (>=)
+ order_qt_example9_ge """
+ WITH left_events AS (
+ SELECT 1 AS event_id, 'AAPL' AS symbol, CAST('2024-01-01 10:00:06' AS DATETIME) AS event_time
+ ),
+ right_events AS (
+ SELECT 1 AS right_id, 'AAPL' AS symbol, CAST('2024-01-01 10:00:00' AS DATETIME) AS ref_time
+ UNION ALL
+ SELECT 2 AS right_id, 'AAPL' AS symbol, CAST('2024-01-01 10:00:08' AS DATETIME) AS ref_time
+ )
+ SELECT l.event_id, l.event_time, r.right_id, r.ref_time
+ FROM left_events l
+ ASOF LEFT JOIN right_events r
+ MATCH_CONDITION(l.event_time >= r.ref_time)
+ ON l.symbol = r.symbol
+ """
+
+ // Example 9: Directional Matching, Not Absolute Nearest (<=)
+ order_qt_example9_le """
+ WITH left_events AS (
+ SELECT 1 AS event_id, 'AAPL' AS symbol, CAST('2024-01-01 10:00:06' AS DATETIME) AS event_time
+ ),
+ right_events AS (
+ SELECT 1 AS right_id, 'AAPL' AS symbol, CAST('2024-01-01 10:00:00' AS DATETIME) AS ref_time
+ UNION ALL
+ SELECT 2 AS right_id, 'AAPL' AS symbol, CAST('2024-01-01 10:00:08' AS DATETIME) AS ref_time
+ )
+ SELECT l.event_id, l.event_time, r.right_id, r.ref_time
+ FROM left_events l
+ ASOF LEFT JOIN right_events r
+ MATCH_CONDITION(l.event_time <= r.ref_time)
+ ON l.symbol = r.symbol
+ """
+
+ // Example 10: Duplicate Match Values (TIMESTAMPTZ) — result is non-deterministic, only verify row count
+ // The doc states this may return either right_id=1 or right_id=2
+ // Cast TIMESTAMPTZ columns to STRING to avoid JDBC decoding issues
+ def result10 = sql """
+ WITH left_events AS (
+ SELECT 1 AS event_id, 'AAPL' AS symbol,
+ CAST('2024-01-01 10:00:05 +00:00' AS TIMESTAMPTZ) AS event_time
+ ),
+ right_events AS (
+ SELECT 1 AS right_id, 'AAPL' AS symbol,
+ CAST('2024-01-01 10:00:00 +00:00' AS TIMESTAMPTZ) AS ref_time, 'snapshot_a' AS tag
+ UNION ALL
+ SELECT 2 AS right_id, 'AAPL' AS symbol,
+ CAST('2024-01-01 10:00:00 +00:00' AS TIMESTAMPTZ) AS ref_time, 'snapshot_b' AS tag
+ )
+ SELECT l.event_id, r.right_id, CAST(r.ref_time AS STRING), r.tag
+ FROM left_events l
+ ASOF LEFT JOIN right_events r
+ MATCH_CONDITION(l.event_time >= r.ref_time)
+ ON l.symbol = r.symbol
+ """
+ // Non-deterministic: one row returned, event_id = 1, right_id is either 1 or 2
+ assert result10.size() == 1
+ assert result10[0][0] == 1
+
+ } catch (Throwable t) {
+ Assertions.fail("examples in docs/query-data/asof-join.md failed to exec, please fix it", t)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/09: [fix](filecache) add check and exception handle for empty block file (#59646) (#61203)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 284ecc531142475233f993c6fcde5b5b2a1be794
Author: zhengyu <zh...@selectdb.com>
AuthorDate: Tue Mar 17 09:17:43 2026 +0800
[fix](filecache) add check and exception handle for empty block file (#59646) (#61203)
current code didn't check and handle cases when block file is fail to
download (empty file). add check and handler for that.
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/io/cache/file_block.cpp | 17 +++-
be/src/io/cache/file_block.h | 1 +
be/test/io/cache/block_file_cache_test.cpp | 130 +++++++++++++++++++++++++++++
3 files changed, 146 insertions(+), 2 deletions(-)
diff --git a/be/src/io/cache/file_block.cpp b/be/src/io/cache/file_block.cpp
index 731c1d311e6..83e80d872c2 100644
--- a/be/src/io/cache/file_block.cpp
+++ b/be/src/io/cache/file_block.cpp
@@ -113,7 +113,12 @@ void FileBlock::reset_downloader_impl(std::lock_guard<std::mutex>& block_lock) {
Status FileBlock::set_downloaded(std::lock_guard<std::mutex>& /* block_lock */) {
DCHECK(_download_state != State::DOWNLOADED);
- DCHECK_NE(_downloaded_size, 0);
+ if (_downloaded_size == 0) {
+ _download_state = State::EMPTY;
+ _downloader_id = 0;
+ return Status::InternalError("Try to set empty block {} as downloaded",
+ _block_range.to_string());
+ }
Status status = _mgr->_storage->finalize(_key);
if (status.ok()) [[likely]] {
_download_state = State::DOWNLOADED;
@@ -147,7 +152,15 @@ Status FileBlock::append(Slice data) {
}
Status FileBlock::finalize() {
- if (_downloaded_size != 0 && _downloaded_size != _block_range.size()) {
+ if (_downloaded_size == 0) {
+ std::lock_guard block_lock(_mutex);
+ _download_state = State::EMPTY;
+ _downloader_id = 0;
+ _cv.notify_all();
+ return Status::InternalError("Try to finalize an empty file block {}",
+ _block_range.to_string());
+ }
+ if (_downloaded_size != _block_range.size()) {
SCOPED_CACHE_LOCK(_mgr->_mutex, _mgr);
size_t old_size = _block_range.size();
_block_range.right = _block_range.left + _downloaded_size - 1;
diff --git a/be/src/io/cache/file_block.h b/be/src/io/cache/file_block.h
index 9cc4f06c6f0..eab74e827a8 100644
--- a/be/src/io/cache/file_block.h
+++ b/be/src/io/cache/file_block.h
@@ -44,6 +44,7 @@ class FileBlock {
friend class BlockFileCache;
friend class CachedRemoteFileReader;
friend struct FileBlockCell;
+ friend class FileBlockTestAccessor;
public:
enum class State {
diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp
index 333619a0bf8..287993e4283 100644
--- a/be/test/io/cache/block_file_cache_test.cpp
+++ b/be/test/io/cache/block_file_cache_test.cpp
@@ -21,6 +21,22 @@
#include "block_file_cache_test_common.h"
namespace doris::io {
+class FileBlockTestAccessor {
+public:
+ static void set_state(FileBlock& block, FileBlock::State state) {
+ block._download_state = state;
+ }
+ static void set_downloader_id(FileBlock& block, uint64_t id) { block._downloader_id = id; }
+ static void set_downloaded_size(FileBlock& block, size_t size) {
+ block._downloaded_size = size;
+ }
+
+ static Status call_set_downloaded(FileBlock& block) {
+ std::lock_guard<std::mutex> lock(block._mutex);
+ return block.set_downloaded(lock);
+ }
+};
+
fs::path caches_dir = fs::current_path() / "lru_cache_test";
std::string cache_base_path = caches_dir / "cache1" / "";
std::string tmp_file = caches_dir / "tmp_file";
@@ -8318,4 +8334,118 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader_direct_read_bytes_check) {
FileCacheFactory::instance()->_capacity = 0;
}
+TEST_F(BlockFileCacheTest, finalize_empty_block) {
+ std::string my_cache_path = caches_dir / "empty_block_test" / "";
+ if (fs::exists(my_cache_path)) {
+ fs::remove_all(my_cache_path);
+ }
+ io::FileCacheSettings settings;
+ settings.capacity = 100;
+ settings.max_file_block_size = 100;
+ io::BlockFileCache mgr(my_cache_path, settings);
+ ASSERT_TRUE(mgr.initialize().ok());
+
+ for (int i = 0; i < 100; i++) {
+ if (mgr.get_async_open_success()) {
+ break;
+ };
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
+
+ auto key = io::BlockFileCache::hash("empty_block_test");
+ io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ context.cache_type = io::FileCacheType::NORMAL;
+
+ {
+ auto holder = mgr.get_or_set(key, 0, 10, context);
+ auto blocks = fromHolder(holder);
+ ASSERT_EQ(blocks.size(), 1);
+ auto block = blocks[0];
+ ASSERT_EQ(block->state(), io::FileBlock::State::EMPTY);
+
+ ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id());
+ ASSERT_EQ(block->state(), io::FileBlock::State::DOWNLOADING);
+
+ // Call finalize without calling append()
+ Status st = block->finalize();
+ ASSERT_FALSE(st.ok());
+ ASSERT_EQ(block->state(), io::FileBlock::State::EMPTY);
+ ASSERT_EQ(block->get_downloader(), 0);
+ }
+ if (fs::exists(my_cache_path)) {
+ fs::remove_all(my_cache_path);
+ }
+}
+
+TEST_F(BlockFileCacheTest, finalize_partial_block) {
+ std::string my_cache_path = caches_dir / "partial_block_test" / "";
+ if (fs::exists(my_cache_path)) {
+ fs::remove_all(my_cache_path);
+ }
+ io::FileCacheSettings settings;
+ settings.capacity = 100;
+ settings.max_file_block_size = 100;
+ io::BlockFileCache mgr(my_cache_path, settings);
+ ASSERT_TRUE(mgr.initialize().ok());
+
+ for (int i = 0; i < 100; i++) {
+ if (mgr.get_async_open_success()) {
+ break;
+ };
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
+
+ auto key = io::BlockFileCache::hash("partial_block_test");
+ io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ context.cache_type = io::FileCacheType::NORMAL;
+
+ {
+ auto holder = mgr.get_or_set(key, 0, 10, context);
+ auto blocks = fromHolder(holder);
+ ASSERT_EQ(blocks.size(), 1);
+ auto block = blocks[0];
+ ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id());
+
+ std::string data(5, '0');
+ ASSERT_TRUE(block->append(Slice(data.data(), data.size())).ok());
+
+ // Finalize a block that only has 5 bytes out of 10
+ Status st = block->finalize();
+ ASSERT_TRUE(st.ok());
+ ASSERT_EQ(block->state(), io::FileBlock::State::DOWNLOADED);
+ ASSERT_EQ(block->range().size(), 5);
+ ASSERT_EQ(block->range().right, 4);
+ }
+
+ // Verify it was shrunk in the cache
+ ASSERT_EQ(mgr.get_used_cache_size(io::FileCacheType::NORMAL), 5);
+
+ if (fs::exists(my_cache_path)) {
+ fs::remove_all(my_cache_path);
+ }
+}
+
+TEST_F(BlockFileCacheTest, set_downloaded_empty_block_branch) {
+ FileCacheKey key;
+ key.hash = io::BlockFileCache::hash("set_downloaded_empty_block_branch");
+ key.offset = 0;
+ key.meta.type = io::FileCacheType::NORMAL;
+ key.meta.expiration_time = 0;
+
+ // mgr is intentionally nullptr: this branch returns before touching storage.
+ io::FileBlock block(key, 10, nullptr, io::FileBlock::State::EMPTY);
+ FileBlockTestAccessor::set_state(block, io::FileBlock::State::DOWNLOADING);
+ FileBlockTestAccessor::set_downloader_id(block, 123);
+ FileBlockTestAccessor::set_downloaded_size(block, 0);
+
+ Status st = FileBlockTestAccessor::call_set_downloaded(block);
+ ASSERT_FALSE(st.ok());
+ ASSERT_EQ(block.state(), io::FileBlock::State::EMPTY);
+ ASSERT_EQ(block.get_downloader(), 0);
+}
+
} // namespace doris::io
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/09: branch-4.0: [Fix](mv) Fix ConcurrentModificationException in PartitionCompensator #61145 (#61402)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a3013183f1a4a853b2ef8a601bc265a5fddc7904
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:10:46 2026 +0800
branch-4.0: [Fix](mv) Fix ConcurrentModificationException in PartitionCompensator #61145 (#61402)
Cherry-picked from #61145
Co-authored-by: Sahil Devgon <37...@users.noreply.github.com>
Co-authored-by: sahil-devgon <s_...@apple.com>
---
.../rules/exploration/mv/PartitionCompensator.java | 19 ++--
.../exploration/mv/PartitionCompensatorTest.java | 123 +++++++++++++++++++++
2 files changed, 133 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
index 3fe966864d0..4c26ab47cec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
@@ -139,19 +139,20 @@ public class PartitionCompensator {
.computeIfAbsent(baseTableNeedUnionTable.key(), k -> new HashSet<>())
.addAll(baseTableNeedUnionTable.value());
}
- // merge all partition to delete or union
- Set<String> needRemovePartitionSet = new HashSet<>();
- mvPartitionNeedRemoveNameMap.values().forEach(needRemovePartitionSet::addAll);
- mvPartitionNeedRemoveNameMap.replaceAll((k, v) -> needRemovePartitionSet);
-
- // consider multi base table partition name not same, how to handle it?
- Set<String> needUnionPartitionSet = new HashSet<>();
- baseTablePartitionNeedUnionNameMap.values().forEach(needUnionPartitionSet::addAll);
- baseTablePartitionNeedUnionNameMap.replaceAll((k, v) -> needUnionPartitionSet);
}
if (allCompensateIsNull) {
return null;
}
+ // merge all partition to delete or union
+ Set<String> needRemovePartitionSet = new HashSet<>();
+ mvPartitionNeedRemoveNameMap.values().forEach(needRemovePartitionSet::addAll);
+ mvPartitionNeedRemoveNameMap.replaceAll((k, v) -> needRemovePartitionSet);
+
+ // consider multi base table partition name not same, how to handle it?
+ Set<String> needUnionPartitionSet = new HashSet<>();
+ baseTablePartitionNeedUnionNameMap.values().forEach(needUnionPartitionSet::addAll);
+ baseTablePartitionNeedUnionNameMap.replaceAll((k, v) -> needUnionPartitionSet);
+
return Pair.of(mvPartitionNeedRemoveNameMap, baseTablePartitionNeedUnionNameMap);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
index 17d75f93fcf..3c1b2bb9519 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.TableIf;
@@ -30,7 +31,9 @@ import org.apache.doris.mtmv.BaseColInfo;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVPartitionInfo;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
+import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
@@ -41,10 +44,13 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -373,6 +379,123 @@ public class PartitionCompensatorTest extends TestWithFeService {
return ctx;
}
+ // Regression test for ConcurrentModificationException when merging partition maps
+ // across multiple related tables. The bug was caused by calling replaceAll() inside
+ // the for-loop after forEach() had already replaced map values with a shared Set
+ // reference, causing a self-modification on the second iteration.
+ // This test calls calcInvalidPartitions() directly with two related tables so that
+ // the exact code path containing the bug is exercised end-to-end.
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTables()
+ throws Exception {
+ // Shared catalog/db for both related base tables
+ CatalogIf<?> baseCatalog = Mockito.mock(CatalogIf.class);
+ Mockito.when(baseCatalog.getName()).thenReturn("cat");
+ Mockito.when(baseCatalog.getId()).thenReturn(1L);
+ DatabaseIf<?> baseDb = Mockito.mock(DatabaseIf.class);
+ Mockito.when(baseDb.getFullName()).thenReturn("db");
+ Mockito.when(baseDb.getId()).thenReturn(2L);
+ Mockito.when(baseDb.getCatalog()).thenReturn(baseCatalog);
+
+ MTMVRelatedTableIf relatedTable1 = mockRelatedTableIf(
+ "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb);
+ MTMVRelatedTableIf relatedTable2 = mockRelatedTableIf(
+ "t2", 20L, ImmutableList.of("cat", "db", "t2"), baseDb);
+
+ // Two MV valid partitions: mv_p1 maps to t1_p1, mv_p2 maps to t2_p2
+ Partition mvP1 = Mockito.mock(Partition.class);
+ Mockito.when(mvP1.getId()).thenReturn(101L);
+ Mockito.when(mvP1.getName()).thenReturn("mv_p1");
+ Partition mvP2 = Mockito.mock(Partition.class);
+ Mockito.when(mvP2.getId()).thenReturn(102L);
+ Mockito.when(mvP2.getName()).thenReturn("mv_p2");
+
+ Map<String, Set<String>> mappingForTable1 = new HashMap<>();
+ mappingForTable1.put("mv_p1", ImmutableSet.of("t1_p1"));
+ Map<String, Set<String>> mappingForTable2 = new HashMap<>();
+ mappingForTable2.put("mv_p2", ImmutableSet.of("t2_p2"));
+ Map<MTMVRelatedTableIf, Map<String, Set<String>>> partitionMappings = new HashMap<>();
+ partitionMappings.put(relatedTable1, mappingForTable1);
+ partitionMappings.put(relatedTable2, mappingForTable2);
+
+ BaseColInfo colInfo1 = new BaseColInfo("date_col", new BaseTableInfo(relatedTable1));
+ BaseColInfo colInfo2 = new BaseColInfo("date_col", new BaseTableInfo(relatedTable2));
+
+ // Separate catalog/db for the MV itself
+ CatalogIf<?> mvCatalog = Mockito.mock(CatalogIf.class);
+ Mockito.when(mvCatalog.getName()).thenReturn("internal");
+ Mockito.when(mvCatalog.getId()).thenReturn(1L);
+ DatabaseIf<?> mvDb = Mockito.mock(DatabaseIf.class);
+ Mockito.when(mvDb.getFullName()).thenReturn("mv_db");
+ Mockito.when(mvDb.getId()).thenReturn(3L);
+ Mockito.when(mvDb.getCatalog()).thenReturn(mvCatalog);
+
+ MTMV mtmv = Mockito.mock(MTMV.class);
+ Mockito.when(mtmv.getName()).thenReturn("mv1");
+ Mockito.when(mtmv.getId()).thenReturn(100L);
+ Mockito.when(mtmv.getDatabase()).thenReturn(mvDb);
+ PartitionInfo mvPartitionInfo = Mockito.mock(PartitionInfo.class);
+ Mockito.when(mtmv.getPartitionInfo()).thenReturn(mvPartitionInfo);
+ Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE);
+ MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class);
+ Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo);
+ Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable1, relatedTable2));
+ Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo1, colInfo2));
+ // All MV partitions contain data
+ Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any())).thenReturn(ImmutableList.of(1L));
+
+ AsyncMaterializationContext matCtx = Mockito.mock(AsyncMaterializationContext.class);
+ Mockito.when(matCtx.getMtmv()).thenReturn(mtmv);
+ Mockito.when(matCtx.calculatePartitionMappings()).thenReturn(partitionMappings);
+
+ // StatementContext: the MV's two valid partitions are available for rewrite
+ Map<BaseTableInfo, Collection<Partition>> canRewriteMap = new HashMap<>();
+ canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(mvP1, mvP2));
+ StatementContext stmtCtx = Mockito.mock(StatementContext.class);
+ Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap);
+
+ CascadesContext cascadesCtx = Mockito.mock(CascadesContext.class);
+ Mockito.when(cascadesCtx.getStatementContext()).thenReturn(stmtCtx);
+
+ // Rewritten plan has no MV scans, so mvNeedRemovePartitionNameSet stays empty
+ Plan rewrittenPlan = Mockito.mock(Plan.class);
+ Mockito.when(rewrittenPlan.collectToList(ArgumentMatchers.any())).thenReturn(Collections.emptyList());
+
+ // Each table contributes one covered and one uncovered partition:
+ // t1 uses {t1_p1 (covered by mv_p1), t1_p2 (not covered)}
+ // t2 uses {t2_p1 (not covered), t2_p2 (covered by mv_p2)}
+ Map<List<String>, Set<String>> queryUsedPartitions = new HashMap<>();
+ queryUsedPartitions.put(ImmutableList.of("cat", "db", "t1"),
+ ImmutableSet.of("t1_p1", "t1_p2"));
+ queryUsedPartitions.put(ImmutableList.of("cat", "db", "t2"),
+ ImmutableSet.of("t2_p1", "t2_p2"));
+
+ // Must not throw ConcurrentModificationException when two related tables each
+ // contribute entries that require the post-loop merge in calcInvalidPartitions()
+ Pair<Map<BaseTableInfo, Set<String>>, Map<BaseColInfo, Set<String>>> result =
+ Assertions.assertDoesNotThrow(() ->
+ PartitionCompensator.calcInvalidPartitions(
+ queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx));
+
+ // The uncovered partitions from both tables should be merged into one unified set
+ Assertions.assertNotNull(result);
+ Set<String> expectedUnion = ImmutableSet.of("t1_p2", "t2_p1");
+ result.value().values()
+ .forEach(v -> Assertions.assertEquals(expectedUnion, v));
+ }
+
+ @SuppressWarnings("unchecked")
+ private static MTMVRelatedTableIf mockRelatedTableIf(
+ String tableName, long tableId, List<String> qualifiers, DatabaseIf<?> db) {
+ MTMVRelatedTableIf table = Mockito.mock(MTMVRelatedTableIf.class);
+ Mockito.when(table.getName()).thenReturn(tableName);
+ Mockito.when(table.getId()).thenReturn(tableId);
+ Mockito.when(table.getDatabase()).thenReturn(db);
+ Mockito.when(table.getFullQualifiers()).thenReturn(qualifiers);
+ return table;
+ }
+
private static BaseTableInfo newBaseTableInfo() {
CatalogIf<?> catalog = Mockito.mock(CatalogIf.class);
Mockito.when(catalog.getId()).thenReturn(1L);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/09: branch-4.0: [enhance](load) exclude version-gap replicas from success counting in quorum success #60953 (#61359)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9ede6bcb384c3d5be1fdab1087206b9a97222f84
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:05:30 2026 +0800
branch-4.0: [enhance](load) exclude version-gap replicas from success counting in quorum success #60953 (#61359)
Cherry-picked from #60953
Co-authored-by: hui lai <la...@selectdb.com>
---
be/src/exec/tablet_info.cpp | 8 +++++
be/src/exec/tablet_info.h | 2 ++
be/src/vec/sink/writer/vtablet_writer.cpp | 13 +++++++-
be/src/vec/sink/writer/vtablet_writer.h | 4 +++
be/src/vec/sink/writer/vtablet_writer_v2.cpp | 38 ++++++++++++++++------
be/src/vec/sink/writer/vtablet_writer_v2.h | 3 ++
.../java/org/apache/doris/catalog/OlapTable.java | 22 +++++++++++++
.../org/apache/doris/planner/OlapTableSink.java | 8 +++++
gensrc/thrift/Descriptors.thrift | 3 ++
9 files changed, 90 insertions(+), 11 deletions(-)
diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
index 85b5b871382..7f61e1ac5ec 100644
--- a/be/src/exec/tablet_info.cpp
+++ b/be/src/exec/tablet_info.cpp
@@ -747,6 +747,14 @@ Status VOlapTablePartitionParam::generate_partition_from(const TOlapTablePartiti
if (t_part.__isset.load_required_replica_num) {
part_result->load_required_replica_num = t_part.load_required_replica_num;
}
+ if (t_part.__isset.tablet_version_gap_backends) {
+ for (const auto& [tablet_id, backend_ids] : t_part.tablet_version_gap_backends) {
+ auto& gap_set = part_result->tablet_version_gap_backends[tablet_id];
+ for (auto backend_id : backend_ids) {
+ gap_set.insert(backend_id);
+ }
+ }
+ }
return Status::OK();
}
diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h
index 1d13abec44a..91885c3ab5e 100644
--- a/be/src/exec/tablet_info.h
+++ b/be/src/exec/tablet_info.h
@@ -167,6 +167,8 @@ struct VOlapTablePartition {
int64_t load_tablet_idx = -1;
int total_replica_num = 0;
int load_required_replica_num = 0;
+ // tablet_id -> set of backend_ids that have version gaps
+ std::unordered_map<int64_t, std::unordered_set<int64_t>> tablet_version_gap_backends;
VOlapTablePartition(vectorized::Block* partition_block)
// the default value of partition bound is -1.
diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index 270b710634c..c8cb3496d2c 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -425,7 +425,13 @@ bool IndexChannel::_quorum_success(const std::unordered_set<int64_t>& unfinished
continue;
}
for (const auto& tablet_id : _tablets_by_channel[node_id]) {
- finished_tablets_replica[tablet_id]++;
+ // Only count non-gap backends for quorum success.
+ // Gap backends' success doesn't count toward majority write.
+ auto gap_it = _parent->_tablet_version_gap_backends.find(tablet_id);
+ if (gap_it == _parent->_tablet_version_gap_backends.end() ||
+ gap_it->second.find(node_id) == gap_it->second.end()) {
+ finished_tablets_replica[tablet_id]++;
+ }
}
}
@@ -1717,6 +1723,11 @@ void VTabletWriter::_build_tablet_replica_info(const int64_t tablet_id,
: partition->load_required_replica_num;
_tablet_replica_info.emplace(
tablet_id, std::make_pair(total_replicas_num, load_required_replicas_num));
+ // Copy version gap backends info for this tablet
+ if (auto it = partition->tablet_version_gap_backends.find(tablet_id);
+ it != partition->tablet_version_gap_backends.end()) {
+ _tablet_version_gap_backends[tablet_id] = it->second;
+ }
} else {
_tablet_replica_info.emplace(tablet_id,
std::make_pair(_num_replicas, (_num_replicas + 1) / 2));
diff --git a/be/src/vec/sink/writer/vtablet_writer.h b/be/src/vec/sink/writer/vtablet_writer.h
index 039dd10a2c7..6249febf8ad 100644
--- a/be/src/vec/sink/writer/vtablet_writer.h
+++ b/be/src/vec/sink/writer/vtablet_writer.h
@@ -775,5 +775,9 @@ private:
// tablet_id -> <total replicas num, load required replicas num>
std::unordered_map<int64_t, std::pair<int, int>> _tablet_replica_info;
+
+ // tablet_id -> set of backend_ids that have version gaps
+ // these backends' success should not be counted for majority write
+ std::unordered_map<int64_t, std::unordered_set<int64_t>> _tablet_version_gap_backends;
};
} // namespace doris::vectorized
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 2017099a4ef..794cd99a30e 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -371,6 +371,11 @@ void VTabletWriterV2::_build_tablet_replica_info(const int64_t tablet_id,
: partition->load_required_replica_num;
_tablet_replica_info[tablet_id] =
std::make_pair(total_replicas_num, load_required_replicas_num);
+ // Copy version gap backends info for this tablet
+ if (auto it = partition->tablet_version_gap_backends.find(tablet_id);
+ it != partition->tablet_version_gap_backends.end()) {
+ _tablet_version_gap_backends[tablet_id] = it->second;
+ }
} else {
_tablet_replica_info[tablet_id] = std::make_pair(_num_replicas, (_num_replicas + 1) / 2);
}
@@ -876,7 +881,13 @@ bool VTabletWriterV2::_quorum_success(
continue;
}
for (const auto& tablet_id : _tablets_by_node[dst_id]) {
- finished_tablets_replica[tablet_id]++;
+ // Only count non-gap backends for quorum success.
+ // Gap backends' success doesn't count toward majority write.
+ auto gap_it = _tablet_version_gap_backends.find(tablet_id);
+ if (gap_it == _tablet_version_gap_backends.end() ||
+ gap_it->second.find(dst_id) == gap_it->second.end()) {
+ finished_tablets_replica[tablet_id]++;
+ }
}
}
@@ -1015,13 +1026,15 @@ void VTabletWriterV2::_calc_tablets_to_commit() {
Status VTabletWriterV2::_create_commit_info(std::vector<TTabletCommitInfo>& tablet_commit_infos,
std::shared_ptr<LoadStreamMap> load_stream_map) {
- std::unordered_map<int64_t, int> failed_tablets;
+ // Track per-tablet non-gap success count and failure reasons
+ std::unordered_map<int64_t, int> success_tablets_replica;
+ std::unordered_set<int64_t> failed_tablets;
std::unordered_map<int64_t, Status> failed_reason;
load_stream_map->for_each([&](int64_t dst_id, LoadStreamStubs& streams) {
size_t num_success_tablets = 0;
size_t num_failed_tablets = 0;
for (auto [tablet_id, reason] : streams.failed_tablets()) {
- failed_tablets[tablet_id]++;
+ failed_tablets.insert(tablet_id);
failed_reason[tablet_id] = reason;
num_failed_tablets++;
}
@@ -1030,20 +1043,25 @@ Status VTabletWriterV2::_create_commit_info(std::vector<TTabletCommitInfo>& tabl
commit_info.tabletId = tablet_id;
commit_info.backendId = dst_id;
tablet_commit_infos.emplace_back(std::move(commit_info));
+ // Only count non-gap backends toward success
+ auto gap_it = _tablet_version_gap_backends.find(tablet_id);
+ if (gap_it == _tablet_version_gap_backends.end() ||
+ gap_it->second.find(dst_id) == gap_it->second.end()) {
+ success_tablets_replica[tablet_id]++;
+ }
num_success_tablets++;
}
LOG(INFO) << "streams to dst_id: " << dst_id << ", success tablets: " << num_success_tablets
<< ", failed tablets: " << num_failed_tablets;
});
- for (auto [tablet_id, replicas] : failed_tablets) {
- auto [total_replicas_num, load_required_replicas_num] = _tablet_replica_info[tablet_id];
- int max_failed_replicas = total_replicas_num == 0
- ? (_num_replicas - 1) / 2
- : total_replicas_num - load_required_replicas_num;
- if (replicas > max_failed_replicas) {
+ for (auto tablet_id : failed_tablets) {
+ int succ_count = success_tablets_replica[tablet_id];
+ int required = _load_required_replicas_num(tablet_id);
+ if (succ_count < required) {
LOG(INFO) << "tablet " << tablet_id
- << " failed on majority backends: " << failed_reason[tablet_id];
+ << " failed on majority backends (success=" << succ_count
+ << ", required=" << required << "): " << failed_reason[tablet_id];
return Status::InternalError("tablet {} failed on majority backends: {}", tablet_id,
failed_reason[tablet_id]);
}
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.h b/be/src/vec/sink/writer/vtablet_writer_v2.h
index e92084ebcd5..550501ad92e 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.h
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.h
@@ -258,6 +258,9 @@ private:
// tablet_id -> <total replicas num, load required replicas num>
std::unordered_map<int64_t, std::pair<int, int>> _tablet_replica_info;
+ // tablet_id -> set of backend_ids that have version gaps
+ std::unordered_map<int64_t, std::unordered_set<int64_t>> _tablet_version_gap_backends;
+
std::atomic<int64_t> _load_back_pressure_version_block_ms {0};
};
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 3c9da127b40..9301b908905 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2525,6 +2525,28 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
return partitionInfo.getReplicaAllocation(partitionId).getTotalReplicaNum();
}
+ public Map<Long, List<Long>> getPartitionVersionGapBackends(long partitionId) {
+ Map<Long, List<Long>> result = new HashMap<>();
+ Partition partition = getPartition(partitionId);
+ if (partition == null) {
+ return result;
+ }
+ for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) {
+ for (Tablet tablet : index.getTablets()) {
+ List<Long> gapBackends = new ArrayList<>();
+ for (Replica replica : tablet.getReplicas()) {
+ if (replica.getLastFailedVersion() >= 0) {
+ gapBackends.add(replica.getBackendIdWithoutException());
+ }
+ }
+ if (!gapBackends.isEmpty()) {
+ result.put(tablet.getId(), gapBackends);
+ }
+ }
+ }
+ return result;
+ }
+
public int getLoadRequiredReplicaNum(long partitionId) {
int totalReplicaNum = getPartitionTotalReplicasNum(partitionId);
int minLoadReplicaNum = getMinLoadReplicaNum();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
index 72e244d3f1f..8520249dd97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
@@ -221,6 +221,10 @@ public class OlapTableSink extends DataSink {
for (TOlapTablePartition partition : tOlapTablePartitionParam.getPartitions()) {
partition.setTotalReplicaNum(dstTable.getPartitionTotalReplicasNum(partition.getId()));
partition.setLoadRequiredReplicaNum(dstTable.getLoadRequiredReplicaNum(partition.getId()));
+ Map<Long, List<Long>> gapBackends = dstTable.getPartitionVersionGapBackends(partition.getId());
+ if (!gapBackends.isEmpty()) {
+ partition.setTabletVersionGapBackends(gapBackends);
+ }
}
tOlapTableLocationParams = createLocation(tSink.getDbId(), dstTable);
@@ -275,6 +279,10 @@ public class OlapTableSink extends DataSink {
for (TOlapTablePartition partition : tOlapTablePartitionParam.getPartitions()) {
partition.setTotalReplicaNum(dstTable.getPartitionTotalReplicasNum(partition.getId()));
partition.setLoadRequiredReplicaNum(dstTable.getLoadRequiredReplicaNum(partition.getId()));
+ Map<Long, List<Long>> gapBackends = dstTable.getPartitionVersionGapBackends(partition.getId());
+ if (!gapBackends.isEmpty()) {
+ partition.setTabletVersionGapBackends(gapBackends);
+ }
}
tOlapTableLocationParams = createLocation(tSink.getDbId(), dstTable);
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index 3be28d58089..51f4fcaf998 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -274,6 +274,9 @@ struct TOlapTablePartition {
11: optional i64 load_tablet_idx
12: optional i32 total_replica_num
13: optional i32 load_required_replica_num
+ // tablet_id -> list of backend_ids that have version gaps (lastFailedVersion >= 0)
+ // used by BE to exclude these backends from success counting in majority write
+ 14: optional map<i64, list<i64>> tablet_version_gap_backends
}
struct TOlapTablePartitionParam {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/09: branch-4.0: [fix](test) fix case salt_join make it disable join reorder #59035 (#61394)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 55b8e03682d9e3591b29bc4c82530d6712f79978
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:09:57 2026 +0800
branch-4.0: [fix](test) fix case salt_join make it disable join reorder #59035 (#61394)
Cherry-picked from #59035
Co-authored-by: feiniaofeiafei <mo...@selectdb.com>
---
.../data/nereids_rules_p0/salt_join/salt_join.out | 300 ++++++++++-----------
.../nereids_rules_p0/salt_join/salt_join.groovy | 84 +++---
2 files changed, 195 insertions(+), 189 deletions(-)
diff --git a/regression-test/data/nereids_rules_p0/salt_join/salt_join.out b/regression-test/data/nereids_rules_p0/salt_join/salt_join.out
index 7a8aa656ca6..e215f700af5 100644
--- a/regression-test/data/nereids_rules_p0/salt_join/salt_join.out
+++ b/regression-test/data/nereids_rules_p0/salt_join/salt_join.out
@@ -88,46 +88,6 @@
2 def 2 2 def 2
3 abc 4 3 abc 4
--- !leading --
-\N def 2 1 def 2
-\N def 2 2 def 2
-\N def 2 2 def 2
-1 abc 9 1 abc 9
-1 def 2 1 def 2
-1 def 2 2 def 2
-1 def 2 2 def 2
-2 abc 2 1 def 2
-2 abc 2 2 def 2
-2 abc 2 2 def 2
-2 def 2 1 def 2
-2 def 2 2 def 2
-2 def 2 2 def 2
-3 abc 4 3 abc 4
-
--- !has_other_equal_condition --
-1 abc 9 1 abc 9
-1 def 2 1 def 2
-2 abc 2 2 def 2
-2 abc 2 2 def 2
-2 def 2 2 def 2
-2 def 2 2 def 2
-3 abc 4 3 abc 4
-
--- !has_other_unequal_condition --
-1 def 2 2 def 2
-1 def 2 2 def 2
-
--- !test_varchar_skew_value --
-1 abc 1 1 abc 9
-1 abc 9 1 abc 9
-1 def 2 1 def 2
-2 def 2 2 def 2
-2 def 2 2 def 2
-3 abc \N 3 abc \N
-3 abc \N 3 abc 4
-3 abc 4 3 abc \N
-3 abc 4 3 abc 4
-
-- !test_multi_join --
\N def 2 1 def 2 1 def 2
\N def 2 1 def 2 2 def 2
@@ -258,76 +218,6 @@
2 def 2 2 def 2 2 def 2
3 abc 4 3 abc 4 3 abc 4
--- !leading_multi_join --
-1 abc 1 1 abc 9 1 abc 9
-1 abc 1 1 abc 9 3 abc \N
-1 abc 1 1 abc 9 3 abc 4
-1 abc 9 1 abc 9 1 abc 9
-1 abc 9 1 abc 9 3 abc \N
-1 abc 9 1 abc 9 3 abc 4
-1 def 2 1 def 2 1 def 2
-1 def 2 1 def 2 2 def 2
-1 def 2 1 def 2 2 def 2
-1 def 2 1 def 2 5 def \N
-1 def 2 1 def 2 5 def 8
-2 def 2 2 def 2 1 def 2
-2 def 2 2 def 2 1 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 5 def \N
-2 def 2 2 def 2 5 def \N
-2 def 2 2 def 2 5 def 8
-2 def 2 2 def 2 5 def 8
-3 abc \N 3 abc \N 1 abc 9
-3 abc \N 3 abc \N 3 abc \N
-3 abc \N 3 abc \N 3 abc 4
-3 abc \N 3 abc 4 1 abc 9
-3 abc \N 3 abc 4 3 abc \N
-3 abc \N 3 abc 4 3 abc 4
-3 abc 4 3 abc \N 1 abc 9
-3 abc 4 3 abc \N 3 abc \N
-3 abc 4 3 abc \N 3 abc 4
-3 abc 4 3 abc 4 1 abc 9
-3 abc 4 3 abc 4 3 abc \N
-3 abc 4 3 abc 4 3 abc 4
-
--- !leading_multi_join_bracket --
-1 abc 1 1 abc 9 1 abc 9
-1 abc 1 1 abc 9 3 abc \N
-1 abc 1 1 abc 9 3 abc 4
-1 abc 9 1 abc 9 1 abc 9
-1 abc 9 1 abc 9 3 abc \N
-1 abc 9 1 abc 9 3 abc 4
-1 def 2 1 def 2 1 def 2
-1 def 2 1 def 2 2 def 2
-1 def 2 1 def 2 2 def 2
-1 def 2 1 def 2 5 def \N
-1 def 2 1 def 2 5 def 8
-2 def 2 2 def 2 1 def 2
-2 def 2 2 def 2 1 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 2 def 2
-2 def 2 2 def 2 5 def \N
-2 def 2 2 def 2 5 def \N
-2 def 2 2 def 2 5 def 8
-2 def 2 2 def 2 5 def 8
-3 abc \N 3 abc \N 1 abc 9
-3 abc \N 3 abc \N 3 abc \N
-3 abc \N 3 abc \N 3 abc 4
-3 abc \N 3 abc 4 1 abc 9
-3 abc \N 3 abc 4 3 abc \N
-3 abc \N 3 abc 4 3 abc 4
-3 abc 4 3 abc \N 1 abc 9
-3 abc 4 3 abc \N 3 abc \N
-3 abc 4 3 abc \N 3 abc 4
-3 abc 4 3 abc 4 1 abc 9
-3 abc 4 3 abc 4 3 abc \N
-3 abc 4 3 abc 4 3 abc 4
-
-- !agg --
\N 3
1 4
@@ -405,46 +295,6 @@
2 def 2 2 def 2
3 abc 4 3 abc 4
--- !shape_leading_inner_subquery --
-PhysicalResultSink
---hashAgg[GLOBAL]
-----hashAgg[LOCAL]
-------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2) and (r1$c$1 = r2$c$2)) otherCondition=()
-----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=()
-------------PhysicalOlapScan[t2]
-------------PhysicalOlapScan[t4]
-----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((skewValue$c$5 = alias1.c1)) otherCondition=()
-------------PhysicalGenerate
---------------PhysicalUnion
-------------PhysicalOlapScan[t1]
---------PhysicalOlapScan[t3]
-
-Hint log:
-Used: leading(alias2 shuffle_skew t1 )
-UnUsed:
-SyntaxError:
-
--- !shape_leading_inner_subquery_switch --
-PhysicalResultSink
---hashAgg[GLOBAL]
-----hashAgg[LOCAL]
-------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2) and (r1$c$1 = r2$c$2)) otherCondition=()
-----------PhysicalOlapScan[t1]
-----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((skewValue$c$5 = alias2.c2)) otherCondition=()
-------------PhysicalGenerate
---------------PhysicalUnion
-------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=()
---------------PhysicalOlapScan[t2]
---------------PhysicalOlapScan[t4]
---------PhysicalOlapScan[t3]
-
-Hint log:
-Used: leading(t1 shuffle_skew alias2 )
-UnUsed:
-SyntaxError:
-
-- !inner_skew_value --
1 \N 1 \N
@@ -970,3 +820,153 @@ Used: [shuffle_skew]_1
UnUsed:
SyntaxError:
+-- !leading_multi_join --
+1 abc 1 1 abc 9 1 abc 9
+1 abc 1 1 abc 9 3 abc \N
+1 abc 1 1 abc 9 3 abc 4
+1 abc 9 1 abc 9 1 abc 9
+1 abc 9 1 abc 9 3 abc \N
+1 abc 9 1 abc 9 3 abc 4
+1 def 2 1 def 2 1 def 2
+1 def 2 1 def 2 2 def 2
+1 def 2 1 def 2 2 def 2
+1 def 2 1 def 2 5 def \N
+1 def 2 1 def 2 5 def 8
+2 def 2 2 def 2 1 def 2
+2 def 2 2 def 2 1 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 5 def \N
+2 def 2 2 def 2 5 def \N
+2 def 2 2 def 2 5 def 8
+2 def 2 2 def 2 5 def 8
+3 abc \N 3 abc \N 1 abc 9
+3 abc \N 3 abc \N 3 abc \N
+3 abc \N 3 abc \N 3 abc 4
+3 abc \N 3 abc 4 1 abc 9
+3 abc \N 3 abc 4 3 abc \N
+3 abc \N 3 abc 4 3 abc 4
+3 abc 4 3 abc \N 1 abc 9
+3 abc 4 3 abc \N 3 abc \N
+3 abc 4 3 abc \N 3 abc 4
+3 abc 4 3 abc 4 1 abc 9
+3 abc 4 3 abc 4 3 abc \N
+3 abc 4 3 abc 4 3 abc 4
+
+-- !leading_multi_join_bracket --
+1 abc 1 1 abc 9 1 abc 9
+1 abc 1 1 abc 9 3 abc \N
+1 abc 1 1 abc 9 3 abc 4
+1 abc 9 1 abc 9 1 abc 9
+1 abc 9 1 abc 9 3 abc \N
+1 abc 9 1 abc 9 3 abc 4
+1 def 2 1 def 2 1 def 2
+1 def 2 1 def 2 2 def 2
+1 def 2 1 def 2 2 def 2
+1 def 2 1 def 2 5 def \N
+1 def 2 1 def 2 5 def 8
+2 def 2 2 def 2 1 def 2
+2 def 2 2 def 2 1 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 2 def 2
+2 def 2 2 def 2 5 def \N
+2 def 2 2 def 2 5 def \N
+2 def 2 2 def 2 5 def 8
+2 def 2 2 def 2 5 def 8
+3 abc \N 3 abc \N 1 abc 9
+3 abc \N 3 abc \N 3 abc \N
+3 abc \N 3 abc \N 3 abc 4
+3 abc \N 3 abc 4 1 abc 9
+3 abc \N 3 abc 4 3 abc \N
+3 abc \N 3 abc 4 3 abc 4
+3 abc 4 3 abc \N 1 abc 9
+3 abc 4 3 abc \N 3 abc \N
+3 abc 4 3 abc \N 3 abc 4
+3 abc 4 3 abc 4 1 abc 9
+3 abc 4 3 abc 4 3 abc \N
+3 abc 4 3 abc 4 3 abc 4
+
+-- !leading --
+\N def 2 1 def 2
+\N def 2 2 def 2
+\N def 2 2 def 2
+1 abc 9 1 abc 9
+1 def 2 1 def 2
+1 def 2 2 def 2
+1 def 2 2 def 2
+2 abc 2 1 def 2
+2 abc 2 2 def 2
+2 abc 2 2 def 2
+2 def 2 1 def 2
+2 def 2 2 def 2
+2 def 2 2 def 2
+3 abc 4 3 abc 4
+
+-- !has_other_equal_condition --
+1 abc 9 1 abc 9
+1 def 2 1 def 2
+2 abc 2 2 def 2
+2 abc 2 2 def 2
+2 def 2 2 def 2
+2 def 2 2 def 2
+3 abc 4 3 abc 4
+
+-- !has_other_unequal_condition --
+1 def 2 2 def 2
+1 def 2 2 def 2
+
+-- !test_varchar_skew_value --
+1 abc 1 1 abc 9
+1 abc 9 1 abc 9
+1 def 2 1 def 2
+2 def 2 2 def 2
+2 def 2 2 def 2
+3 abc \N 3 abc \N
+3 abc \N 3 abc 4
+3 abc 4 3 abc \N
+3 abc 4 3 abc 4
+
+-- !shape_leading_inner_subquery --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=()
+--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2) and (r1$c$1 = r2$c$2)) otherCondition=()
+----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=()
+------------PhysicalOlapScan[t2]
+------------PhysicalOlapScan[t4]
+----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((skewValue$c$5 = alias1.c1)) otherCondition=()
+------------PhysicalGenerate
+--------------PhysicalUnion
+------------PhysicalOlapScan[t1]
+--------PhysicalOlapScan[t3]
+
+Hint log:
+Used: leading(alias2 shuffle_skew t1 )
+UnUsed:
+SyntaxError:
+
+-- !shape_leading_inner_subquery_switch --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=()
+--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2) and (r1$c$1 = r2$c$2)) otherCondition=()
+----------PhysicalOlapScan[t1]
+----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((skewValue$c$5 = alias2.c2)) otherCondition=()
+------------PhysicalGenerate
+--------------PhysicalUnion
+------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=()
+--------------PhysicalOlapScan[t2]
+--------------PhysicalOlapScan[t4]
+--------PhysicalOlapScan[t3]
+
+Hint log:
+Used: leading(t1 shuffle_skew alias2 )
+UnUsed:
+SyntaxError:
+
diff --git a/regression-test/suites/nereids_rules_p0/salt_join/salt_join.groovy b/regression-test/suites/nereids_rules_p0/salt_join/salt_join.groovy
index bea4814efb3..fbf17db586e 100644
--- a/regression-test/suites/nereids_rules_p0/salt_join/salt_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/salt_join/salt_join.groovy
@@ -20,6 +20,7 @@ suite("salt_join") {
sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
sql "set disable_nereids_rules='prune_empty_partition'"
sql "set runtime_filter_mode=OFF"
+ sql "set disable_join_reorder=true"
sql "drop table if exists test_skew9;"
sql """create table test_skew9(a int,c varchar(100), b int) distributed by hash(a) buckets 32 properties("replication_num"="1");"""
sql """insert into test_skew9 values(1,'abc',9),(1,'abc',1),(1,'def',2),(null,'def',2),(2,'abc',2),(3,'abc',4),(5,'abc',6),(2,'def',2),(5,'abc',null),(3,'abc',null)"""
@@ -41,18 +42,6 @@ suite("salt_join") {
qt_skew_value_only_null """
select * from test_skew9 tl inner join [shuffle[skew(tl.b(null))]] test_skew10 tr on tl.b = tr.b order by 1,2,3,4,5,6;
"""
- qt_leading """
- select /*+leading(tl shuffle [skew(tl.b(1,2))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.b=tr.b order by 1,2,3,4,5,6;
- """
- qt_has_other_equal_condition """
- select /*+leading(tl shuffle [skew(tl.b(1,2))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.b=tr.b and tl.a=tr.a order by 1,2,3,4,5,6;
- """
- qt_has_other_unequal_condition """
- select /*+leading(tl shuffle [skew(tl.b(1,2))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.b=tr.b and tl.a<tr.a order by 1,2,3,4,5,6;
- """
- qt_test_varchar_skew_value """
- select /*+leading(tl shuffle [skew(tl.c("abc","def"))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.c=tr.c and tl.a=tr.a order by 1,2,3,4,5,6;
- """
qt_test_multi_join """
select * from test_skew9 tl inner join [shuffle[skew(tl.b(1,2,null))]] test_skew10 tr on tl.b = tr.b left join [shuffle[skew(tl.b(1,2,null))]] test_skew10 tt on tl.b=tt.b
order by 1,2,3,4,5,6,7,8,9;
@@ -65,16 +54,6 @@ suite("salt_join") {
select * from test_skew9 tl left join [shuffle[skew(tl.b(1,2,null))]] test_skew10 tr on tl.b = tr.b right join [shuffle[skew(tl.b(1,2,null))]] test_skew10 tt on tl.b=tt.b
order by 1,2,3,4,5,6,7,8,9;
"""
- qt_leading_multi_join """
- select /*+leading(tl shuffle [skew(tl.c("abc","def"))] tr shuffle[skew(tl.c("abc","def"))] tt) */ * from
- test_skew9 tl join test_skew10 tr on tl.c=tr.c and tl.a=tr.a inner join test_skew10 tt on tl.c = tt.c
- order by 1,2,3,4,5,6,7,8,9;
- """
- qt_leading_multi_join_bracket """
- select /*+leading(tl shuffle [skew(tl.c("abc","def"))] {tr shuffle[skew(tr.c("abc","def"))] tt}) */ * from
- test_skew9 tl join test_skew10 tr on tl.c=tr.c and tl.a=tr.a inner join test_skew10 tt on tr.c = tt.c
- order by 1,2,3,4,5,6,7,8,9;
- """
// has other operator
// agg
@@ -186,23 +165,6 @@ suite("salt_join") {
order by 1,2,3,4,5,6;
"""
- sql """drop table if exists t1;"""
- sql """drop table if exists t2;"""
- sql """drop table if exists t3;"""
- sql """drop table if exists t4;"""
- sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');"""
- sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');"""
- sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');"""
- sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');"""
- qt_shape_leading_inner_subquery """
- explain shape plan
- select count(*) from (select /*+leading(alias2 shuffle[skew(alias2.c2(1,2))] t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;
- """
- qt_shape_leading_inner_subquery_switch """
- explain shape plan
- select count(*) from (select /*+leading(t1 shuffle[skew(t1.c1(1,2))] alias2) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;
- """
-
// test null safe equal
sql "drop table if exists test_null_safe1"
sql "drop table if exists test_null_safe2"
@@ -279,4 +241,48 @@ suite("salt_join") {
qt_null_safe_equal_inner_other_and_null_shape "explain shape plan select t1.a,t1.b,t2.a,t2.b from test_null_safe3 t1 inner join[shuffle[skew(t1.b(1,2,null))]] test_null_safe4 t2 on t1.b<=>t2.b;"
qt_null_safe_equal_left_other_and_null_shape "explain shape plan select t1.a,t1.b,t2.a,t2.b from test_null_safe3 t1 left join[shuffle[skew(t1.b(1,2,null))]] test_null_safe4 t2 on t1.b<=>t2.b;"
qt_null_safe_equal_right_other_and_null_shape "explain shape plan select t1.a,t1.b,t2.a,t2.b from test_null_safe3 t1 right join[shuffle[skew(t2.b(1,2,null))]] test_null_safe4 t2 on t1.b<=>t2.b;"
+
+ sql "set disable_join_reorder=false"
+
+ qt_leading_multi_join """
+ select /*+leading(tl shuffle [skew(tl.c("abc","def"))] tr shuffle[skew(tl.c("abc","def"))] tt) */ * from
+ test_skew9 tl join test_skew10 tr on tl.c=tr.c and tl.a=tr.a inner join test_skew10 tt on tl.c = tt.c
+ order by 1,2,3,4,5,6,7,8,9;
+ """
+ qt_leading_multi_join_bracket """
+ select /*+leading(tl shuffle [skew(tl.c("abc","def"))] {tr shuffle[skew(tr.c("abc","def"))] tt}) */ * from
+ test_skew9 tl join test_skew10 tr on tl.c=tr.c and tl.a=tr.a inner join test_skew10 tt on tr.c = tt.c
+ order by 1,2,3,4,5,6,7,8,9;
+ """
+
+ qt_leading """
+ select /*+leading(tl shuffle [skew(tl.b(1,2))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.b=tr.b order by 1,2,3,4,5,6;
+ """
+ qt_has_other_equal_condition """
+ select /*+leading(tl shuffle [skew(tl.b(1,2))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.b=tr.b and tl.a=tr.a order by 1,2,3,4,5,6;
+ """
+ qt_has_other_unequal_condition """
+ select /*+leading(tl shuffle [skew(tl.b(1,2))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.b=tr.b and tl.a<tr.a order by 1,2,3,4,5,6;
+ """
+ qt_test_varchar_skew_value """
+ select /*+leading(tl shuffle [skew(tl.c("abc","def"))] tr) */ * from test_skew9 tl join test_skew10 tr on tl.c=tr.c and tl.a=tr.a order by 1,2,3,4,5,6;
+ """
+
+ sql """drop table if exists t1;"""
+ sql """drop table if exists t2;"""
+ sql """drop table if exists t3;"""
+ sql """drop table if exists t4;"""
+ sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');"""
+ sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');"""
+ sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');"""
+ sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');"""
+
+ qt_shape_leading_inner_subquery """
+ explain shape plan
+ select count(*) from (select /*+leading(alias2 shuffle[skew(alias2.c2(1,2))] t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;
+ """
+ qt_shape_leading_inner_subquery_switch """
+ explain shape plan
+ select count(*) from (select /*+leading(t1 shuffle[skew(t1.c1(1,2))] alias2) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;
+ """
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/09: branch-4.0: [fix](es-catalog) Fix query error when ES keyword field contains array data #61236 (#61361)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c6d944981d09abfaafe578eaf866d3321b8c1910
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:06:17 2026 +0800
branch-4.0: [fix](es-catalog) Fix query error when ES keyword field contains array data #61236 (#61361)
Cherry-picked from #61236
Co-authored-by: Mingyu Chen (Rayner) <mo...@163.com>
---
be/src/exec/es/es_scroll_parser.cpp | 20 +++-
.../scripts/data/test_keyword_array.json | 3 +
.../elasticsearch/scripts/es_init.sh | 18 +++-
.../scripts/index/es7_test_keyword_array.json | 28 +++++
.../es/test_es_keyword_array_type.out | 33 ++++++
.../es/test_es_keyword_array_type.groovy | 115 +++++++++++++++++++++
6 files changed, 212 insertions(+), 5 deletions(-)
diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp
index 983d980e66d..6c3ead474b1 100644
--- a/be/src/exec/es/es_scroll_parser.cpp
+++ b/be/src/exec/es/es_scroll_parser.cpp
@@ -439,8 +439,12 @@ Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pu
return Status::OK();
}
if constexpr (T == TYPE_STRING || T == TYPE_CHAR || T == TYPE_VARCHAR) {
- RETURN_ERROR_IF_COL_IS_ARRAY(col, sub_type, true);
- if (!col.IsString()) {
+ // When ES mapping is keyword/text but actual data is an array,
+ // serialize the array to JSON string instead of throwing an error.
+ // This is valid in ES since any field can hold array values.
+ if (col.IsArray()) {
+ val = json_value_to_string(col);
+ } else if (!col.IsString()) {
val = json_value_to_string(col);
} else {
val = col.GetString();
@@ -709,14 +713,22 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc,
if (pure_doc_value) {
if (col.Empty()) {
break;
+ } else if (col.Size() > 1) {
+ // doc_values with multiple elements means actual array data
+ // in ES keyword/text field, serialize as JSON array string
+ val = json_value_to_string(col);
} else if (!col[0].IsString()) {
val = json_value_to_string(col[0]);
} else {
val = col[0].GetString();
}
} else {
- RETURN_ERROR_IF_COL_IS_ARRAY(col, type, true);
- if (!col.IsString()) {
+ // When ES mapping is keyword/text but actual data is an array,
+ // serialize the array to JSON string instead of throwing an error.
+ // This is valid in ES since any field can hold array values.
+ if (col.IsArray()) {
+ val = json_value_to_string(col);
+ } else if (!col.IsString()) {
val = json_value_to_string(col);
} else {
val = col.GetString();
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/test_keyword_array.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/test_keyword_array.json
new file mode 100644
index 00000000000..72979d2a459
--- /dev/null
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/test_keyword_array.json
@@ -0,0 +1,3 @@
+{"id": 1, "name": "Alice", "tags": ["tag1", "tag2", "tag3"], "description": ["first line", "second line"], "emails": ["alice@example.com", "alice@work.com"], "single_value": "hello"}
+{"id": 2, "name": "Bob", "tags": "single_tag", "description": "simple text", "emails": ["bob@example.com"], "single_value": "world"}
+{"id": 3, "name": "Charlie", "tags": ["tag_a", "tag_b"], "description": ["only line"], "emails": "charlie@example.com", "single_value": ["val1", "val2"]}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
index d45e4b5fb4a..51f81ced938 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
@@ -23,7 +23,7 @@ generate_bulk_request() {
local data_file=$4
local output_file=$5
- // clear output file
+ # clear output file
echo "" > "$output_file"
local id=1
@@ -174,6 +174,14 @@ curl -X POST "http://${ES_7_HOST}:9200/_bulk" --data-binary "@$bulk_request_file
# put _meta for composite_type_array
curl "http://${ES_7_HOST}:9200/composite_type_array/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta_composite_type_array.json"
+# create index test_keyword_array for keyword array type testing
+curl "http://${ES_7_HOST}:9200/test_keyword_array" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test_keyword_array.json'
+# put data for test_keyword_array
+keyword_array_data_file="/mnt/scripts/data/test_keyword_array.json"
+keyword_array_bulk_file="/mnt/scripts/data/bulk_request_keyword_array_es7.json"
+generate_bulk_request "test_keyword_array" "_doc" "item_" "$keyword_array_data_file" "$keyword_array_bulk_file"
+curl -X POST "http://${ES_7_HOST}:9200/_bulk" --data-binary "@$keyword_array_bulk_file" -H "Content-Type: application/json"
+
# es8
# create index test1
curl "http://${ES_8_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json"
@@ -217,4 +225,12 @@ curl -X POST "http://${ES_8_HOST}:9200/_bulk" --data-binary "@$bulk_request_file
# put _meta for composite_type_array
curl "http://${ES_8_HOST}:9200/composite_type_array/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta_composite_type_array.json"
+# create index test_keyword_array for keyword array type testing
+curl "http://${ES_8_HOST}:9200/test_keyword_array" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test_keyword_array.json'
+# put data for test_keyword_array
+keyword_array_data_file="/mnt/scripts/data/test_keyword_array.json"
+keyword_array_bulk_file="/mnt/scripts/data/bulk_request_keyword_array_es8.json"
+generate_bulk_request "test_keyword_array" "" "item_" "$keyword_array_data_file" "$keyword_array_bulk_file"
+curl -X POST "http://${ES_8_HOST}:9200/_bulk" --data-binary "@$keyword_array_bulk_file" -H "Content-Type: application/json"
+
touch /tmp/SUCCESS
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test_keyword_array.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test_keyword_array.json
new file mode 100644
index 00000000000..3bd8e786408
--- /dev/null
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test_keyword_array.json
@@ -0,0 +1,28 @@
+{
+ "settings": {
+ "number_of_shards": 1,
+ "number_of_replicas": 0
+ },
+ "mappings": {
+ "properties": {
+ "id": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "keyword"
+ },
+ "description": {
+ "type": "text"
+ },
+ "emails": {
+ "type": "keyword"
+ },
+ "single_value": {
+ "type": "keyword"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/regression-test/data/external_table_p0/es/test_es_keyword_array_type.out b/regression-test/data/external_table_p0/es/test_es_keyword_array_type.out
new file mode 100644
index 00000000000..13e75a94f95
--- /dev/null
+++ b/regression-test/data/external_table_p0/es/test_es_keyword_array_type.out
@@ -0,0 +1,33 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !es7_keyword_array_01 --
+1 Alice ["tag1","tag2","tag3"] ["alice@example.com","alice@work.com"] hello
+2 Bob single_tag ["bob@example.com"] world
+3 Charlie ["tag_a","tag_b"] charlie@example.com ["val1","val2"]
+
+-- !es7_keyword_array_02 --
+1 Alice ["first line","second line"]
+2 Bob simple text
+3 Charlie ["only line"]
+
+-- !es7_keyword_array_03 --
+1 ["tag1","tag2","tag3"]
+
+-- !es7_keyword_array_04 --
+3 ["val1","val2"]
+
+-- !es8_keyword_array_01 --
+1 Alice ["tag1","tag2","tag3"] ["alice@example.com","alice@work.com"] hello
+2 Bob single_tag ["bob@example.com"] world
+3 Charlie ["tag_a","tag_b"] charlie@example.com ["val1","val2"]
+
+-- !es8_keyword_array_02 --
+1 Alice ["first line","second line"]
+2 Bob simple text
+3 Charlie ["only line"]
+
+-- !es8_keyword_array_03 --
+1 ["tag1","tag2","tag3"]
+
+-- !es8_keyword_array_04 --
+3 ["val1","val2"]
+
diff --git a/regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy b/regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy
new file mode 100644
index 00000000000..2c153c940f9
--- /dev/null
+++ b/regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test for https://github.com/apache/doris/issues/XXXXX
+// When ES mapping defines a field as keyword/text, but the actual data stored
+// is an array, Doris should not throw an error. Instead, it should serialize
+// the array to a JSON string representation.
+
+suite("test_es_keyword_array_type", "p0,external,es,external_docker,external_docker_es") {
+ String enabled = context.config.otherConfigs.get("enableEsTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String es_7_port = context.config.otherConfigs.get("es_7_port")
+ String es_8_port = context.config.otherConfigs.get("es_8_port")
+
+ sql """drop catalog if exists test_es_keyword_array_es7;"""
+ sql """drop catalog if exists test_es_keyword_array_es8;"""
+
+ // Create ES7 catalog
+ sql """create catalog if not exists test_es_keyword_array_es7 properties(
+ "type"="es",
+ "hosts"="http://${externalEnvIp}:$es_7_port",
+ "nodes_discovery"="false",
+ "enable_keyword_sniff"="true",
+ "enable_docvalue_scan"="false"
+ );
+ """
+
+ // Create ES8 catalog
+ sql """create catalog if not exists test_es_keyword_array_es8 properties(
+ "type"="es",
+ "hosts"="http://${externalEnvIp}:$es_8_port",
+ "nodes_discovery"="false",
+ "enable_keyword_sniff"="true",
+ "enable_docvalue_scan"="false"
+ );
+ """
+
+ // Test ES7
+ sql """switch test_es_keyword_array_es7"""
+ // Wait for metadata sync
+ def maxRetries = 30
+ def retryCount = 0
+ def success = false
+ while (!success && retryCount < maxRetries) {
+ try {
+ sql """select * from test_keyword_array"""
+ success = true
+ } catch (Exception e) {
+ if (e.getMessage().contains("EsTable metadata has not been synced, Try it later")) {
+ logger.info("Waiting for ES metadata sync... Attempt ${retryCount + 1}")
+ retryCount++
+ sleep(1000)
+ } else {
+ throw e
+ }
+ }
+ }
+ if (!success) {
+ throw new RuntimeException("Failed to sync ES metadata after ${maxRetries} attempts")
+ }
+
+ // Test: Query keyword fields that contain array data should not throw error
+ // Previously this would fail with: "Expected value of type: STRING; but found type: Array"
+ order_qt_es7_keyword_array_01 """select id, name, tags, emails, single_value from test_keyword_array order by id"""
+ order_qt_es7_keyword_array_02 """select id, name, description from test_keyword_array order by id"""
+ order_qt_es7_keyword_array_03 """select id, tags from test_keyword_array where id = 1"""
+ order_qt_es7_keyword_array_04 """select id, single_value from test_keyword_array where id = 3"""
+
+ // Test ES8
+ sql """switch test_es_keyword_array_es8"""
+ retryCount = 0
+ success = false
+ while (!success && retryCount < maxRetries) {
+ try {
+ sql """select * from test_keyword_array"""
+ success = true
+ } catch (Exception e) {
+ if (e.getMessage().contains("EsTable metadata has not been synced, Try it later")) {
+ logger.info("Waiting for ES metadata sync... Attempt ${retryCount + 1}")
+ retryCount++
+ sleep(1000)
+ } else {
+ throw e
+ }
+ }
+ }
+ if (!success) {
+ throw new RuntimeException("Failed to sync ES metadata after ${maxRetries} attempts")
+ }
+
+ order_qt_es8_keyword_array_01 """select id, name, tags, emails, single_value from test_keyword_array order by id"""
+ order_qt_es8_keyword_array_02 """select id, name, description from test_keyword_array order by id"""
+ order_qt_es8_keyword_array_03 """select id, tags from test_keyword_array where id = 1"""
+ order_qt_es8_keyword_array_04 """select id, single_value from test_keyword_array where id = 3"""
+
+ // Cleanup
+ sql """drop catalog if exists test_es_keyword_array_es7;"""
+ sql """drop catalog if exists test_es_keyword_array_es8;"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/09: branch-4.0: [opt](cloud) Support fdb locality aware load balancing #61312 (#61375)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 70c4f0d85a548177e9e43b459cde2eadafd9390e
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:07:44 2026 +0800
branch-4.0: [opt](cloud) Support fdb locality aware load balancing #61312 (#61375)
Cherry-picked from #61312
Co-authored-by: walter <ma...@selectdb.com>
---
cloud/src/common/config.h | 5 ++++
cloud/src/meta-store/txn_kv.cpp | 57 ++++++++++++++++++++++++++++++++++++++++-
2 files changed, 61 insertions(+), 1 deletion(-)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 9e27321ac2c..214f7599995 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -34,6 +34,11 @@ CONF_Bool(enable_fdb_external_client_directory, "true");
// The directory path of external foundationdb client library.
// eg: /path/to/dir1:/path/to/dir2:...
CONF_String(fdb_external_client_directory, "./lib/fdb/7.3.69/");
+// Enable FDB locality-aware load balancing. When enabled, fdb_zone_id and fdb_dc_id
+// will be set on the database for better location-aware request routing.
+CONF_Bool(enable_fdb_locality_load_balance, "false");
+CONF_String(fdb_zone_id, "");
+CONF_String(fdb_dc_id, "");
CONF_String(http_token, "greedisgood9999");
// use volatile mem kv for test. MUST NOT be `true` in production environment.
CONF_Bool(use_mem_kv, "false");
diff --git a/cloud/src/meta-store/txn_kv.cpp b/cloud/src/meta-store/txn_kv.cpp
index 70740ac3747..1e79061c72b 100644
--- a/cloud/src/meta-store/txn_kv.cpp
+++ b/cloud/src/meta-store/txn_kv.cpp
@@ -313,6 +313,10 @@ inline constexpr size_t write_clear_range_approximate_size(size_t begin_size, si
} // anonymous namespace
+std::string_view KNOB_LOAD_BALANCE_ZONE_ID_LOCALITY_ENABLED =
+ "load_balance_zone_id_locality_enabled=1";
+std::string_view KNOB_LOAD_BALANCE_DC_ID_LOCALITY_ENABLED = "load_balance_dc_id_locality_enabled=1";
+
// Ref https://apple.github.io/foundationdb/api-error-codes.html#developer-guide-error-codes.
constexpr fdb_error_t FDB_ERROR_CODE_TIMED_OUT = 1004;
constexpr fdb_error_t FDB_ERROR_CODE_TXN_TOO_OLD = 1007;
@@ -406,6 +410,33 @@ int Network::init() {
LOG(INFO) << "set fdb external client directory: " << config::fdb_external_client_directory;
}
+ if (config::enable_fdb_locality_load_balance) {
+ if (!config::fdb_zone_id.empty()) {
+ err = fdb_network_set_option(
+ FDB_NET_OPTION_KNOB,
+ (const uint8_t*)KNOB_LOAD_BALANCE_ZONE_ID_LOCALITY_ENABLED.data(),
+ KNOB_LOAD_BALANCE_ZONE_ID_LOCALITY_ENABLED.size());
+ if (err) {
+ LOG(WARNING) << "failed to set fdb load balance zone id locality enabled, err: "
+ << fdb_get_error(err);
+ return 1;
+ }
+ LOG(INFO) << "set fdb load balance zone id locality enabled";
+ }
+ if (!config::fdb_dc_id.empty()) {
+ err = fdb_network_set_option(
+ FDB_NET_OPTION_KNOB,
+ (const uint8_t*)KNOB_LOAD_BALANCE_DC_ID_LOCALITY_ENABLED.data(),
+ KNOB_LOAD_BALANCE_DC_ID_LOCALITY_ENABLED.size());
+ if (err) {
+ LOG(WARNING) << "failed to set fdb load balance dc id locality enabled, err: "
+ << fdb_get_error(err);
+ return 1;
+ }
+ LOG(INFO) << "set fdb load balance dc id locality enabled";
+ }
+ }
+
// ATTN: Network can be configured only once,
// even if fdb_stop_network() is called successfully
err = fdb_setup_network(); // Must be called only once before any
@@ -457,7 +488,6 @@ void Network::stop() {
// =============================================================================
int Database::init() {
- // TODO: process opt
fdb_error_t err = fdb_create_database(cluster_file_path_.c_str(), &db_);
if (err) {
LOG(WARNING) << __PRETTY_FUNCTION__ << " fdb_create_database error: " << fdb_get_error(err)
@@ -465,6 +495,31 @@ int Database::init() {
return 1;
}
+ if (config::enable_fdb_locality_load_balance) {
+ if (!config::fdb_zone_id.empty()) {
+ err = fdb_database_set_option(db_, FDB_DB_OPTION_MACHINE_ID,
+ (const uint8_t*)config::fdb_zone_id.c_str(),
+ config::fdb_zone_id.size());
+ if (err) {
+ LOG(WARNING) << "failed to set FDB_DB_OPTION_MACHINE_ID: " << fdb_get_error(err)
+ << ", zone_id: " << config::fdb_zone_id;
+ return 1;
+ }
+ LOG(INFO) << "set FDB_DB_OPTION_MACHINE_ID (zone_id): " << config::fdb_zone_id;
+ }
+ if (!config::fdb_dc_id.empty()) {
+ err = fdb_database_set_option(db_, FDB_DB_OPTION_DATACENTER_ID,
+ (const uint8_t*)config::fdb_dc_id.c_str(),
+ config::fdb_dc_id.size());
+ if (err) {
+ LOG(WARNING) << "failed to set FDB_DB_OPTION_DATACENTER_ID: " << fdb_get_error(err)
+ << ", dc_id: " << config::fdb_dc_id;
+ return 1;
+ }
+ LOG(INFO) << "set FDB_DB_OPTION_DATACENTER_ID (dc_id): " << config::fdb_dc_id;
+ }
+ }
+
return 0;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org