You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/21 06:05:01 UTC
[doris] branch branch-2.0-alpha updated (19870d99cd -> 19887c7586)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
from 19870d99cd [branch-2.0-alpha] fix mtmv thread interruption issue
new cf3f04567d [Fix](Nereids) Check bound status in analyze straight after bounding (#18581)
new de36bf3c07 [fix](editLog) add sufficient replay logic and edit log for altering light schema change (#18746)
new 0ceff07202 [fix](outfile) unify broker and hdfs path in outfile (#18809)
new 233975b6f6 [MemLeak](pipeline) fix mem leak by exchange node in pipeline (#18864)
new 1dc2011da6 [chore](cold_heat) fix cold heat case to use correct http api (#18870)
new 95c4b53c4e [Bug](table-function) fix wrong result when seprator of explode_split size more than one (#18824)
new 7d2a96468b [Fix](dynamic table) fix dynamic table with insert into and column al… (#18808)
new 19887c7586 Author: Mingyu Chen <mo...@163.com> Date: Fri Apr 21 11:24:20 2023 +0800
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/olap/memtable.cpp | 11 +-
be/src/vec/columns/column_object.cpp | 15 ++-
be/src/vec/columns/column_object.h | 30 +++++
be/src/vec/exprs/table_function/vexplode_split.cpp | 3 +-
be/src/vec/sink/vdata_stream_sender.h | 13 ++-
...CHelper.java => AlterLightSchChangeHelper.java} | 52 ++++-----
.../apache/doris/alter/SchemaChangeHandler.java | 19 +++-
.../org/apache/doris/analysis/OutFileClause.java | 9 +-
.../org/apache/doris/catalog/TableProperty.java | 3 -
.../org/apache/doris/journal/JournalEntity.java | 7 +-
.../doris/nereids/analyzer/NereidsAnalyzer.java | 4 +
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../nereids/rules/analysis/CheckAnalysis.java | 24 ++--
.../doris/nereids/rules/analysis/CheckBound.java | 74 +++++++++++++
...ionLog.java => AlterLightSchemaChangeInfo.java} | 29 ++---
.../java/org/apache/doris/persist/EditLog.java | 8 +-
.../doris/analysis/OutFileClauseFunctionTest.java | 122 +++++++++++++--------
.../rules/analysis/BindSlotReferenceTest.java | 2 +-
.../sql_functions/table_function/explode_split.out | 6 +
.../create_table_use_partition_policy.groovy | 12 +-
.../create_table_use_policy.groovy | 2 +-
.../modify_replica_use_partition.groovy | 4 +-
.../table_modify_resouce_and_policy.groovy | 2 +-
.../test_set_session_default_val.groovy | 4 +-
.../suites/datev2/ssb_sf0.1_p1/load.groovy | 2 +-
.../suites/datev2/ssb_sf1_p2/load.groovy | 2 +-
.../suites/datev2/tpcds_sf1_p1/load.groovy | 4 +-
.../suites/dynamic_table_p0/load.groovy | 4 +-
.../nereids_p0/except/test_bound_exception.groovy | 55 ++++++++++
.../table_function/explode_split.groovy | 1 +
regression-test/suites/ssb_sf0.1_p1/load.groovy | 2 +-
regression-test/suites/ssb_sf100_p2/load.groovy | 2 +-
regression-test/suites/ssb_sf1_p2/load.groovy | 2 +-
regression-test/suites/tpcds_sf100_p2/load.groovy | 2 +-
regression-test/suites/tpcds_sf1_p1/load.groovy | 4 +-
.../suites/tpcds_sf1_unique_p1/load.groovy | 2 +-
36 files changed, 391 insertions(+), 147 deletions(-)
rename fe/fe-core/src/main/java/org/apache/doris/alter/{AlterLSCHelper.java => AlterLightSchChangeHelper.java} (87%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckBound.java
copy fe/fe-core/src/main/java/org/apache/doris/persist/{ModifyTableEngineOperationLog.java => AlterLightSchemaChangeInfo.java} (67%)
create mode 100644 regression-test/suites/nereids_p0/except/test_bound_exception.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 08/08: Author: Mingyu Chen Date: Fri Apr 21 11:24:20 2023 +0800
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 19887c7586244fb6adb4b0a39ae17fe1145e1756
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Fri Apr 21 11:24:20 2023 +0800
Author: Mingyu Chen <mo...@163.com>
Date: Fri Apr 21 11:24:20 2023 +0800
[improvement](test) remove set global (#18807)
---
.../suites/correctness/test_set_session_default_val.groovy | 4 ++--
regression-test/suites/datev2/ssb_sf0.1_p1/load.groovy | 2 +-
regression-test/suites/datev2/ssb_sf1_p2/load.groovy | 2 +-
regression-test/suites/datev2/tpcds_sf1_p1/load.groovy | 4 ++--
regression-test/suites/ssb_sf0.1_p1/load.groovy | 2 +-
regression-test/suites/ssb_sf100_p2/load.groovy | 2 +-
regression-test/suites/ssb_sf1_p2/load.groovy | 2 +-
regression-test/suites/tpcds_sf100_p2/load.groovy | 2 +-
regression-test/suites/tpcds_sf1_p1/load.groovy | 4 ++--
regression-test/suites/tpcds_sf1_unique_p1/load.groovy | 2 +-
10 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/regression-test/suites/correctness/test_set_session_default_val.groovy b/regression-test/suites/correctness/test_set_session_default_val.groovy
index b39cd7f1c4..26a2aec639 100644
--- a/regression-test/suites/correctness/test_set_session_default_val.groovy
+++ b/regression-test/suites/correctness/test_set_session_default_val.groovy
@@ -17,8 +17,8 @@
suite("test_set_session_default_val") {
def default_timeout = sql """show variables where variable_name = 'insert_timeout';"""
- sql """set global insert_timeout=3000;"""
+ sql """set insert_timeout=3000;"""
sql """set session insert_timeout=${default_timeout[0][1]};"""
def session_timeout = sql """show variables where variable_name = 'insert_timeout';"""
assertEquals(default_timeout, session_timeout)
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/datev2/ssb_sf0.1_p1/load.groovy b/regression-test/suites/datev2/ssb_sf0.1_p1/load.groovy
index 6ed3a68b91..e047c25614 100644
--- a/regression-test/suites/datev2/ssb_sf0.1_p1/load.groovy
+++ b/regression-test/suites/datev2/ssb_sf0.1_p1/load.groovy
@@ -88,7 +88,7 @@ suite("load") {
def rowCount = sql "select count(*) from ${table}"
if (rowCount[0][0] != table_rows) {
sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
- sql "set global insert_timeout=3600"
+ sql "set insert_timeout=3600"
def r = sql "select @@insert_timeout"
assertEquals(3600, r[0][0])
year_cons = [
diff --git a/regression-test/suites/datev2/ssb_sf1_p2/load.groovy b/regression-test/suites/datev2/ssb_sf1_p2/load.groovy
index 5006e1a384..59d909b22d 100644
--- a/regression-test/suites/datev2/ssb_sf1_p2/load.groovy
+++ b/regression-test/suites/datev2/ssb_sf1_p2/load.groovy
@@ -88,7 +88,7 @@ suite("load") {
def rowCount = sql "select count(*) from ${table}"
if (rowCount[0][0] != table_rows) {
sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
- sql "set global insert_timeout=3600"
+ sql "set insert_timeout=3600"
def r = sql "select @@insert_timeout"
assertEquals(3600, r[0][0])
year_cons = [
diff --git a/regression-test/suites/datev2/tpcds_sf1_p1/load.groovy b/regression-test/suites/datev2/tpcds_sf1_p1/load.groovy
index 78927b93f1..d370954f73 100644
--- a/regression-test/suites/datev2/tpcds_sf1_p1/load.groovy
+++ b/regression-test/suites/datev2/tpcds_sf1_p1/load.groovy
@@ -55,7 +55,7 @@ suite("load") {
sql new File("""${context.file.parent}/ddl/${table}.sql""").text
}
- sql "set global exec_mem_limit=8G;"
+ sql "set exec_mem_limit=8G;"
for (String tableName in tables) {
streamLoad {
@@ -98,4 +98,4 @@ suite("load") {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/ssb_sf0.1_p1/load.groovy b/regression-test/suites/ssb_sf0.1_p1/load.groovy
index 9057b857f4..5151bff68e 100644
--- a/regression-test/suites/ssb_sf0.1_p1/load.groovy
+++ b/regression-test/suites/ssb_sf0.1_p1/load.groovy
@@ -87,7 +87,7 @@ suite("load") {
def rowCount = sql "select count(*) from ${table}"
if (rowCount[0][0] != table_rows) {
sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
- sql "set global insert_timeout=3600"
+ sql "set insert_timeout=3600"
def r = sql "select @@insert_timeout"
year_cons = [
'lo_orderdate<19930101',
diff --git a/regression-test/suites/ssb_sf100_p2/load.groovy b/regression-test/suites/ssb_sf100_p2/load.groovy
index bcf423f7d4..7d9461cbd5 100644
--- a/regression-test/suites/ssb_sf100_p2/load.groovy
+++ b/regression-test/suites/ssb_sf100_p2/load.groovy
@@ -77,7 +77,7 @@ suite('load') {
def rowCount = sql "select count(*) from ${table}"
if (rowCount[0][0] != table_rows) {
sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
- sql "set global insert_timeout=3600"
+ sql "set insert_timeout=3600"
sql "sync"
def r = sql "select @@insert_timeout"
assertEquals(3600, r[0][0])
diff --git a/regression-test/suites/ssb_sf1_p2/load.groovy b/regression-test/suites/ssb_sf1_p2/load.groovy
index ee0163b641..383844e120 100644
--- a/regression-test/suites/ssb_sf1_p2/load.groovy
+++ b/regression-test/suites/ssb_sf1_p2/load.groovy
@@ -89,7 +89,7 @@ suite("load") {
def rowCount = sql "select count(*) from ${table}"
if (rowCount[0][0] != table_rows) {
sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
- sql "set global insert_timeout=3600"
+ sql "set insert_timeout=3600"
def r = sql "select @@insert_timeout"
assertEquals(3600, r[0][0])
year_cons = [
diff --git a/regression-test/suites/tpcds_sf100_p2/load.groovy b/regression-test/suites/tpcds_sf100_p2/load.groovy
index 24460ecd74..ccc22a1273 100644
--- a/regression-test/suites/tpcds_sf100_p2/load.groovy
+++ b/regression-test/suites/tpcds_sf100_p2/load.groovy
@@ -40,7 +40,7 @@ suite('load') {
// set fe configuration
sql "ADMIN SET FRONTEND CONFIG ('max_bytes_per_broker_scanner' = '161061273600')"
- sql "set global exec_mem_limit=16G;"
+ sql "set exec_mem_limit=16G;"
def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
tables.each { table, rows ->
diff --git a/regression-test/suites/tpcds_sf1_p1/load.groovy b/regression-test/suites/tpcds_sf1_p1/load.groovy
index d1713217af..1b527dcc81 100644
--- a/regression-test/suites/tpcds_sf1_p1/load.groovy
+++ b/regression-test/suites/tpcds_sf1_p1/load.groovy
@@ -55,7 +55,7 @@ suite("load") {
sql new File("""${context.file.parent}/ddl/${table}.sql""").text
}
- sql "set global exec_mem_limit=8G;"
+ sql "set exec_mem_limit=8G;"
for (String tableName in tables) {
streamLoad {
@@ -99,4 +99,4 @@ suite("load") {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/tpcds_sf1_unique_p1/load.groovy b/regression-test/suites/tpcds_sf1_unique_p1/load.groovy
index c06e1ea61a..b48d538fa3 100644
--- a/regression-test/suites/tpcds_sf1_unique_p1/load.groovy
+++ b/regression-test/suites/tpcds_sf1_unique_p1/load.groovy
@@ -94,7 +94,7 @@ suite("load") {
sql new File("""${context.file.parent}/ddl/${table}.sql""").text
}
- sql "set global exec_mem_limit=8G;"
+ sql "set exec_mem_limit=8G;"
for (String tableName in tables) {
streamLoad {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/08: [MemLeak](pipeline) fix mem leak by exchange node in pipeline (#18864)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 233975b6f602baeae11206d96e87daceb97e929a
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Fri Apr 21 09:06:55 2023 +0800
[MemLeak](pipeline) fix mem leak by exchange node in pipeline (#18864)
---
be/src/vec/sink/vdata_stream_sender.h | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/be/src/vec/sink/vdata_stream_sender.h b/be/src/vec/sink/vdata_stream_sender.h
index d0357cdbcc..da7c8cabd7 100644
--- a/be/src/vec/sink/vdata_stream_sender.h
+++ b/be/src/vec/sink/vdata_stream_sender.h
@@ -426,6 +426,9 @@ public:
// rpc (or OK if there wasn't one that hasn't been reported yet).
// if batch is nullptr, send the eof packet
Status send_block(PBlock* block, bool eos = false) override {
+ std::unique_ptr<PBlock> pblock_ptr;
+ pblock_ptr.reset(block);
+
if (eos) {
if (_eos_send) {
return Status::OK();
@@ -434,8 +437,7 @@ public:
}
}
if (eos || block->column_metas_size()) {
- RETURN_IF_ERROR(_buffer->add_block(
- {this, block ? std::make_unique<PBlock>(*block) : nullptr, eos}));
+ RETURN_IF_ERROR(_buffer->add_block({this, std::move(pblock_ptr), eos}));
}
return Status::OK();
}
@@ -460,15 +462,14 @@ public:
return send_local_block(eos);
}
- PBlock* block_ptr = nullptr;
+ auto block_ptr = std::make_unique<PBlock>();
if (_mutable_block) {
- block_ptr = new PBlock(); // TODO: need a pool of PBlock()
auto block = _mutable_block->to_block();
- RETURN_IF_ERROR(_parent->serialize_block(&block, block_ptr));
+ RETURN_IF_ERROR(_parent->serialize_block(&block, block_ptr.get()));
block.clear_column_data();
_mutable_block->set_muatable_columns(block.mutate_columns());
}
- RETURN_IF_ERROR(send_block(block_ptr, eos));
+ RETURN_IF_ERROR(send_block(block_ptr.release(), eos));
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/08: [fix](outfile) unify broker and hdfs path in outfile (#18809)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0ceff0720294bfe5b7c60d7e9475a099b3058ed8
Author: slothever <18...@users.noreply.github.com>
AuthorDate: Thu Apr 20 21:01:39 2023 +0800
[fix](outfile) unify broker and hdfs path in outfile (#18809)
unify broker and hdfs path in outfile
fix fe ut and add outfile case
---
.../org/apache/doris/analysis/OutFileClause.java | 9 +-
.../doris/analysis/OutFileClauseFunctionTest.java | 122 +++++++++++++--------
2 files changed, 87 insertions(+), 44 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index 1ec20fd625..6315fe029c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -548,7 +548,14 @@ public class OutFileClause {
} else {
isLocalOutput = false;
}
-
+ if (properties != null) {
+ String namePrefix = properties.containsKey(PROP_BROKER_NAME)
+ ? BROKER_PROP_PREFIX + HdfsResource.DSF_NAMESERVICES : HdfsResource.DSF_NAMESERVICES;
+ String dfsNameServices = properties.getOrDefault(namePrefix, "");
+ if (!Strings.isNullOrEmpty(dfsNameServices) && !filePath.contains(dfsNameServices)) {
+ filePath = filePath.replace(HDFS_FILE_PREFIX, HDFS_FILE_PREFIX + dfsNameServices);
+ }
+ }
if (Strings.isNullOrEmpty(filePath)) {
throw new AnalysisException("Must specify file in OUTFILE clause");
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFunctionTest.java
index ffbc6cb8d2..c54c089ed1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFunctionTest.java
@@ -17,83 +17,119 @@
package org.apache.doris.analysis;
-import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.jmockit.Deencapsulation;
-import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TFileFormatType;
-import org.apache.doris.utframe.UtFrameUtils;
+import org.apache.doris.utframe.TestWithFeService;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.UUID;
-
-public class OutFileClauseFunctionTest {
-
- private static String baseDir = "fe";
- private static String runningDir = baseDir + "/mocked/MaterializedViewFunctionTest/"
- + UUID.randomUUID().toString() + "/";
- private static ConnectContext ctx;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+public class OutFileClauseFunctionTest extends TestWithFeService {
private static final String DB_NAME = "db1";
- @BeforeClass
- public static void beforeClass() throws Exception {
+ @Override
+ protected void runBeforeAll() throws Exception {
FeConstants.default_scheduler_interval_millisecond = 10;
FeConstants.runningUnitTest = true;
Config.enable_outfile_to_local = true;
- ctx = UtFrameUtils.createDefaultCtx();
- UtFrameUtils.createDorisCluster(runningDir);
- CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt("create database db1;", ctx);
- Env.getCurrentEnv().createDb(createDbStmt);
+ connectContext = createDefaultCtx();
+ createDatabase(DB_NAME);
+ useDatabase(DB_NAME);
String createTableSQL = "create table " + DB_NAME
+ ".test (k1 int, k2 varchar ) "
+ "distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
- CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTableSQL, ctx);
- Env.getCurrentEnv().createTable(createTableStmt);
- }
-
- @AfterClass
- public static void tearDown() {
- File file = new File(runningDir);
- file.delete();
+ createTable(createTableSQL);
}
@Test
public void testSelectStmtOutFileClause() throws Exception {
String query1 = "select * from db1.test into outfile \"file:///" + runningDir + "/result_\";";
- QueryStmt analyzedQueryStmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(query1, ctx);
- Assert.assertTrue(analyzedQueryStmt.hasOutFileClause());
+ QueryStmt analyzedQueryStmt = createStmt(query1);
+ Assertions.assertTrue(analyzedQueryStmt.hasOutFileClause());
OutFileClause outFileClause = analyzedQueryStmt.getOutFileClause();
boolean isOutFileClauseAnalyzed = Deencapsulation.getField(outFileClause, "isAnalyzed");
- Assert.assertTrue(isOutFileClauseAnalyzed);
- Assert.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_CSV_PLAIN);
+ Assertions.assertTrue(isOutFileClauseAnalyzed);
+ Assertions.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_CSV_PLAIN);
}
@Test
public void testSetOperationStmtOutFileClause() throws Exception {
String query1 = "select * from db1.test union select * from db1.test into outfile \"file:///" + runningDir + "/result_\";";
- QueryStmt analyzedSetOperationStmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(query1, ctx);
- Assert.assertTrue(analyzedSetOperationStmt.hasOutFileClause());
+ QueryStmt analyzedSetOperationStmt = createStmt(query1);
+ Assertions.assertTrue(analyzedSetOperationStmt.hasOutFileClause());
OutFileClause outFileClause = analyzedSetOperationStmt.getOutFileClause();
boolean isOutFileClauseAnalyzed = Deencapsulation.getField(outFileClause, "isAnalyzed");
- Assert.assertTrue(isOutFileClauseAnalyzed);
- Assert.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_CSV_PLAIN);
+ Assertions.assertTrue(isOutFileClauseAnalyzed);
+ Assertions.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_CSV_PLAIN);
}
@Test
public void testParquetFormat() throws Exception {
String query1 = "select * from db1.test union select * from db1.test into outfile \"file:///" + runningDir + "/result_\" FORMAT AS PARQUET;";
- QueryStmt analyzedSetOperationStmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(query1, ctx);
- Assert.assertTrue(analyzedSetOperationStmt.hasOutFileClause());
+ QueryStmt analyzedSetOperationStmt = createStmt(query1);
+ Assertions.assertTrue(analyzedSetOperationStmt.hasOutFileClause());
OutFileClause outFileClause = analyzedSetOperationStmt.getOutFileClause();
boolean isOutFileClauseAnalyzed = Deencapsulation.getField(outFileClause, "isAnalyzed");
- Assert.assertTrue(isOutFileClauseAnalyzed);
- Assert.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_PARQUET);
+ Assertions.assertTrue(isOutFileClauseAnalyzed);
+ Assertions.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_PARQUET);
+ }
+
+ @Test
+ public void testHdfsFile() throws Exception {
+ String loc1 = "'hdfs://hacluster/data/test/'";
+ String loc2 = "'hdfs:///data/test/'";
+
+ String query1 = "select * from db1.test \n"
+ + "into outfile "
+ + loc1
+ + "\n"
+ + "format as csv\n"
+ + "properties(\n"
+ + "'column_separator' = ',',\n"
+ + "'line_delimiter' = '\\n',\n"
+ + "'broker.name' = 'broker',\n"
+ + "'broker.fs.defaultFS'='hdfs://hacluster/',\n"
+ + "'broker.dfs.nameservices'='hacluster',\n"
+ + "'broker.dfs.ha.namenodes.hacluster'='n1,n2',\n"
+ + "'broker.dfs.namenode.rpc-address.hacluster.n1'='master1:8020',\n"
+ + "'broker.dfs.namenode.rpc-address.hacluster.n2'='master2:8020',\n"
+ + "'broker.dfs.client.failover.proxy.provider.hacluster'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'"
+ + ");";
+ QueryStmt analyzedOutStmt = createStmt(query1);
+ Assertions.assertTrue(analyzedOutStmt.hasOutFileClause());
+
+ OutFileClause outFileClause = analyzedOutStmt.getOutFileClause();
+ boolean isOutFileClauseAnalyzed = Deencapsulation.getField(outFileClause, "isAnalyzed");
+ Assertions.assertTrue(isOutFileClauseAnalyzed);
+
+ QueryStmt analyzedOutStmtLoc2 = createStmt(query1.replace(loc1, loc2));
+ Assertions.assertTrue(analyzedOutStmtLoc2.hasOutFileClause());
+
+ String query2 = "select * from db1.test \n"
+ + "into outfile "
+ + loc1
+ + "\n"
+ + "format as csv\n"
+ + "properties(\n"
+ + "'column_separator' = ',',\n"
+ + "'line_delimiter' = '\\n',\n"
+ + "'fs.defaultFS'='hdfs://hacluster/',\n"
+ + "'dfs.nameservices'='hacluster',\n"
+ + "'dfs.ha.namenodes.hacluster'='n1,n2',\n"
+ + "'dfs.namenode.rpc-address.hacluster.n1'='master1:8020',\n"
+ + "'dfs.namenode.rpc-address.hacluster.n2'='master2:8020',\n"
+ + "'dfs.client.failover.proxy.provider.hacluster'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'"
+ + ");";
+ QueryStmt analyzedOutStmt2 = createStmt(query2);
+ Assertions.assertTrue(analyzedOutStmt2.hasOutFileClause());
+
+ OutFileClause outFileClause2 = analyzedOutStmt2.getOutFileClause();
+ boolean isOutFileClauseAnalyzed2 = Deencapsulation.getField(outFileClause2, "isAnalyzed");
+ Assertions.assertTrue(isOutFileClauseAnalyzed2);
+
+ QueryStmt analyzedOutStmt2Loc2 = createStmt(query2.replace(loc1, loc2));
+ Assertions.assertTrue(analyzedOutStmt2Loc2.hasOutFileClause());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/08: [Fix](dynamic table) fix dynamic table with insert into and column al… (#18808)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7d2a96468b251e55fc5264bad5986c4158722a23
Author: lihangyu <15...@163.com>
AuthorDate: Fri Apr 21 11:19:00 2023 +0800
[Fix](dynamic table) fix dynamic table with insert into and column al… (#18808)
1. The num_rows should be correctly set
2. insert into has no dynamic column
---
be/src/olap/memtable.cpp | 11 +++++---
be/src/vec/columns/column_object.cpp | 15 ++++++-----
be/src/vec/columns/column_object.h | 30 ++++++++++++++++++++++
.../suites/dynamic_table_p0/load.groovy | 4 +--
4 files changed, 48 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index db18782dc3..2b4bb79324 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -393,12 +393,14 @@ void MemTable::unfold_variant_column(vectorized::Block& block) {
if (block.rows() == 0) {
return;
}
- vectorized::ColumnWithTypeAndName variant_column =
- block.get_by_name(BeConsts::DYNAMIC_COLUMN_NAME);
+ vectorized::ColumnWithTypeAndName* variant_column =
+ block.try_get_by_name(BeConsts::DYNAMIC_COLUMN_NAME);
+ if (!variant_column) {
+ return;
+ }
// remove it
- block.erase(BeConsts::DYNAMIC_COLUMN_NAME);
vectorized::ColumnObject& object_column =
- assert_cast<vectorized::ColumnObject&>(variant_column.column->assume_mutable_ref());
+ assert_cast<vectorized::ColumnObject&>(variant_column->column->assume_mutable_ref());
// extend
for (auto& entry : object_column.get_subcolumns()) {
if (entry->path.get_path() == vectorized::ColumnObject::COLUMN_NAME_DUMMY) {
@@ -407,6 +409,7 @@ void MemTable::unfold_variant_column(vectorized::Block& block) {
block.insert({entry->data.get_finalized_column().get_ptr(),
entry->data.get_least_common_type(), entry->path.get_path()});
}
+ block.erase(BeConsts::DYNAMIC_COLUMN_NAME);
}
void MemTable::serialize_block_to_row_column(vectorized::Block& block) {
diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index 1af9f3f630..44acacd535 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -516,6 +516,12 @@ const ColumnPtr& ColumnObject::Subcolumn::get_finalized_column_ptr() const {
return data[0];
}
+void ColumnObject::Subcolumn::remove_nullable() {
+ assert(is_finalized());
+ data[0] = doris::vectorized::remove_nullable(data[0]);
+ least_common_type.remove_nullable();
+}
+
ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_)
: type(std::move(type_)),
base_type(getBaseTypeOfArray(type)),
@@ -903,7 +909,6 @@ void align_variant_by_name_and_type(ColumnObject& dst, const ColumnObject& src,
// if src and dst is empty, we just increase the num_rows of dst and fill
// num_rows of default values when meet new data
size_t num_rows = dst.rows();
- bool need_inc_row_num = true;
for (auto& entry : dst.get_subcolumns()) {
const auto* src_subcol = src.get_subcolumn(entry->path);
if (src_subcol == nullptr) {
@@ -915,6 +920,7 @@ void align_variant_by_name_and_type(ColumnObject& dst, const ColumnObject& src,
const auto& src_column = src_subcol->get_finalized_column();
inserter(src_column, &entry->data.get_finalized_column());
}
+ dst.set_num_rows(entry->data.get_finalized_column().size());
}
for (const auto& entry : src.get_subcolumns()) {
// encounter a new column
@@ -924,15 +930,12 @@ void align_variant_by_name_and_type(ColumnObject& dst, const ColumnObject& src,
auto new_column = type->create_column();
new_column->insert_many_defaults(num_rows);
inserter(entry->data.get_finalized_column(), new_column.get());
- if (dst.empty()) {
- // add_sub_column updated num_rows of dst object
- need_inc_row_num = false;
- }
+ dst.set_num_rows(new_column->size());
dst.add_sub_column(entry->path, std::move(new_column));
}
}
num_rows += row_cnt;
- if (need_inc_row_num) {
+ if (dst.empty()) {
dst.incr_num_rows(row_cnt);
}
#ifndef NDEBUG
diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h
index af5cb5cc9a..290ec0ec1e 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -29,6 +29,24 @@
#include <vec/json/json_parser.h>
#include "common/status.h"
+#include "vec/columns/column.h"
+#include "vec/columns/subcolumn_tree.h"
+#include "vec/common/cow.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/json/path_in_data.h"
+
+class SipHash;
+
+namespace doris {
+namespace vectorized {
+class Arena;
+} // namespace vectorized
+} // namespace doris
+
namespace doris::vectorized {
/// Info that represents a scalar or array field in a decomposed view.
@@ -120,6 +138,8 @@ public:
const ColumnPtr& get_finalized_column_ptr() const;
+ void remove_nullable();
+
friend class ColumnObject;
private:
@@ -135,6 +155,8 @@ public:
size_t get_dimensions() const { return num_dimensions; }
+ void remove_nullable() { type = doris::vectorized::remove_nullable(type); }
+
private:
DataTypePtr type;
DataTypePtr base_type;
@@ -183,6 +205,12 @@ public:
// return null if not found
const Subcolumn* get_subcolumn(const PathInData& key) const;
+ /** More efficient methods of manipulation */
+ [[noreturn]] IColumn& get_data() { LOG(FATAL) << "Not implemented method get_data()"; }
+ [[noreturn]] const IColumn& get_data() const {
+ LOG(FATAL) << "Not implemented method get_data()";
+ }
+
// return null if not found
Subcolumn* get_subcolumn(const PathInData& key);
@@ -190,6 +218,8 @@ public:
void incr_num_rows(size_t n) { num_rows += n; }
+ void set_num_rows(size_t n) { num_rows = n; }
+
size_t rows() const { return num_rows; }
/// Adds a subcolumn from existing IColumn.
diff --git a/regression-test/suites/dynamic_table_p0/load.groovy b/regression-test/suites/dynamic_table_p0/load.groovy
index f4bd89aba6..4837ca3cd1 100644
--- a/regression-test/suites/dynamic_table_p0/load.groovy
+++ b/regression-test/suites/dynamic_table_p0/load.groovy
@@ -123,8 +123,8 @@ suite("regression_test_dynamic_table", "dynamic_table"){
json_load_unique("btc_transactions.json", "test_btc_json")
json_load_unique("ghdata_sample.json", "test_ghdata_json")
json_load_unique("nbagames_sample.json", "test_nbagames_json")
- // sql """insert into test_ghdata_json_unique select * from test_ghdata_json_unique"""
- // sql """insert into test_btc_json_unique select * from test_btc_json_unique"""
+ sql """insert into test_ghdata_json_unique select * from test_ghdata_json"""
+ sql """insert into test_btc_json_unique select * from test_btc_json"""
// load more
table_name = "gharchive";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/08: [fix](editLog) add sufficient replay logic and edit log for altering light schema change (#18746)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit de36bf3c074d602021f78c5803d5ca8edab861bb
Author: 奕冷 <82...@users.noreply.github.com>
AuthorDate: Thu Apr 20 19:20:03 2023 +0800
[fix](editLog) add sufficient replay logic and edit log for altering light schema change (#18746)
---
...CHelper.java => AlterLightSchChangeHelper.java} | 52 ++++++++---------
.../apache/doris/alter/SchemaChangeHandler.java | 19 +++++-
.../org/apache/doris/catalog/TableProperty.java | 3 -
.../org/apache/doris/journal/JournalEntity.java | 7 ++-
.../doris/persist/AlterLightSchemaChangeInfo.java | 68 ++++++++++++++++++++++
.../java/org/apache/doris/persist/EditLog.java | 8 ++-
6 files changed, 123 insertions(+), 34 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterLSCHelper.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterLightSchChangeHelper.java
similarity index 87%
rename from fe/fe-core/src/main/java/org/apache/doris/alter/AlterLSCHelper.java
rename to fe/fe-core/src/main/java/org/apache/doris/alter/AlterLightSchChangeHelper.java
index 1b808e10ea..d643aec58b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterLSCHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterLightSchChangeHelper.java
@@ -28,7 +28,7 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
-import org.apache.doris.persist.ModifyTablePropertyOperationLog;
+import org.apache.doris.persist.AlterLightSchemaChangeInfo;
import org.apache.doris.proto.InternalService.PFetchColIdsRequest;
import org.apache.doris.proto.InternalService.PFetchColIdsRequest.Builder;
import org.apache.doris.proto.InternalService.PFetchColIdsRequest.PFetchColIdParam;
@@ -42,6 +42,8 @@ import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TStatusCode;
import com.google.common.base.Preconditions;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
import java.io.IOException;
import java.util.ArrayList;
@@ -59,13 +61,15 @@ import java.util.concurrent.TimeoutException;
/**
* For alter light_schema_change table property
*/
-public class AlterLSCHelper {
+public class AlterLightSchChangeHelper {
+
+ private static final Logger LOG = LogManager.getLogger(AlterLightSchChangeHelper.class);
private final Database db;
private final OlapTable olapTable;
- public AlterLSCHelper(Database db, OlapTable olapTable) {
+ public AlterLightSchChangeHelper(Database db, OlapTable olapTable) {
this.db = db;
this.olapTable = olapTable;
}
@@ -77,9 +81,10 @@ public class AlterLSCHelper {
*/
public void enableLightSchemaChange() throws DdlException {
final Map<Long, PFetchColIdsRequest> params = initParams();
- final PFetchColIdsResponse response = callForColumnIds(params);
- updateTableMeta(response);
- modifyTableProperty();
+ final AlterLightSchemaChangeInfo info = callForColumnsInfo(params);
+ updateTableMeta(info);
+ Env.getCurrentEnv().getEditLog().logAlterLightSchemaChange(info);
+ LOG.info("successfully enable `light_schema_change`");
}
/**
@@ -133,10 +138,12 @@ public class AlterLSCHelper {
/**
* @param beIdToRequest rpc param for corresponding BEs
- * @return indexIds to each tablet schema info which consists of columnName to corresponding column unique id pairs
+ * @return contains indexIds to each tablet schema info which consists of columnName to corresponding
+ * column unique id pairs
* @throws DdlException as a wrapper for rpc failures
*/
- private PFetchColIdsResponse callForColumnIds(Map<Long, PFetchColIdsRequest> beIdToRequest) throws DdlException {
+ private AlterLightSchemaChangeInfo callForColumnsInfo(Map<Long, PFetchColIdsRequest> beIdToRequest)
+ throws DdlException {
final List<Future<PFetchColIdsResponse>> futureList = new ArrayList<>();
// start a rpc in a pipeline way
try {
@@ -173,14 +180,14 @@ public class AlterLSCHelper {
} catch (TimeoutException e) {
throw new DdlException("fetch columnIds RPC result timeout", e);
}
- return compactToUniqResp(resultList);
+ return compactToAlterLscInfo(resultList);
}
/**
* Since the result collected from several BEs may contain repeated indexes in distributed storage scenarios,
* we should do consistency check for the result for the same index, and get the unique result.
*/
- private PFetchColIdsResponse compactToUniqResp(List<PFetchColIdsResponse> resultList) {
+ private AlterLightSchemaChangeInfo compactToAlterLscInfo(List<PFetchColIdsResponse> resultList) {
final PFetchColIdsResponse.Builder builder = PFetchColIdsResponse.newBuilder();
Map<Long, Map<String, Integer>> indexIdToTabletInfo = new HashMap<>();
resultList.forEach(response -> {
@@ -197,27 +204,25 @@ public class AlterLSCHelper {
"index: " + indexId + "got inconsistent schema in storage");
}
});
- return builder.build();
+ return new AlterLightSchemaChangeInfo(db.getId(), olapTable.getId(), indexIdToTabletInfo);
}
- private void updateTableMeta(PFetchColIdsResponse response) throws DdlException {
- Preconditions.checkState(response.isInitialized());
+ public void updateTableMeta(AlterLightSchemaChangeInfo info) throws DdlException {
+ Preconditions.checkNotNull(info, "passed in info should be not null");
// update index-meta once and for all
// schema pair: <maxColId, columns>
final List<Pair<Integer, List<Column>>> schemaPairs = new ArrayList<>();
final List<Long> indexIds = new ArrayList<>();
- response.getEntriesList().forEach(entry -> {
- final long indexId = entry.getIndexId();
+ info.getIndexIdToColumnInfo().forEach((indexId, colNameToId) -> {
final List<Column> columns = olapTable.getSchemaByIndexId(indexId, true);
- final Map<String, Integer> colNameToId = entry.getColNameToIdMap();
Preconditions.checkState(columns.size() == colNameToId.size(),
- "size mismatch for columns meta from BE");
+ "size mismatch for original columns meta and that in change info");
int maxColId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
final List<Column> newSchema = new ArrayList<>();
for (Column column : columns) {
final String columnName = column.getName();
final int columnId = Preconditions.checkNotNull(colNameToId.get(columnName),
- "failed to fetch column id of column:{" + columnName + "} from BE");
+ "failed to fetch column id of column:{" + columnName + "}");
final Column newColumn = new Column(column);
newColumn.setUniqueId(columnId);
newSchema.add(newColumn);
@@ -226,7 +231,8 @@ public class AlterLSCHelper {
schemaPairs.add(Pair.of(maxColId, newSchema));
indexIds.add(indexId);
});
- Preconditions.checkState(schemaPairs.size() == indexIds.size());
+ Preconditions.checkState(schemaPairs.size() == indexIds.size(),
+ "impossible state, size of schemaPairs and indexIds should be the same");
// update index-meta once and for all
try {
for (int i = 0; i < indexIds.size(); i++) {
@@ -238,14 +244,8 @@ public class AlterLSCHelper {
} catch (IOException e) {
throw new DdlException("fail to reset index schema", e);
}
- }
-
- private void modifyTableProperty() {
// write table property
olapTable.setEnableLightSchemaChange(true);
- //write edit log
- ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), olapTable.getId(),
- olapTable.getTableProperty().getProperties());
- Env.getCurrentEnv().getEditLog().logAlterLightSchemaChange(info);
+ LOG.info("successfully update table meta for `light_schema_change`");
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index ffce0bbb9b..c9ce361e3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -73,6 +73,7 @@ import org.apache.doris.common.util.ListComparator;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.persist.AlterLightSchemaChangeInfo;
import org.apache.doris.persist.RemoveAlterJobV2OperationLog;
import org.apache.doris.persist.TableAddOrDropColumnsInfo;
import org.apache.doris.persist.TableAddOrDropInvertedIndicesInfo;
@@ -1890,8 +1891,22 @@ public class SchemaChangeHandler extends AlterHandler {
private void enableLightSchemaChange(Database db, OlapTable olapTable) throws DdlException {
- final AlterLSCHelper alterLSCHelper = new AlterLSCHelper(db, olapTable);
- alterLSCHelper.enableLightSchemaChange();
+ final AlterLightSchChangeHelper alterLightSchChangeHelper = new AlterLightSchChangeHelper(db, olapTable);
+ alterLightSchChangeHelper.enableLightSchemaChange();
+ }
+
+ public void replayAlterLightSchChange(AlterLightSchemaChangeInfo info) throws MetaNotFoundException {
+ Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrMetaException(info.getDbId());
+ OlapTable olapTable = (OlapTable) db.getTableOrMetaException(info.getTableId(), TableType.OLAP);
+ olapTable.writeLock();
+ final AlterLightSchChangeHelper alterLightSchChangeHelper = new AlterLightSchChangeHelper(db, olapTable);
+ try {
+ alterLightSchChangeHelper.updateTableMeta(info);
+ } catch (DdlException e) {
+ LOG.warn("failed to replay alter light schema change", e);
+ } finally {
+ olapTable.writeUnlock();
+ }
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index add5162d75..4078b7473d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -107,9 +107,6 @@ public class TableProperty implements Writable {
buildInMemory();
buildStoragePolicy();
break;
- case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE:
- buildEnableLightSchemaChange();
- break;
default:
break;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index ccbbd764a3..2162fc9702 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -61,6 +61,7 @@ import org.apache.doris.mtmv.metadata.DropMTMVTask;
import org.apache.doris.mtmv.metadata.MTMVJob;
import org.apache.doris.mtmv.metadata.MTMVTask;
import org.apache.doris.mysql.privilege.UserPropertyInfo;
+import org.apache.doris.persist.AlterLightSchemaChangeInfo;
import org.apache.doris.persist.AlterMultiMaterializedView;
import org.apache.doris.persist.AlterRoutineLoadJobOperationLog;
import org.apache.doris.persist.AlterUserOperationLog;
@@ -625,7 +626,6 @@ public class JournalEntity implements Writable {
}
case OperationType.OP_DYNAMIC_PARTITION:
case OperationType.OP_MODIFY_IN_MEMORY:
- case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE:
case OperationType.OP_MODIFY_REPLICATION_NUM: {
data = ModifyTablePropertyOperationLog.read(in);
isRead = true;
@@ -808,6 +808,11 @@ public class JournalEntity implements Writable {
isRead = true;
break;
}
+ case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE: {
+ data = AlterLightSchemaChangeInfo.read(in);
+ isRead = true;
+ break;
+ }
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}", opCode, e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterLightSchemaChangeInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterLightSchemaChangeInfo.java
new file mode 100644
index 0000000000..e76ebe9d64
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterLightSchemaChangeInfo.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Map;
+
+public class AlterLightSchemaChangeInfo implements Writable {
+
+ @SerializedName(value = "dbId")
+ private Long dbId;
+
+ @SerializedName(value = "tableId")
+ private Long tableId;
+
+ @SerializedName("indexIdToColumnInfo")
+ private Map<Long, Map<String, Integer>> indexIdToColumnInfo;
+
+ public AlterLightSchemaChangeInfo(long dbId, long tableId, Map<Long, Map<String, Integer>> info) {
+ this.dbId = dbId;
+ this.tableId = tableId;
+ this.indexIdToColumnInfo = info;
+ }
+
+ public Long getDbId() {
+ return dbId;
+ }
+
+ public Long getTableId() {
+ return tableId;
+ }
+
+ public Map<Long, Map<String, Integer>> getIndexIdToColumnInfo() {
+ return indexIdToColumnInfo;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
+ }
+
+ public static AlterLightSchemaChangeInfo read(DataInput in) throws IOException {
+ return GsonUtils.GSON.fromJson(Text.readString(in), AlterLightSchemaChangeInfo.class);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index fc837efd70..63d3f348aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -765,7 +765,6 @@ public class EditLog {
}
case OperationType.OP_DYNAMIC_PARTITION:
case OperationType.OP_MODIFY_IN_MEMORY:
- case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE:
case OperationType.OP_MODIFY_REPLICATION_NUM: {
ModifyTablePropertyOperationLog log = (ModifyTablePropertyOperationLog) journal.getData();
env.replayModifyTableProperty(opCode, log);
@@ -897,6 +896,11 @@ public class EditLog {
env.getSchemaChangeHandler().replayModifyTableLightSchemaChange(info);
break;
}
+ case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE: {
+ final AlterLightSchemaChangeInfo info = (AlterLightSchemaChangeInfo) journal.getData();
+ env.getSchemaChangeHandler().replayAlterLightSchChange(info);
+ break;
+ }
case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_INVERTED_INDICES: {
final TableAddOrDropInvertedIndicesInfo info =
(TableAddOrDropInvertedIndicesInfo) journal.getData();
@@ -1607,7 +1611,7 @@ public class EditLog {
logEdit(OperationType.OP_MODIFY_IN_MEMORY, info);
}
- public void logAlterLightSchemaChange(ModifyTablePropertyOperationLog info) {
+ public void logAlterLightSchemaChange(AlterLightSchemaChangeInfo info) {
logEdit(OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE, info);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/08: [Fix](Nereids) Check bound status in analyze straight after bounding (#18581)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cf3f04567d3ec74e00532f50f75d3d39989a0f97
Author: LiBinfeng <46...@users.noreply.github.com>
AuthorDate: Thu Apr 20 18:50:13 2023 +0800
[Fix](Nereids) Check bound status in analyze straight after bounding (#18581)
Probleam:
Dead loop cause of keep pushing analyze tasks into job stack. When doing analyze process and generate new operators, the same analyze rule would be pushed again, so it cause dead loop. And analyze process generate new operators when trying to bound order by key and aggregate function.
Solve:
We need to make it throw exception before complex analyze and rewrite process, so checking whether all expressions being bound should be done twice. One is done after bounding all expression, another is done after all analyze process in case of generate new expressions and new operators.
Example:
Cases were put in file: regression-test/suites/nereids_p0/except/test_bound_exception.groovy
---
.../doris/nereids/analyzer/NereidsAnalyzer.java | 4 ++
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../nereids/rules/analysis/CheckAnalysis.java | 24 +++----
.../doris/nereids/rules/analysis/CheckBound.java | 74 ++++++++++++++++++++++
.../rules/analysis/BindSlotReferenceTest.java | 2 +-
.../nereids_p0/except/test_bound_exception.groovy | 55 ++++++++++++++++
6 files changed, 148 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
index 38d75e2a01..2413e775c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySe
import org.apache.doris.nereids.rules.analysis.BindExpression;
import org.apache.doris.nereids.rules.analysis.BindRelation;
import org.apache.doris.nereids.rules.analysis.CheckAnalysis;
+import org.apache.doris.nereids.rules.analysis.CheckBound;
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
import org.apache.doris.nereids.rules.analysis.NormalizeRepeat;
@@ -53,6 +54,9 @@ public class NereidsAnalyzer extends BatchRewriteJob {
new UserAuthentication(),
new BindExpression()
),
+ bottomUp(
+ new CheckBound()
+ ),
bottomUp(
new ProjectToGlobalAggregate(),
// this rule check's the logicalProject node's isDisinct property
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 9406267fc4..1397c10153 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -85,6 +85,7 @@ public enum RuleType {
// check analysis rule
CHECK_AGGREGATE_ANALYSIS(RuleTypeClass.CHECK),
CHECK_ANALYSIS(RuleTypeClass.CHECK),
+ CHECK_BOUND(RuleTypeClass.CHECK),
CHECK_DATATYPES(RuleTypeClass.CHECK),
// rewrite rules
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
index 8d45087b84..c16f7ad460 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
@@ -78,17 +78,19 @@ public class CheckAnalysis implements AnalysisRuleFactory {
.flatMap(Set::stream)
.collect(Collectors.toSet());
if (!unbounds.isEmpty()) {
- throw new AnalysisException(String.format("unbounded object %s.",
- StringUtils.join(unbounds.stream()
- .map(unbound -> {
- if (unbound instanceof UnboundSlot) {
- return ((UnboundSlot) unbound).toSql();
- } else if (unbound instanceof UnboundFunction) {
- return ((UnboundFunction) unbound).toSql();
- }
- return unbound.toString();
- })
- .collect(Collectors.toSet()), ", ")));
+ throw new AnalysisException(String.format("unbounded object %s in %s clause.",
+ StringUtils.join(unbounds.stream()
+ .map(unbound -> {
+ if (unbound instanceof UnboundSlot) {
+ return ((UnboundSlot) unbound).toSql();
+ } else if (unbound instanceof UnboundFunction) {
+ return ((UnboundFunction) unbound).toSql();
+ }
+ return unbound.toString();
+ })
+ .collect(Collectors.toSet()), ", "),
+ plan.getType().toString().substring("LOGICAL_".length())
+ ));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckBound.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckBound.java
new file mode 100644
index 0000000000..e04ff2c680
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckBound.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.analysis;
+
+import org.apache.doris.nereids.analyzer.Unbound;
+import org.apache.doris.nereids.analyzer.UnboundFunction;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Check bound rule to check semantic correct after bounding of expression by Nereids.
+ * Also give operator information without LOGICAL_
+ */
+public class CheckBound implements AnalysisRuleFactory {
+
+ @Override
+ public List<Rule> buildRules() {
+ return ImmutableList.of(
+ RuleType.CHECK_BOUND.build(
+ any().then(plan -> {
+ checkBound(plan);
+ return null;
+ })
+ )
+ );
+ }
+
+ private void checkBound(Plan plan) {
+ Set<Unbound> unbounds = plan.getExpressions().stream()
+ .<Set<Unbound>>map(e -> e.collect(Unbound.class::isInstance))
+ .flatMap(Set::stream)
+ .collect(Collectors.toSet());
+ if (!unbounds.isEmpty()) {
+ throw new AnalysisException(String.format("unbounded object %s in %s clause.",
+ StringUtils.join(unbounds.stream()
+ .map(unbound -> {
+ if (unbound instanceof UnboundSlot) {
+ return ((UnboundSlot) unbound).toSql();
+ } else if (unbound instanceof UnboundFunction) {
+ return ((UnboundFunction) unbound).toSql();
+ }
+ return unbound.toString();
+ })
+ .collect(Collectors.toSet()), ", "),
+ plan.getType().toString().substring("LOGICAL_".length())
+ ));
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java
index ad8ca7bbc5..5ddb7f5643 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java
@@ -54,7 +54,7 @@ class BindSlotReferenceTest {
new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student));
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
() -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze(project));
- Assertions.assertEquals("unbounded object foo.", exception.getMessage());
+ Assertions.assertEquals("unbounded object foo in PROJECT clause.", exception.getMessage());
}
@Test
diff --git a/regression-test/suites/nereids_p0/except/test_bound_exception.groovy b/regression-test/suites/nereids_p0/except/test_bound_exception.groovy
new file mode 100644
index 0000000000..95479f009d
--- /dev/null
+++ b/regression-test/suites/nereids_p0/except/test_bound_exception.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_bound_exception") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ def tbName = "test_bound_exception"
+ def dbName = "test_bound_db"
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+ sql "USE ${dbName}"
+
+ sql """ DROP TABLE IF EXISTS ${tbName} """
+ sql """
+ create table if not exists ${tbName} (id int, name char(10))
+ distributed by hash(id) buckets 1 properties("replication_num"="1");
+ """
+ test {
+ sql "SELECT id FROM ${tbName} GROUP BY id ORDER BY id123"
+ exception "errCode = 2, detailMessage = Unexpected exception: unbounded object id123 in SORT clause."
+ }
+ test {
+ sql "SELECT id123 FROM ${tbName} ORDER BY id"
+ exception "errCode = 2, detailMessage = Unexpected exception: unbounded object id123 in PROJECT clause."
+ }
+ test {
+ sql "SELECT id123 FROM ${tbName} GROUP BY id ORDER BY id"
+ exception "errCode = 2, detailMessage = Unexpected exception: unbounded object id123 in AGGREGATE clause."
+ }
+ test {
+ sql "SELECT id FROM ${tbName} GROUP BY id123 ORDER BY id"
+ exception "errCode = 2, detailMessage = Unexpected exception: cannot bind GROUP BY KEY: id123"
+ }
+ test {
+ sql "SELECT id FROM ${tbName} WHERE id = (SELECT id from ${tbName} ORDER BY id123 LIMIT 1) ORDER BY id"
+ exception "errCode = 2, detailMessage = Unexpected exception: unbounded object id123 in SORT clause."
+ }
+ test {
+ sql "SELECT id FROM ${tbName} WHERE id123 = 123 ORDER BY id"
+ exception "errCode = 2, detailMessage = Unexpected exception: Invalid call to dataType on unbound object"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/08: [chore](cold_heat) fix cold heat case to use correct http api (#18870)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1dc2011da663379f9787d15c7249db7869401ab0
Author: AlexYue <yj...@gmail.com>
AuthorDate: Fri Apr 21 10:43:52 2023 +0800
[chore](cold_heat) fix cold heat case to use correct http api (#18870)
---
.../create_table_use_partition_policy.groovy | 12 ++++++------
.../cold_heat_separation_p2/create_table_use_policy.groovy | 2 +-
.../modify_replica_use_partition.groovy | 4 ++--
.../table_modify_resouce_and_policy.groovy | 2 +-
4 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy.groovy
index 983c00f768..915443fb5d 100644
--- a/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy.groovy
+++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy.groovy
@@ -43,7 +43,7 @@ suite("create_table_use_partition_policy") {
data_sizes[0] = obj.local_data_size
data_sizes[1] = obj.remote_data_size
}
- fetchBeHttp(clos, meta_url)
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
}
// used as passing out parameter to fetchDataSize
List<Long> sizes = [-1, -1]
@@ -247,7 +247,7 @@ suite("create_table_use_partition_policy") {
"""
log.info( "test tablets not empty")
assertTrue(tablets.size() > 0)
- fetchDataSize(sizes, tbalets[0])
+ fetchDataSize(sizes, tablets[0])
// while (tablets[0][8] == 0) {
// log.info( "test local size is zero, sleep 10s")
// sleep(10000)
@@ -257,8 +257,8 @@ suite("create_table_use_partition_policy") {
// }
LocalDataSize1 = sizes[0]
RemoteDataSize1 = sizes[1]
- log.info( "test local size not zero")
- assertTrue(LocalDataSize1 != 0)
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
log.info( "test remote size is zero")
assertEquals(RemoteDataSize1, 0)
@@ -359,8 +359,8 @@ suite("create_table_use_partition_policy") {
assertTrue(tablets.size() > 0)
LocalDataSize1 = sizes[0]
RemoteDataSize1 = sizes[1]
- log.info( "test local size not zero")
- assertTrue(LocalDataSize1 != 0)
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
log.info( "test remote size is zero")
assertEquals(RemoteDataSize1, 0)
diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_policy.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_policy.groovy
index eb52dae2bb..7c9d5d794d 100644
--- a/regression-test/suites/cold_heat_separation_p2/create_table_use_policy.groovy
+++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_policy.groovy
@@ -43,7 +43,7 @@ suite("create_table_use_policy") {
data_sizes[0] = obj.local_data_size
data_sizes[1] = obj.remote_data_size
}
- fetchBeHttp(clos, meta_url)
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
}
// used as passing out parameter to fetchDataSize
List<Long> sizes = [-1, -1]
diff --git a/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy
index 8fad952360..4ef34f25a5 100644
--- a/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy
+++ b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy
@@ -43,7 +43,7 @@ suite("modify_replica_use_partition") {
data_sizes[0] = obj.local_data_size
data_sizes[1] = obj.remote_data_size
}
- fetchBeHttp(clos, meta_url)
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
}
// used as passing out parameter to fetchDataSize
List<Long> sizes = [-1, -1]
@@ -401,7 +401,7 @@ suite("modify_replica_use_partition") {
PARTITION BY RANGE(`L_SHIPDATE`)
(
PARTITION `p202301` VALUES LESS THAN ("2017-02-01") ("replication_num" = "3"),
- PARTITION `p202302` VALUES LESS THAN ("2017-03-01") ("replication_num" = "1"),
+ PARTITION `p202302` VALUES LESS THAN ("2017-03-01") ("replication_num" = "1")
)
DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
PROPERTIES (
diff --git a/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy
index 7ce18addaa..5632a778e7 100644
--- a/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy
+++ b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy
@@ -43,7 +43,7 @@ suite("table_modify_resouce") {
data_sizes[0] = obj.local_data_size
data_sizes[1] = obj.remote_data_size
}
- fetchBeHttp(clos, meta_url)
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
}
// used as passing out parameter to fetchDataSize
List<Long> sizes = [-1, -1]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/08: [Bug](table-function) fix wrong result when seprator of explode_split size more than one (#18824)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 95c4b53c4e2600f207d6c95506a89f6194ccdf5a
Author: Pxl <px...@qq.com>
AuthorDate: Fri Apr 21 11:00:47 2023 +0800
[Bug](table-function) fix wrong result when seprator of explode_split size more than one (#18824)
fix wrong result when seprator of explode_split size more than one
---
be/src/vec/exprs/table_function/vexplode_split.cpp | 3 +--
.../data/nereids_p0/sql_functions/table_function/explode_split.out | 6 ++++++
.../nereids_p0/sql_functions/table_function/explode_split.groovy | 1 +
3 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/exprs/table_function/vexplode_split.cpp b/be/src/vec/exprs/table_function/vexplode_split.cpp
index d2438d8c9a..abc9b9a42b 100644
--- a/be/src/vec/exprs/table_function/vexplode_split.cpp
+++ b/be/src/vec/exprs/table_function/vexplode_split.cpp
@@ -84,11 +84,10 @@ Status VExplodeSplitTableFunction::process_row(size_t row_idx) {
if (first != second) {
output.emplace_back(strv.substr(std::distance(strv.begin(), first),
std::distance(first, second)));
- first = std::next(second);
} else {
output.emplace_back("", 0);
- first = std::next(second, delims.size());
}
+ first = std::next(second, delims.size());
if (second == last) {
break;
diff --git a/regression-test/data/nereids_p0/sql_functions/table_function/explode_split.out b/regression-test/data/nereids_p0/sql_functions/table_function/explode_split.out
index a723f46dab..a430812264 100644
--- a/regression-test/data/nereids_p0/sql_functions/table_function/explode_split.out
+++ b/regression-test/data/nereids_p0/sql_functions/table_function/explode_split.out
@@ -20,3 +20,9 @@
6 b
6 c
+-- !explode_split --
+啊
+啊
+额
+啊
+
diff --git a/regression-test/suites/nereids_p0/sql_functions/table_function/explode_split.groovy b/regression-test/suites/nereids_p0/sql_functions/table_function/explode_split.groovy
index c28f43c4b3..9f64864be7 100644
--- a/regression-test/suites/nereids_p0/sql_functions/table_function/explode_split.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/table_function/explode_split.groovy
@@ -41,4 +41,5 @@ suite("explode_split") {
select k1, e1 from example1
lateral view explode_split(k2, ',') tmp as e1 """
+ qt_explode_split """ select e1 from (select 1 k1) as t lateral view explode_split("啊，啊，额，啊","，") tmp1 as e1; """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org