You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/04/14 13:56:35 UTC
[incubator-doris] branch dev-1.0.1 updated (4ba2fdb96e -> 20dd0201f8)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
from 4ba2fdb96e [fix](error-code) replace invalid format specifier (#8940)
new cccf6b5f84 [improvement] Use System.currentTimeMillis() to get the current millisecond (#8828)
new 2fc7fead93 [fix](load) start transaction before we need it (#8819) (#8908)
new 141341bdb7 [improvement]Disable mini load (#8955)
new d91032ca44 [Vectorizd] Let VAssertRowNumNode handle return value of child->get_next (#8969)
new dd7bd2eb81 [fix](load) wait _send_batch_thread_pool_token rather than shutdown. (#8970)
new bb82a06833 [fix] fix NPE when initialize GlobalState (#8990)
new 20dd0201f8 [fix](load) fix bug of infinite loop in orc scanner (#9007)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/orc_scanner.cpp | 5 +++++
be/src/exec/tablet_sink.cpp | 5 ++++-
be/src/vec/exec/vassert_num_rows_node.cpp | 2 +-
docs/en/administrator-guide/config/fe_config.md | 4 ++++
docs/zh-CN/administrator-guide/config/fe_config.md | 4 ++++
.../java/org/apache/doris/analysis/Analyzer.java | 26 ++++++++++++++--------
.../main/java/org/apache/doris/common/Config.java | 5 +++++
.../org/apache/doris/httpv2/rest/LoadAction.java | 14 +++++++++---
.../org/apache/doris/load/loadv2/SparkLoadJob.java | 7 ++++++
.../doris/load/loadv2/SparkLoadPendingTask.java | 1 -
.../apache/doris/load/sync/canal/CanalUtils.java | 6 ++---
.../org/apache/doris/master/ReportHandler.java | 4 ++--
.../apache/doris/planner/StreamLoadPlanner.java | 2 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 4 ++--
.../apache/doris/rewrite/FoldConstantsRule.java | 2 +-
15 files changed, 67 insertions(+), 24 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 06/07: [fix] fix NPE when initialize GlobalState (#8990)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit bb82a06833fb743d1d53e2b5585d6e8a0bbd68b5
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Thu Apr 14 11:44:41 2022 +0800
[fix] fix NPE when initialize GlobalState (#8990)
Introduced from #8695
The context object may be null for StreamLoadPlanner
---
.../java/org/apache/doris/analysis/Analyzer.java | 26 ++++++++++++++--------
.../org/apache/doris/master/ReportHandler.java | 4 ++--
2 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index e46831100d..693a90d958 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -47,9 +47,9 @@ import org.apache.doris.rewrite.InferFiltersRule;
import org.apache.doris.rewrite.NormalizeBinaryPredicatesRule;
import org.apache.doris.rewrite.RewriteAliasFunctionRule;
import org.apache.doris.rewrite.RewriteBinaryPredicatesRule;
+import org.apache.doris.rewrite.RewriteDateLiteralRule;
import org.apache.doris.rewrite.RewriteEncryptKeyRule;
import org.apache.doris.rewrite.RewriteFromUnixTimeRule;
-import org.apache.doris.rewrite.RewriteDateLiteralRule;
import org.apache.doris.rewrite.mvrewrite.CountDistinctToBitmap;
import org.apache.doris.rewrite.mvrewrite.CountDistinctToBitmapOrHLLRule;
import org.apache.doris.rewrite.mvrewrite.CountFieldToSum;
@@ -330,15 +330,23 @@ public class Analyzer {
mvRewriteRules.add(CountFieldToSum.INSTANCE);
mvExprRewriter = new ExprRewriter(mvRewriteRules);
- // compute max exec mem could be used for broadcast join
- long perNodeMemLimit = context.getSessionVariable().getMaxExecMemByte();
- double autoBroadcastJoinThresholdPercentage = context.getSessionVariable().autoBroadcastJoinThreshold;
- if (autoBroadcastJoinThresholdPercentage > 1) {
- autoBroadcastJoinThresholdPercentage = 1.0;
- } else if (autoBroadcastJoinThresholdPercentage <= 0) {
- autoBroadcastJoinThresholdPercentage = -1.0;
+ // context maybe null. eg, for StreamLoadPlanner.
+ // and autoBroadcastJoinThreshold is only used for Query's DistributedPlanner.
+ // so it is ok to not set autoBroadcastJoinThreshold if context is null
+ if (context != null) {
+ // compute max exec mem could be used for broadcast join
+ long perNodeMemLimit = context.getSessionVariable().getMaxExecMemByte();
+ double autoBroadcastJoinThresholdPercentage = context.getSessionVariable().autoBroadcastJoinThreshold;
+ if (autoBroadcastJoinThresholdPercentage > 1) {
+ autoBroadcastJoinThresholdPercentage = 1.0;
+ } else if (autoBroadcastJoinThresholdPercentage <= 0) {
+ autoBroadcastJoinThresholdPercentage = -1.0;
+ }
+ autoBroadcastJoinThreshold = (long) (perNodeMemLimit * autoBroadcastJoinThresholdPercentage);
+ } else {
+ // autoBroadcastJoinThreshold is a "final" field, must set an initial value for it
+ autoBroadcastJoinThreshold = 0;
}
- autoBroadcastJoinThreshold = (long)(perNodeMemLimit * autoBroadcastJoinThresholdPercentage);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index ea4abd9c09..d0e7b9ae5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -704,7 +704,7 @@ public class ReportHandler extends Daemon {
needDelete = false;
++addToMetaCounter;
} catch (MetaNotFoundException e) {
- LOG.warn("failed add to meta. tablet[{}], backend[{}]. {}",
+ LOG.info("failed add to meta. tablet[{}], backend[{}]. {}",
tabletId, backendId, e.getMessage());
needDelete = true;
}
@@ -717,7 +717,7 @@ public class ReportHandler extends Daemon {
// drop replica
DropReplicaTask task = new DropReplicaTask(backendId, tabletId, backendTabletInfo.getSchemaHash());
batchTask.addTask(task);
- LOG.warn("delete tablet[" + tabletId + " - " + backendTabletInfo.getSchemaHash()
+ LOG.info("delete tablet[" + tabletId + " - " + backendTabletInfo.getSchemaHash()
+ "] from backend[" + backendId + "] because not found in meta");
++deleteFromBackendCounter;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 07/07: [fix](load) fix bug of infinite loop in orc scanner (#9007)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 20dd0201f8e1b2b22eb76c95d176c17fe7d5c120
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Thu Apr 14 11:46:48 2022 +0800
[fix](load) fix bug of infinite loop in orc scanner (#9007)
When encounter unqualified data, orc scanner may not be able
to quit correctly.
---
be/src/exec/orc_scanner.cpp | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/be/src/exec/orc_scanner.cpp b/be/src/exec/orc_scanner.cpp
index 25031c3016..0c69945271 100644
--- a/be/src/exec/orc_scanner.cpp
+++ b/be/src/exec/orc_scanner.cpp
@@ -361,6 +361,11 @@ Status ORCScanner::get_next(Tuple* tuple, MemPool* tuple_pool, bool* eof, bool*
*fill_tuple = _success;
break;
}
+ if (_scanner_eof) {
+ *eof = true;
+ } else {
+ *eof = false;
+ }
return Status::OK();
} catch (orc::ParseError& e) {
std::stringstream str_error;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 05/07: [fix](load) wait _send_batch_thread_pool_token rather than shutdown. (#8970)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit dd7bd2eb813e06a85548f28a2cde37421605f7f0
Author: dataroaring <98...@users.noreply.github.com>
AuthorDate: Thu Apr 14 10:05:14 2022 +0800
[fix](load) wait _send_batch_thread_pool_token rather than shutdown. (#8970)
We can not shutdown _send_batch_thread_pool_token, because _packet_in_flight
has to be clear finally. Otherwise a never ended join on rpc would happen.
It is difficult to handle concurrent problem if a flag setter is not guaranteed to run.
---
be/src/exec/tablet_sink.cpp | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp
index 4a46d538ee..5604483012 100644
--- a/be/src/exec/tablet_sink.cpp
+++ b/be/src/exec/tablet_sink.cpp
@@ -1087,7 +1087,10 @@ Status OlapTableSink::close(RuntimeState* state, Status close_status) {
_stop_background_threads_latch.count_down();
if (_sender_thread) {
_sender_thread->join();
- _send_batch_thread_pool_token->shutdown();
+ // We have to wait all task in _send_batch_thread_pool_token finished,
+ // because it is difficult to handle concurrent problem if we just
+ // shutdown it.
+ _send_batch_thread_pool_token->wait();
}
Expr::close(_output_expr_ctxs, state);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/07: [improvement] Use System.currentTimeMillis() to get the current millisecond (#8828)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit cccf6b5f84581d5d783ce8f8ebf20c66467e49ec
Author: leo65535 <le...@163.com>
AuthorDate: Thu Apr 14 10:03:37 2022 +0800
[improvement] Use System.currentTimeMillis() to get the current millisecond (#8828)
---
.../src/main/java/org/apache/doris/load/sync/canal/CanalUtils.java | 6 +++---
.../src/main/java/org/apache/doris/planner/StreamLoadPlanner.java | 2 +-
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 4 ++--
.../src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java | 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalUtils.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalUtils.java
index 86c5124eeb..02dd6da573 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalUtils.java
@@ -116,7 +116,7 @@ public class CanalUtils {
public static void printRow(CanalEntry.RowChange rowChange, CanalEntry.Header header) {
long executeTime = header.getExecuteTime();
- long delayTime = new Date().getTime() - executeTime;
+ long delayTime = System.currentTimeMillis() - executeTime;
Date date = new Date(executeTime);
CanalEntry.EventType eventType = rowChange.getEventType();
logger.info(row_format, header.getLogfileName(),
@@ -187,7 +187,7 @@ public class CanalUtils {
public static void transactionBegin(CanalEntry.Entry entry) {
long executeTime = entry.getHeader().getExecuteTime();
- long delayTime = new Date().getTime() - executeTime;
+ long delayTime = System.currentTimeMillis() - executeTime;
Date date = new Date(executeTime);
CanalEntry.TransactionBegin begin = null;
try {
@@ -206,7 +206,7 @@ public class CanalUtils {
public static void transactionEnd(CanalEntry.Entry entry) {
long executeTime = entry.getHeader().getExecuteTime();
- long delayTime = new Date().getTime() - executeTime;
+ long delayTime = System.currentTimeMillis() - executeTime;
Date date = new Date(executeTime);
CanalEntry.TransactionEnd end = null;
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
index a454647b1c..602ed433bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
@@ -194,7 +194,7 @@ public class StreamLoadPlanner {
params.setQueryOptions(queryOptions);
TQueryGlobals queryGlobals = new TQueryGlobals();
queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
- queryGlobals.setTimestampMs(new Date().getTime());
+ queryGlobals.setTimestampMs(System.currentTimeMillis());
queryGlobals.setTimeZone(taskInfo.getTimezone());
queryGlobals.setLoadZeroTolerance(taskInfo.getMaxFilterRatio() <= 0.0);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 502b3bfa39..4502d9d597 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -235,7 +235,7 @@ public class Coordinator {
setFromUserProperty(analyzer);
this.queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
- this.queryGlobals.setTimestampMs(new Date().getTime());
+ this.queryGlobals.setTimestampMs(System.currentTimeMillis());
this.queryGlobals.setLoadZeroTolerance(false);
if (context.getSessionVariable().getTimeZone().equals("CST")) {
this.queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE);
@@ -263,7 +263,7 @@ public class Coordinator {
this.scanNodes = scanNodes;
this.queryOptions = new TQueryOptions();
this.queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
- this.queryGlobals.setTimestampMs(new Date().getTime());
+ this.queryGlobals.setTimestampMs(System.currentTimeMillis());
this.queryGlobals.setTimeZone(timezone);
this.queryGlobals.setLoadZeroTolerance(loadZeroTolerance);
this.tResourceInfo = new TResourceInfo("", "");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
index 918eeebad1..6f9bfa8042 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
@@ -350,7 +350,7 @@ public class FoldConstantsRule implements ExprRewriteRule {
TQueryGlobals queryGlobals = new TQueryGlobals();
queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
- queryGlobals.setTimestampMs(new Date().getTime());
+ queryGlobals.setTimestampMs(System.currentTimeMillis());
queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE);
if (context.getSessionVariable().getTimeZone().equals("CST")) {
queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 02/07: [fix](load) start transaction before we need it (#8819) (#8908)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 2fc7fead9344620dd9510b8155992a62e3f3de8b
Author: zhengshengjun <74...@users.noreply.github.com>
AuthorDate: Wed Apr 13 09:50:26 2022 +0800
[fix](load) start transaction before we need it (#8819) (#8908)
---
.../src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java | 7 +++++++
.../java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java | 1 -
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
index 90c887b393..17c39b904a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
@@ -201,6 +201,13 @@ public class SparkLoadJob extends BulkLoadJob {
@Override
protected void unprotectedExecuteJob() throws LoadException {
+ try {
+ beginTxn();
+ } catch (UserException e) {
+ LOG.warn("failed to begin transaction for spark load job {}", id, e);
+ throw new LoadException(e.getMessage());
+ }
+
// create pending task
LoadTask task = new SparkLoadPendingTask(this, fileGroupAggInfo.getAggKeyToFileGroups(),
sparkResource, brokerDesc);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
index 85e123544a..d9f9467a3b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
@@ -110,7 +110,6 @@ public class SparkLoadPendingTask extends LoadTask {
@Override
void executeTask() throws UserException {
LOG.info("begin to execute spark pending task. load job id: {}", loadJobId);
- ((SparkLoadJob) callback).beginTxn();
submitEtlJob();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 04/07: [Vectorizd] Let VAssertRowNumNode handle return value of child->get_next (#8969)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit d91032ca446bc70aff6459d8fd6268616ae95197
Author: dataroaring <98...@users.noreply.github.com>
AuthorDate: Tue Apr 12 19:56:03 2022 +0800
[Vectorizd] Let VAssertRowNumNode handle return value of child->get_next (#8969)
---
be/src/vec/exec/vassert_num_rows_node.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/vec/exec/vassert_num_rows_node.cpp b/be/src/vec/exec/vassert_num_rows_node.cpp
index b499175bed..d802ba12e0 100644
--- a/be/src/vec/exec/vassert_num_rows_node.cpp
+++ b/be/src/vec/exec/vassert_num_rows_node.cpp
@@ -49,7 +49,7 @@ Status VAssertNumRowsNode::open(RuntimeState* state) {
Status VAssertNumRowsNode::get_next(RuntimeState* state, Block* block, bool* eos) {
RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::GETNEXT));
SCOPED_TIMER(_runtime_profile->total_time_counter());
- child(0)->get_next(state, block, eos);
+ RETURN_IF_ERROR(child(0)->get_next(state, block, eos));
_num_rows_returned += block->rows();
bool assert_res = false;
switch (_assertion) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 03/07: [improvement]Disable mini load (#8955)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 141341bdb76d7235932eaab2ea21228ffc7c04ed
Author: jiafeng.zhang <zh...@gmail.com>
AuthorDate: Tue Apr 12 16:01:03 2022 +0800
[improvement]Disable mini load (#8955)
Disable miniload by default
---
docs/en/administrator-guide/config/fe_config.md | 4 ++++
docs/zh-CN/administrator-guide/config/fe_config.md | 4 ++++
.../src/main/java/org/apache/doris/common/Config.java | 5 +++++
.../main/java/org/apache/doris/httpv2/rest/LoadAction.java | 14 +++++++++++---
4 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md
index 02db4f03b8..281d841ff5 100644
--- a/docs/en/administrator-guide/config/fe_config.md
+++ b/docs/en/administrator-guide/config/fe_config.md
@@ -209,6 +209,10 @@ Default:100 * 1024 * 1024 (100MB)
This is the maximum number of bytes of the file uploaded by the put or post method, the default value: 100MB
+### **`disable_mini_load`**
+
+Whether to disable the mini load data import method, the default:true (Disabled)
+
### frontend_address
Status: Deprecated, not recommended use. This parameter may be deleted later Type: string Description: Explicitly set the IP address of FE instead of using *InetAddress.getByName* to get the IP address. Usually in *InetAddress.getByName* When the expected results cannot be obtained. Only IP address is supported, not hostname. Default value: 0.0.0.0
diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md
index 89ad2a208c..e16e58c365 100644
--- a/docs/zh-CN/administrator-guide/config/fe_config.md
+++ b/docs/zh-CN/administrator-guide/config/fe_config.md
@@ -207,6 +207,10 @@ workers 线程池默认不做设置,根据自己需要进行设置
这个是 put 或 post 方法上传文件的最大字节数,默认值:100MB
+### **`disable_mini_load`**
+
+是否禁用mini load数据导入方式,默认是:true (禁用)
+
### `default_max_filter_ratio`
默认值:0
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index 33b9988e52..b70ec7c5a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -348,6 +348,11 @@ public class Config extends ConfigBase {
*/
@ConfField public static int jetty_server_max_http_post_size = 100 * 1024 * 1024;
+ /**
+ * Mini load disabled by default
+ */
+ @ConfField public static boolean disable_mini_load = true;
+
/**
* The backlog_num for mysql nio server
* When you enlarge this backlog_num, you should enlarge the value in
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
index d5b2b71e58..b9c85dbdfb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
@@ -19,7 +19,9 @@ package org.apache.doris.httpv2.rest;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.cluster.ClusterNamespace;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.httpv2.entity.RestBaseResult;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -32,6 +34,7 @@ import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@@ -59,9 +62,14 @@ public class LoadAction extends RestBaseController {
@RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", method = RequestMethod.PUT)
public Object load(HttpServletRequest request, HttpServletResponse response,
@PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) {
- this.isStreamLoad = false;
- executeCheckPassword(request, response);
- return executeWithoutPassword(request, response, db, table);
+ if(Config.disable_mini_load) {
+ ResponseEntity entity = ResponseEntityBuilder.notFound("The mini load operation has been disabled by default, if you need to add disable_mini_load=false in fe.conf.");
+ return entity;
+ } else {
+ this.isStreamLoad = false;
+ executeCheckPassword(request, response);
+ return executeWithoutPassword(request, response, db, table);
+ }
}
@RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load", method = RequestMethod.PUT)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org