You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/22 13:25:45 UTC
[incubator-doris] branch master updated: [Refactor] simplify some code in routine load (#9532)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3391de482b [Refactor] simplify some code in routine load (#9532)
3391de482b is described below
commit 3391de482b603e5ebd63b21835fd69dc60627920
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Sun May 22 21:25:39 2022 +0800
[Refactor] simplify some code in routine load (#9532)
---
.../doris/analysis/CreateRoutineLoadStmt.java | 4 +-
.../doris/load/routineload/RoutineLoadJob.java | 70 ++++++++++------------
2 files changed, 32 insertions(+), 42 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
index c4c6f78593..45a0a7ff42 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
@@ -461,8 +461,8 @@ public class CreateRoutineLoadStmt extends DdlStmt {
format = ""; // if it's not json, then it's mean csv and set empty
} else if (format.equalsIgnoreCase("json")) {
format = "json";
- jsonPaths = jobProperties.get(JSONPATHS);
- jsonRoot = jobProperties.get(JSONROOT);
+ jsonPaths = jobProperties.getOrDefault(JSONPATHS, "");
+ jsonRoot = jobProperties.getOrDefault(JSONROOT, "");
stripOuterArray = Boolean.valueOf(jobProperties.getOrDefault(STRIP_OUTER_ARRAY, "false"));
numAsString = Boolean.valueOf(jobProperties.getOrDefault(NUM_AS_STRING, "false"));
fuzzyParse = Boolean.valueOf(jobProperties.getOrDefault(FUZZY_PARSE, "false"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
index 2c62ef004e..4ee92ca025 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
@@ -299,42 +299,37 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
if (Strings.isNullOrEmpty(stmt.getFormat()) || stmt.getFormat().equals("csv")) {
jobProperties.put(PROPS_FORMAT, "csv");
- jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false");
- jobProperties.put(PROPS_NUM_AS_STRING, "false");
- jobProperties.put(PROPS_JSONPATHS, "");
- jobProperties.put(PROPS_JSONROOT, "");
- jobProperties.put(PROPS_FUZZY_PARSE, "false");
} else if (stmt.getFormat().equals("json")) {
jobProperties.put(PROPS_FORMAT, "json");
- if (!Strings.isNullOrEmpty(stmt.getJsonPaths())) {
- jobProperties.put(PROPS_JSONPATHS, stmt.getJsonPaths());
- } else {
- jobProperties.put(PROPS_JSONPATHS, "");
- }
- if (!Strings.isNullOrEmpty(stmt.getJsonRoot())) {
- jobProperties.put(PROPS_JSONROOT, stmt.getJsonRoot());
- } else {
- jobProperties.put(PROPS_JSONROOT, "");
- }
- if (stmt.isStripOuterArray()) {
- jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "true");
- } else {
- jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false");
- }
- if (stmt.isNumAsString()) {
- jobProperties.put(PROPS_NUM_AS_STRING, "true");
- } else {
- jobProperties.put(PROPS_NUM_AS_STRING, "false");
- }
- if (stmt.isFuzzyParse()) {
- jobProperties.put(PROPS_FUZZY_PARSE, "true");
- } else {
- jobProperties.put(PROPS_FUZZY_PARSE, "false");
- }
-
} else {
throw new UserException("Invalid format type.");
}
+
+ if (!Strings.isNullOrEmpty(stmt.getJsonPaths())) {
+ jobProperties.put(PROPS_JSONPATHS, stmt.getJsonPaths());
+ } else {
+ jobProperties.put(PROPS_JSONPATHS, "");
+ }
+ if (!Strings.isNullOrEmpty(stmt.getJsonRoot())) {
+ jobProperties.put(PROPS_JSONROOT, stmt.getJsonRoot());
+ } else {
+ jobProperties.put(PROPS_JSONROOT, "");
+ }
+ if (stmt.isStripOuterArray()) {
+ jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "true");
+ } else {
+ jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false");
+ }
+ if (stmt.isNumAsString()) {
+ jobProperties.put(PROPS_NUM_AS_STRING, "true");
+ } else {
+ jobProperties.put(PROPS_NUM_AS_STRING, "false");
+ }
+ if (stmt.isFuzzyParse()) {
+ jobProperties.put(PROPS_FUZZY_PARSE, "true");
+ } else {
+ jobProperties.put(PROPS_FUZZY_PARSE, "false");
+ }
}
private void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) {
@@ -342,10 +337,8 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
columnDescs = new ImportColumnDescs();
if (routineLoadDesc.getColumnsInfo() != null) {
ImportColumnsStmt columnsStmt = routineLoadDesc.getColumnsInfo();
- if (columnsStmt.getColumns() != null || columnsStmt.getColumns().size() != 0) {
- for (ImportColumnDesc columnDesc : columnsStmt.getColumns()) {
- columnDescs.descs.add(columnDesc);
- }
+ if (columnsStmt.getColumns() != null) {
+ columnDescs.descs.addAll(columnsStmt.getColumns());
}
}
if (routineLoadDesc.getPrecedingFilter() != null) {
@@ -1316,7 +1309,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
public List<List<String>> getTasksShowInfo() {
List<List<String>> rows = Lists.newArrayList();
- routineLoadTaskInfoList.stream().forEach(entity -> {
+ routineLoadTaskInfoList.forEach(entity -> {
try {
entity.setTxnStatus(Catalog.getCurrentCatalog().getGlobalTransactionMgr().getDatabaseTransactionMgr(dbId).getTransactionState(entity.getTxnId()).getTransactionStatus());
rows.add(entity.getTaskShowInfo());
@@ -1481,10 +1474,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
return false;
}
Preconditions.checkState(endTimestamp != -1, endTimestamp);
- if ((System.currentTimeMillis() - endTimestamp) > Config.label_keep_max_second * 1000) {
- return true;
- }
- return false;
+ return (System.currentTimeMillis() - endTimestamp) > Config.label_keep_max_second * 1000;
}
public boolean isFinal() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org