You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/09/07 02:05:55 UTC
[incubator-doris] branch branch-0.13 updated (2271ccd -> bbefdad)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-0.13
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git.
from 2271ccd [Bug] Fix bug that table ids is not set right for hadoop load job (#4535)
new 706faaf [Log] Add log for trace broker (#4505)
new af3ff75 [BUG] Tablet is not readable and delete handler report -1903 error, when condition value contains \n (#4531)
new 1c31ca7 [Spark load][Bug] fix that cancelling a spark load in the `PENDING` phase will not succeed (#4536)
new bbefdad [Spark Load][Bug] Keep the column splitting in spark load consistent with broker load / mini load (#4532)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/olap/delete_handler.cpp | 8 ++++++--
.../org/apache/doris/common/util/BrokerUtil.java | 2 +-
.../doris/load/loadv2/BrokerLoadPendingTask.java | 4 ++--
.../doris/load/loadv2/SparkEtlJobHandler.java | 17 ++++++++++++----
.../doris/load/loadv2/SparkLauncherMonitor.java | 5 +++++
.../doris/load/loadv2/SparkLoadAppHandle.java | 4 ++++
.../org/apache/doris/load/loadv2/SparkLoadJob.java | 4 ++++
.../doris/load/loadv2/SparkLoadPendingTask.java | 11 ++++++-----
.../doris/load/loadv2/SparkEtlJobHandlerTest.java | 4 ++--
.../org/apache/doris/load/loadv2/dpp/SparkDpp.java | 23 +++++++++++++++++++++-
.../apache/doris/load/loadv2/etl/EtlJobConfig.java | 9 +--------
11 files changed, 66 insertions(+), 25 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 03/04: [Spark load][Bug] fix that cancelling a
spark load in the `PENDING` phase will not succeed (#4536)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.13
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 1c31ca7b42161dbf2b628e9cad5f081d0753da16
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Sun Sep 6 20:32:47 2020 +0800
[Spark load][Bug] fix that cancelling a spark load in the `PENDING` phase will not succeed (#4536)
---
.../apache/doris/load/loadv2/SparkEtlJobHandler.java | 17 +++++++++++++----
.../apache/doris/load/loadv2/SparkLauncherMonitor.java | 5 +++++
.../apache/doris/load/loadv2/SparkLoadAppHandle.java | 4 ++++
.../java/org/apache/doris/load/loadv2/SparkLoadJob.java | 4 ++++
.../apache/doris/load/loadv2/SparkLoadPendingTask.java | 11 ++++++-----
.../doris/load/loadv2/SparkEtlJobHandlerTest.java | 4 ++--
6 files changed, 34 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
index f3e2f91..21f6431 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
@@ -77,7 +77,7 @@ public class SparkEtlJobHandler {
private static final String YARN_KILL_CMD = "%s --config %s application -kill %s";
public void submitEtlJob(long loadJobId, String loadLabel, EtlJobConfig etlJobConfig, SparkResource resource,
- BrokerDesc brokerDesc, SparkPendingTaskAttachment attachment) throws LoadException {
+ BrokerDesc brokerDesc, SparkLoadAppHandle handle, SparkPendingTaskAttachment attachment) throws LoadException {
// delete outputPath
deleteEtlOutputPath(etlJobConfig.outputPath, brokerDesc);
@@ -142,13 +142,12 @@ public class SparkEtlJobHandler {
}
// start app
- SparkLoadAppHandle handle = null;
State state = null;
String appId = null;
String errMsg = "start spark app failed. error: ";
try {
Process process = launcher.launch();
- handle = new SparkLoadAppHandle(process);
+ handle.setProcess(process);
if (!FeConstants.runningUnitTest) {
SparkLauncherMonitor.LogMonitor logMonitor = SparkLauncherMonitor.createLogMonitor(handle);
logMonitor.setSubmitTimeoutMs(GET_APPID_TIMEOUT_MS);
@@ -264,8 +263,18 @@ public class SparkEtlJobHandler {
}
public void killEtlJob(SparkLoadAppHandle handle, String appId, long loadJobId, SparkResource resource) throws LoadException {
- Preconditions.checkNotNull(appId);
if (resource.isYarnMaster()) {
+ // The appId may be empty when the load job is in PENDING phase. This is because the appId is
+ // parsed from the spark launcher process's output (spark launcher process submit job and then
+ // return appId). In this case, the spark job has still not been submitted, we only need to kill
+ // the spark launcher process.
+ if (Strings.isNullOrEmpty(appId)) {
+ appId = handle.getAppId();
+ if (Strings.isNullOrEmpty(appId)) {
+ handle.kill();
+ return;
+ }
+ }
// prepare yarn config
String configDir = resource.prepareYarnConfig();
// yarn client path
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java
index 9a66448..9af39c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java
@@ -106,6 +106,11 @@ public class SparkLauncherMonitor {
// UNKNOWN/SUBMITTED for a long time.
@Override
public void run() {
+ if (handle.getState() == SparkLoadAppHandle.State.KILLED) {
+ // If handle has been killed, kill the process
+ process.destroyForcibly();
+ return;
+ }
BufferedReader outReader = null;
String line = null;
long startTime = System.currentTimeMillis();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadAppHandle.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadAppHandle.java
index 2289655..c732027 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadAppHandle.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadAppHandle.java
@@ -134,6 +134,10 @@ public class SparkLoadAppHandle implements Writable {
public String getLogPath() { return this.logPath; }
+ public void setProcess(Process process) {
+ this.process = process;
+ }
+
public void setState(State state) {
this.state = state;
this.fireEvent(false);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
index f2460c0..9297295 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
@@ -715,6 +715,10 @@ public class SparkLoadJob extends BulkLoadJob {
return etlStartTimestamp;
}
+ public SparkLoadAppHandle getHandle() {
+ return sparkLoadAppHandle;
+ }
+
public void clearSparkLauncherLog() {
String logPath = sparkLoadAppHandle.getLogPath();
if (!Strings.isNullOrEmpty(logPath)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
index 185e84a..dbf5247 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
@@ -57,16 +57,15 @@ import org.apache.doris.load.loadv2.etl.EtlJobConfig.EtlTable;
import org.apache.doris.load.loadv2.etl.EtlJobConfig.FilePatternVersion;
import org.apache.doris.load.loadv2.etl.EtlJobConfig.SourceType;
import org.apache.doris.transaction.TransactionState;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -84,6 +83,7 @@ public class SparkLoadPendingTask extends LoadTask {
private final long loadJobId;
private final long transactionId;
private EtlJobConfig etlJobConfig;
+ private SparkLoadAppHandle sparkLoadAppHandle;
public SparkLoadPendingTask(SparkLoadJob loadTaskCallback,
Map<FileGroupAggKey, List<BrokerFileGroup>> aggKeyToBrokerFileGroups,
@@ -98,6 +98,7 @@ public class SparkLoadPendingTask extends LoadTask {
this.loadJobId = loadTaskCallback.getId();
this.loadLabel = loadTaskCallback.getLabel();
this.transactionId = loadTaskCallback.getTransactionId();
+ this.sparkLoadAppHandle = loadTaskCallback.getHandle();
this.failMsg = new FailMsg(FailMsg.CancelType.ETL_SUBMIT_FAIL);
}
@@ -115,7 +116,7 @@ public class SparkLoadPendingTask extends LoadTask {
// handler submit etl job
SparkEtlJobHandler handler = new SparkEtlJobHandler();
- handler.submitEtlJob(loadJobId, loadLabel, etlJobConfig, resource, brokerDesc, sparkAttachment);
+ handler.submitEtlJob(loadJobId, loadLabel, etlJobConfig, resource, brokerDesc, sparkLoadAppHandle, sparkAttachment);
LOG.info("submit spark etl job success. load job id: {}, attachment: {}", loadJobId, sparkAttachment);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkEtlJobHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkEtlJobHandlerTest.java
index 904a8b1..73df628 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkEtlJobHandlerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkEtlJobHandlerTest.java
@@ -167,7 +167,7 @@ public class SparkEtlJobHandlerTest {
BrokerDesc brokerDesc = new BrokerDesc(broker, Maps.newHashMap());
SparkPendingTaskAttachment attachment = new SparkPendingTaskAttachment(pendingTaskId);
SparkEtlJobHandler handler = new SparkEtlJobHandler();
- handler.submitEtlJob(loadJobId, label, etlJobConfig, resource, brokerDesc, attachment);
+ handler.submitEtlJob(loadJobId, label, etlJobConfig, resource, brokerDesc, handle, attachment);
// check submit etl job success
Assert.assertEquals(appId, attachment.getAppId());
@@ -203,7 +203,7 @@ public class SparkEtlJobHandlerTest {
BrokerDesc brokerDesc = new BrokerDesc(broker, Maps.newHashMap());
SparkPendingTaskAttachment attachment = new SparkPendingTaskAttachment(pendingTaskId);
SparkEtlJobHandler handler = new SparkEtlJobHandler();
- handler.submitEtlJob(loadJobId, label, etlJobConfig, resource, brokerDesc, attachment);
+ handler.submitEtlJob(loadJobId, label, etlJobConfig, resource, brokerDesc, handle, attachment);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 02/04: [BUG] Tablet is not readable and delete
handler report -1903 error, when condition value contains \n (#4531)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.13
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit af3ff75ea71ab926175ff5df6598ef5c47f2c207
Author: ZhangYu0123 <67...@users.noreply.github.com>
AuthorDate: Sun Sep 6 20:29:44 2020 +0800
[BUG] Tablet is not readable and delete handler report -1903 error, when condition value contains \n (#4531)
---
be/src/olap/delete_handler.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp
index ed6da56..a58b3e4 100644
--- a/be/src/olap/delete_handler.cpp
+++ b/be/src/olap/delete_handler.cpp
@@ -197,9 +197,13 @@ bool DeleteHandler::_parse_condition(const std::string& condition_str, TConditio
smatch what;
try {
- // Condition string format
+ // Condition string format, the format is (column_name)(op)(value)
+ // eg: condition_str="c1 = 1597751948193618247 and length(source)<1;\n;\n"
+ // group1: (\w+) matchs "c1"
+ // group2: ((?:=)|(?:!=)|(?:>>)|(?:<<)|(?:>=)|(?:<=)|(?:\*=)|(?:IS)) matchs "="
+ // group3: ((?:[\s\S]+)?) matchs "1597751948193618247 and length(source)<1;\n;\n"
const char* const CONDITION_STR_PATTERN =
- R"((\w+)\s*((?:=)|(?:!=)|(?:>>)|(?:<<)|(?:>=)|(?:<=)|(?:\*=)|(?:IS))\s*((?:[\S ]+)?))";
+ R"((\w+)\s*((?:=)|(?:!=)|(?:>>)|(?:<<)|(?:>=)|(?:<=)|(?:\*=)|(?:IS))\s*((?:[\s\S]+)?))";
regex ex(CONDITION_STR_PATTERN);
if (regex_match(condition_str, what, ex)) {
if (condition_str.size() != what[0].str().size()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 04/04: [Spark Load][Bug] Keep the column
splitting in spark load consistent with broker load / mini load (#4532)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.13
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit bbefdadc9a9225e96a28940c2477c16904594aae
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Sun Sep 6 20:33:26 2020 +0800
[Spark Load][Bug] Keep the column splitting in spark load consistent with broker load / mini load (#4532)
---
.../org/apache/doris/load/loadv2/dpp/SparkDpp.java | 23 +++++++++++++++++++++-
.../apache/doris/load/loadv2/etl/EtlJobConfig.java | 9 +--------
2 files changed, 23 insertions(+), 9 deletions(-)
diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java
index 6e5a714..fd71add 100644
--- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java
+++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java
@@ -62,6 +62,7 @@ import java.io.IOException;
import java.math.BigInteger;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -553,12 +554,13 @@ public final class SparkDpp implements java.io.Serializable {
for (EtlJobConfig.EtlColumn column : baseIndex.columns) {
parsers.add(ColumnParser.create(column));
}
+ char separator = (char)fileGroup.columnSeparator.getBytes(Charset.forName("UTF-8"))[0];
// now we first support csv file
// TODO: support parquet file and orc file
JavaRDD<Row> rowRDD = sourceDataRdd.flatMap(
record -> {
scannedRowsAcc.add(1);
- String[] attributes = record.split(fileGroup.columnSeparator);
+ String[] attributes = splitLine(record, separator);
List<Row> result = new ArrayList<>();
boolean validRow = true;
if (attributes.length != columnSize) {
@@ -640,6 +642,25 @@ public final class SparkDpp implements java.io.Serializable {
return srcSchema;
}
+ // This method is to keep the splitting consistent with broker load / mini load
+ private String[] splitLine(String line, char sep) {
+ if (line == null || line.equals("")) {
+ return new String[0];
+ }
+ int index = 0;
+ int lastIndex = 0;
+ // line-begin char and line-end char are considered to be 'delimeter'
+ List<String> values = new ArrayList<>();
+ for (int i = 0 ; i < line.length(); i++, index++) {
+ if (line.charAt(index) == sep) {
+ values.add(line.substring(lastIndex, index));
+ lastIndex = index + 1;
+ }
+ }
+ values.add(line.substring(lastIndex, index));
+ return values.toArray(new String[0]);
+ }
+
// partition keys will be parsed into double from json
// so need to convert it to partition columns' type
private Object convertPartitionKey(Object srcValue, Class dstClass) throws SparkDppException {
diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/EtlJobConfig.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/EtlJobConfig.java
index 9ee4d83..8238aea 100644
--- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/EtlJobConfig.java
+++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/EtlJobConfig.java
@@ -503,20 +503,13 @@ public class EtlJobConfig implements Serializable {
this.filePaths = filePaths;
this.fileFieldNames = fileFieldNames;
this.columnsFromPath = columnsFromPath;
+ this.columnSeparator = Strings.isNullOrEmpty(columnSeparator) ? "\t" : columnSeparator;
this.lineDelimiter = lineDelimiter;
this.isNegative = isNegative;
this.fileFormat = fileFormat;
this.columnMappings = columnMappings;
this.where = where;
this.partitions = partitions;
-
- // Convert some special characters in column separator
- char sep = Strings.isNullOrEmpty(columnSeparator) ? '\t' : columnSeparator.charAt(0);
- if (".$|()[]{}^?*+\\".indexOf(sep) != -1) {
- this.columnSeparator = new String(new char[]{'\\', sep});
- } else {
- this.columnSeparator = Character.toString(sep);
- }
}
// for data from table
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/04: [Log] Add log for trace broker (#4505)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.13
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 706faaf83eb25fccf0aedca69f5c4fa3c7441134
Author: ZhangYu0123 <67...@users.noreply.github.com>
AuthorDate: Sun Sep 6 20:25:08 2020 +0800
[Log] Add log for trace broker (#4505)
Add tracing broker log. When fe get filestatus for distributing load task to broker,
the broker maybe get empty files and not give correct error code.
Add this log to easy track which broker process filestatus operation and we can get the error log.
---
fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java | 2 +-
.../main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
index 9e62c1c..9b641a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
@@ -408,7 +408,7 @@ public class BrokerUtil {
return pair;
}
- private static TNetworkAddress getAddress(BrokerDesc brokerDesc) throws UserException {
+ public static TNetworkAddress getAddress(BrokerDesc brokerDesc) throws UserException {
FsBroker broker = null;
try {
String localIP = FrontendOptions.getLocalHostAddress();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java
index 3d39460..a4cff88 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java
@@ -87,8 +87,8 @@ public class BrokerLoadPendingTask extends LoadTask {
}
tableTotalFileSize += groupFileSize;
tableTotalFileNum += fileStatuses.size();
- LOG.info("get {} files in file group {} for table {}. size: {}. job: {}",
- fileStatuses.size(), groupNum, entry.getKey(), groupFileSize, callback.getCallbackId());
+ LOG.info("get {} files in file group {} for table {}. size: {}. job: {}, broker: {} ",
+ fileStatuses.size(), groupNum, entry.getKey(), groupFileSize, callback.getCallbackId(), BrokerUtil.getAddress(brokerDesc));
groupNum++;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org