You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2019/12/19 11:09:34 UTC
[incubator-doris] branch master updated: [Export] Fix bug that NPE
may be thrown when executing "show export; " (#2509)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5111f8c [Export] Fix bug that NPE may be thrown when executing "show export;" (#2509)
5111f8c is described below
commit 5111f8cfe8e8dc3f038182f70e585ae05e961cb5
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Thu Dec 19 19:09:25 2019 +0800
[Export] Fix bug that NPE may be thrown when executing "show export;" (#2509)
Some export job from old version of Doris may not has timeout property,
which will cause NPE.
2 more changes:
1. Change the default BE config "max_runnings_transactions" to 2000.
2. Add a new metric to FE to show the master ip:port.
---
be/src/common/config.h | 2 +-
.../main/java/org/apache/doris/load/ExportJob.java | 15 +++++++++++++--
.../java/org/apache/doris/metric/MetricVisitor.java | 3 +++
.../doris/metric/PrometheusMetricVisitor.java | 4 ++++
.../java/org/apache/doris/planner/OlapScanNode.java | 21 +++++++++++----------
.../main/java/org/apache/doris/qe/Coordinator.java | 4 ++--
6 files changed, 34 insertions(+), 15 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 53fe10e..51cad63 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -494,7 +494,7 @@ namespace config {
// max number of txns in txn manager
// this is a self protection to avoid too many txns saving in manager
- CONF_Int64(max_runnings_transactions, "200");
+ CONF_Int64(max_runnings_transactions, "2000");
} // namespace config
diff --git a/fe/src/main/java/org/apache/doris/load/ExportJob.java b/fe/src/main/java/org/apache/doris/load/ExportJob.java
index cc3dd68..05b04b1 100644
--- a/fe/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/src/main/java/org/apache/doris/load/ExportJob.java
@@ -36,6 +36,7 @@ import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
@@ -401,11 +402,21 @@ public class ExportJob implements Writable {
}
public int getTimeoutSecond() {
- return Integer.parseInt(properties.get(LoadStmt.TIMEOUT_PROPERTY));
+ if (properties.containsKey(LoadStmt.TIMEOUT_PROPERTY)) {
+ return Integer.parseInt(properties.get(LoadStmt.TIMEOUT_PROPERTY));
+ } else {
+ // for compatibility, some export job in old version does not have this property. use default.
+ return Config.export_task_default_timeout_second;
+ }
}
public int getTabletNumberPerTask() {
- return Integer.parseInt(properties.get(ExportStmt.TABLET_NUMBER_PER_TASK_PROP));
+ if (properties.containsKey(ExportStmt.TABLET_NUMBER_PER_TASK_PROP)) {
+ return Integer.parseInt(properties.get(ExportStmt.TABLET_NUMBER_PER_TASK_PROP));
+ } else {
+ // for compatibility, some export job in old version does not have this property. use default.
+ return Config.export_tablet_num_per_task;
+ }
}
public List<String> getPartitions() {
diff --git a/fe/src/main/java/org/apache/doris/metric/MetricVisitor.java b/fe/src/main/java/org/apache/doris/metric/MetricVisitor.java
index 48a930e..7abb998 100644
--- a/fe/src/main/java/org/apache/doris/metric/MetricVisitor.java
+++ b/fe/src/main/java/org/apache/doris/metric/MetricVisitor.java
@@ -21,6 +21,9 @@ import org.apache.doris.monitor.jvm.JvmStats;
import com.codahale.metrics.Histogram;
+/*
+ * MetricVisitor will visit the metrics in metric repo and print them in StringBuilder
+ */
public abstract class MetricVisitor {
protected String prefix;
diff --git a/fe/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java b/fe/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
index ab7222b..bd8c309 100644
--- a/fe/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
+++ b/fe/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
@@ -182,6 +182,10 @@ public class PrometheusMetricVisitor extends MetricVisitor {
.append(Catalog.getCurrentSystemInfo().getBackendIds(true).size()).append("\n");
sb.append(NODE_INFO).append("{type=\"be_node_num\", state=\"decommissioned\"} ")
.append(Catalog.getCurrentSystemInfo().getDecommissionedBackendIds().size()).append("\n");
+ sb.append(NODE_INFO).append("{type=\"broker_node_num\", state=\"dead\"} ").append(
+ Catalog.getCurrentCatalog().getBrokerMgr().getAllBrokers().stream().filter(b -> !b.isAlive).count()).append("\n");
+ sb.append(NODE_INFO).append("{type=\"master_fe_ip\", state=\"master\"} ").append(
+ Catalog.getCurrentCatalog().getMasterIp() + ":" + Catalog.getCurrentCatalog().getMasterHttpPort()).append("\n");
return;
}
}
diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 012742b..b113388 100644
--- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -17,15 +17,6 @@
package org.apache.doris.planner;
-import com.google.common.base.Joiner;
-import com.google.common.base.Objects;
-import com.google.common.base.Objects.ToStringHelper;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Range;
-
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.BaseTableRef;
import org.apache.doris.analysis.TupleDescriptor;
@@ -59,6 +50,16 @@ import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.TScanRange;
import org.apache.doris.thrift.TScanRangeLocation;
import org.apache.doris.thrift.TScanRangeLocations;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Objects;
+import com.google.common.base.Objects.ToStringHelper;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -95,6 +96,7 @@ public class OlapScanNode extends ScanNode {
private HashSet<Long> scanBackendIds = new HashSet<>();
private Map<Long, Integer> tabletId2BucketSeq = Maps.newHashMap();
+ // a bucket seq may map to many tablets, and each tablet has a TScanRangeLocations.
public ArrayListMultimap<Integer, TScanRangeLocations> bucketSeq2locations= ArrayListMultimap.create();
// Constructs node to scan given data files of table 'tbl'.
@@ -270,7 +272,6 @@ public class OlapScanNode extends ScanNode {
replicas = localReplicas;
} else {
replicas = allQueryableReplicas;
-
}
Collections.shuffle(replicas);
diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java
index 5e273b6..b42da95 100644
--- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -938,7 +938,7 @@ public class Coordinator {
}
}
- //One fragment could only have one HashJoinNode
+ // One fragment could only have one HashJoinNode
private boolean isColocateJoin(PlanNode node) {
if (Config.disable_colocate_join) {
return false;
@@ -1054,10 +1054,10 @@ public class Coordinator {
scanRangeParams.scan_range = location.scan_range;
scanRangeParamsList.add(scanRangeParams);
}
-
}
}
+ // randomly choose a backend from the TScanRangeLocations for a certain bucket sequence.
private void getExecHostPortForBucketSeq(TScanRangeLocations seqLocation, Integer bucketSeq) throws Exception {
int randomLocation = new Random().nextInt(seqLocation.locations.size());
Reference<Long> backendIdRef = new Reference<Long>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org