You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/08/19 20:33:35 UTC
[20/50] [abbrv] hadoop git commit: HDFS-10560. DiskBalancer: Reuse
ObjectMapper instance to improve the performance. Contributed by Yiqun Lin.
HDFS-10560. DiskBalancer: Reuse ObjectMapper instance to improve the performance. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b047bc72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b047bc72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b047bc72
Branch: refs/heads/HDFS-7240
Commit: b047bc7270f3461156e4d08423c728ee9c67dba5
Parents: b427ce1
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Aug 16 10:20:08 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Aug 16 10:20:08 2016 -0700
----------------------------------------------------------------------
.../server/datanode/DiskBalancerWorkItem.java | 11 ++++++---
.../server/datanode/DiskBalancerWorkStatus.java | 26 +++++++++++---------
.../hdfs/server/datanode/DiskBalancer.java | 5 ++--
.../server/diskbalancer/command/Command.java | 6 +++--
.../connectors/JsonNodeConnector.java | 8 +++---
.../datamodel/DiskBalancerCluster.java | 11 ++++++---
.../datamodel/DiskBalancerVolume.java | 11 ++++++---
7 files changed, 46 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
index f46a987..592a89f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.htrace.fasterxml.jackson.annotation.JsonInclude;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.IOException;
@@ -34,6 +35,10 @@ import java.io.IOException;
@InterfaceStability.Unstable
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public class DiskBalancerWorkItem {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerWorkItem.class);
+
private long startTime;
private long secondsElapsed;
private long bytesToCopy;
@@ -74,8 +79,7 @@ public class DiskBalancerWorkItem {
*/
public static DiskBalancerWorkItem parseJson(String json) throws IOException {
Preconditions.checkNotNull(json);
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerWorkItem.class);
+ return READER.readValue(json);
}
/**
@@ -169,8 +173,7 @@ public class DiskBalancerWorkItem {
* @throws IOException
*/
public String toJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
+ return MAPPER.writeValueAsString(this);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
index 14789b6..94bf6a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import org.codehaus.jackson.map.SerializationConfig;
import static org.codehaus.jackson.map.type.TypeFactory.defaultInstance;
@@ -38,6 +39,15 @@ import java.util.LinkedList;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DiskBalancerWorkStatus {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final ObjectMapper MAPPER_WITH_INDENT_OUTPUT =
+ new ObjectMapper().enable(
+ SerializationConfig.Feature.INDENT_OUTPUT);
+ private static final ObjectReader READER_WORKSTATUS =
+ new ObjectMapper().reader(DiskBalancerWorkStatus.class);
+ private static final ObjectReader READER_WORKENTRY = new ObjectMapper()
+ .reader(defaultInstance().constructCollectionType(List.class,
+ DiskBalancerWorkEntry.class));
private final List<DiskBalancerWorkEntry> currentState;
private Result result;
@@ -92,10 +102,7 @@ public class DiskBalancerWorkStatus {
this.result = result;
this.planID = planID;
this.planFile = planFile;
- ObjectMapper mapper = new ObjectMapper();
- this.currentState = mapper.readValue(currentState,
- defaultInstance().constructCollectionType(
- List.class, DiskBalancerWorkEntry.class));
+ this.currentState = READER_WORKENTRY.readValue(currentState);
}
@@ -141,15 +148,11 @@ public class DiskBalancerWorkStatus {
* @throws IOException
**/
public String currentStateString() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- mapper.enable(SerializationConfig.Feature.INDENT_OUTPUT);
- return mapper.writeValueAsString(currentState);
+ return MAPPER_WITH_INDENT_OUTPUT.writeValueAsString(currentState);
}
public String toJsonString() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
-
+ return MAPPER.writeValueAsString(this);
}
/**
@@ -160,8 +163,7 @@ public class DiskBalancerWorkStatus {
*/
public static DiskBalancerWorkStatus parseJson(String json) throws
IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerWorkStatus.class);
+ return READER_WORKSTATUS.readValue(json);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index 523c0a6..ec72d97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -34,8 +34,8 @@ import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerConstants;
import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.Step;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.util.Time;
-import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -263,8 +263,7 @@ public class DiskBalancer {
for (Map.Entry<String, FsVolumeSpi> entry : volMap.entrySet()) {
pathMap.put(entry.getKey(), entry.getValue().getBasePath());
}
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(pathMap);
+ return JsonUtil.toJsonString(pathMap);
} catch (DiskBalancerException ex) {
throw ex;
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index 3110c1a..a1c15ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.tools.DiskBalancer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +70,8 @@ import java.util.TreeSet;
* Common interface for command handling.
*/
public abstract class Command extends Configured {
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(HashMap.class);
static final Logger LOG = LoggerFactory.getLogger(Command.class);
private Map<String, String> validArgs = new HashMap<>();
private URI clusterURI;
@@ -441,11 +444,10 @@ public abstract class Command extends Configured {
ClientDatanodeProtocol dnClient = getDataNodeProxy(dnAddress);
String volumeNameJson = dnClient.getDiskBalancerSetting(
DiskBalancerConstants.DISKBALANCER_VOLUME_NAME);
- ObjectMapper mapper = new ObjectMapper();
@SuppressWarnings("unchecked")
Map<String, String> volumeMap =
- mapper.readValue(volumeNameJson, HashMap.class);
+ READER.readValue(volumeNameJson);
for (DiskBalancerVolumeSet set : node.getVolumeSets().values()) {
for (DiskBalancerVolume vol : set.getVolumes()) {
if (volumeMap.containsKey(vol.getUuid())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
index cc79648..b47beff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.hdfs.server.diskbalancer.connectors;
import com.google.common.base.Preconditions;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel
.DiskBalancerDataNode;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.File;
import java.net.URL;
@@ -35,6 +37,8 @@ import java.util.List;
public class JsonNodeConnector implements ClusterConnector {
private static final Logger LOG =
LoggerFactory.getLogger(JsonNodeConnector.class);
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerCluster.class);
private final URL clusterURI;
/**
@@ -56,9 +60,7 @@ public class JsonNodeConnector implements ClusterConnector {
Preconditions.checkNotNull(this.clusterURI);
String dataFilePath = this.clusterURI.getPath();
LOG.info("Reading cluster info from file : " + dataFilePath);
- ObjectMapper mapper = new ObjectMapper();
- DiskBalancerCluster cluster =
- mapper.readValue(new File(dataFilePath), DiskBalancerCluster.class);
+ DiskBalancerCluster cluster = READER.readValue(new File(dataFilePath));
String message = String.format("Found %d node(s)",
cluster.getNodes().size());
LOG.info(message);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
index 17a6ebb..8d7fb2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
import com.google.common.base.Preconditions;
+
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,9 +26,11 @@ import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.Planner;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.PlannerFactory;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.File;
import java.io.IOException;
@@ -69,6 +72,8 @@ public class DiskBalancerCluster {
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerCluster.class);
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerCluster.class);
private final Set<String> exclusionList;
private final Set<String> inclusionList;
private ClusterConnector clusterConnector;
@@ -118,8 +123,7 @@ public class DiskBalancerCluster {
* @throws IOException
*/
public static DiskBalancerCluster parseJson(String json) throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerCluster.class);
+ return READER.readValue(json);
}
/**
@@ -232,8 +236,7 @@ public class DiskBalancerCluster {
* @throws IOException
*/
public String toJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
+ return JsonUtil.toJsonString(this);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
index a6a8bdc..8b627b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.IOException;
@@ -30,6 +32,9 @@ import java.io.IOException;
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class DiskBalancerVolume {
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerVolume.class);
+
private String path;
private long capacity;
private String storageType;
@@ -58,8 +63,7 @@ public class DiskBalancerVolume {
* @throws IOException
*/
public static DiskBalancerVolume parseJson(String json) throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerVolume.class);
+ return READER.readValue(json);
}
/**
@@ -305,8 +309,7 @@ public class DiskBalancerVolume {
* @throws IOException
*/
public String toJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
+ return JsonUtil.toJsonString(this);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org