You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/10/28 18:04:23 UTC
[25/50] [abbrv] hadoop git commit: HDFS-11047. Remove deep copies of
FinalizedReplica to alleviate heap consumption on DataNode. Contributed by
Xiaobing Zhou
HDFS-11047. Remove deep copies of FinalizedReplica to alleviate heap consumption on DataNode. Contributed by Xiaobing Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e03ee52
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e03ee52
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e03ee52
Branch: refs/heads/HDFS-9806
Commit: 9e03ee527988ff85af7f2c224c5570b69d09279a
Parents: f3ac1f4
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Oct 27 15:58:09 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Oct 27 16:00:27 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/DirectoryScanner.java | 14 +++++++-------
.../server/datanode/fsdataset/FsDatasetSpi.java | 11 ++++++++++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 16 +++++++++++-----
3 files changed, 28 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e03ee52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 58071dc..e2baf32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -398,14 +399,13 @@ public class DirectoryScanner implements Runnable {
diffs.put(bpid, diffRecord);
statsRecord.totalBlocks = blockpoolReport.length;
- List<ReplicaInfo> bl = dataset.getFinalizedBlocks(bpid);
- ReplicaInfo[] memReport = bl.toArray(new ReplicaInfo[bl.size()]);
- Arrays.sort(memReport); // Sort based on blockId
+ final List<ReplicaInfo> bl = dataset.getFinalizedBlocks(bpid);
+ Collections.sort(bl); // Sort based on blockId
int d = 0; // index for blockpoolReport
int m = 0; // index for memReprot
- while (m < memReport.length && d < blockpoolReport.length) {
- ReplicaInfo memBlock = memReport[m];
+ while (m < bl.size() && d < blockpoolReport.length) {
+ ReplicaInfo memBlock = bl.get(m);
ScanInfo info = blockpoolReport[d];
if (info.getBlockId() < memBlock.getBlockId()) {
if (!dataset.isDeletingBlock(bpid, info.getBlockId())) {
@@ -452,8 +452,8 @@ public class DirectoryScanner implements Runnable {
++m;
}
}
- while (m < memReport.length) {
- ReplicaInfo current = memReport[m++];
+ while (m < bl.size()) {
+ ReplicaInfo current = bl.get(m++);
addDifference(diffRecord, statsRecord,
current.getBlockId(), current.getVolume());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e03ee52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index f2ffa83..e113212 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -229,7 +229,16 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
*/
VolumeFailureSummary getVolumeFailureSummary();
- /** @return a list of finalized blocks for the given block pool. */
+ /**
+ * Gets a list of references to the finalized blocks for the given block pool.
+ * <p>
+ * Callers of this function should call
+ * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being
+ * changed during list iteration.
+ * </p>
+ * @return a list of references to the finalized blocks for the given block
+ * pool.
+ */
List<ReplicaInfo> getFinalizedBlocks(String bpid);
/** @return a list of finalized blocks for the given block pool. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e03ee52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index abeda2c..0f40f2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1714,17 +1714,23 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
/**
- * Get the list of finalized blocks from in-memory blockmap for a block pool.
+ * Gets a list of references to the finalized blocks for the given block pool.
+ * <p>
+ * Callers of this function should call
+ * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being
+ * changed during list iteration.
+ * </p>
+ * @return a list of references to the finalized blocks for the given block
+ * pool.
*/
@Override
public List<ReplicaInfo> getFinalizedBlocks(String bpid) {
try (AutoCloseableLock lock = datasetLock.acquire()) {
- ArrayList<ReplicaInfo> finalized =
- new ArrayList<ReplicaInfo>(volumeMap.size(bpid));
+ final List<ReplicaInfo> finalized = new ArrayList<ReplicaInfo>(
+ volumeMap.size(bpid));
for (ReplicaInfo b : volumeMap.replicas(bpid)) {
if (b.getState() == ReplicaState.FINALIZED) {
- finalized.add(new ReplicaBuilder(ReplicaState.FINALIZED)
- .from(b).build());
+ finalized.add(b);
}
}
return finalized;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org