You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/09/08 14:57:55 UTC
[30/48] hadoop git commit: HDFS-9038. DFS reserved space is
erroneously counted towards non-DFS used. (Brahma Reddy Battula)
HDFS-9038. DFS reserved space is erroneously counted towards non-DFS used. (Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f23abfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f23abfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f23abfa
Branch: refs/heads/HADOOP-13341
Commit: 5f23abfa30ea29a5474513c463b4d462c0e824ee
Parents: f6c0b75
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 6 13:37:21 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 6 13:37:21 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 33 +++++++++++++---
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 41 +++++++++++++-------
.../hdfs/server/protocol/StorageReport.java | 10 ++++-
.../src/main/proto/hdfs.proto | 2 +
.../blockmanagement/DatanodeDescriptor.java | 3 ++
.../server/blockmanagement/DatanodeStats.java | 6 ++-
.../blockmanagement/DatanodeStorageInfo.java | 8 +++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 3 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 37 ++++++++++++++++--
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 24 ++++++++++++
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../blockmanagement/TestBlockManager.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../extdataset/ExternalDatasetImpl.java | 2 +-
.../fsdataset/impl/TestFsVolumeList.java | 40 +++++++++++++++++++
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../namenode/TestNamenodeCapacityReport.java | 36 ++++++++++++++---
.../namenode/metrics/TestNameNodeMetrics.java | 4 +-
19 files changed, 219 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index cd32a53..e9ee8b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -43,6 +43,7 @@ import static org.apache.hadoop.hdfs.DFSUtilClient.percent2String;
public class DatanodeInfo extends DatanodeID implements Node {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private long remaining;
private long blockPoolUsed;
private long cacheCapacity;
@@ -89,6 +90,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(from);
this.capacity = from.getCapacity();
this.dfsUsed = from.getDfsUsed();
+ this.nonDfsUsed = from.getNonDfsUsed();
this.remaining = from.getRemaining();
this.blockPoolUsed = from.getBlockPoolUsed();
this.cacheCapacity = from.getCacheCapacity();
@@ -105,6 +107,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(nodeID);
this.capacity = 0L;
this.dfsUsed = 0L;
+ this.nonDfsUsed = 0L;
this.remaining = 0L;
this.blockPoolUsed = 0L;
this.cacheCapacity = 0L;
@@ -158,10 +161,26 @@ public class DatanodeInfo extends DatanodeID implements Node {
final int xceiverCount, final String networkLocation,
final AdminStates adminState,
final String upgradeDomain) {
- super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
- infoSecurePort, ipcPort);
+ this(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort, capacity, dfsUsed, 0L, remaining, blockPoolUsed,
+ cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic,
+ xceiverCount, networkLocation, adminState, upgradeDomain);
+ }
+
+ /** Constructor. */
+ public DatanodeInfo(final String ipAddr, final String hostName,
+ final String datanodeUuid, final int xferPort, final int infoPort,
+ final int infoSecurePort, final int ipcPort, final long capacity,
+ final long dfsUsed, final long nonDfsUsed, final long remaining,
+ final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+ final long lastUpdate, final long lastUpdateMonotonic,
+ final int xceiverCount, final String networkLocation,
+ final AdminStates adminState, final String upgradeDomain) {
+ super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort);
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = blockPoolUsed;
this.cacheCapacity = cacheCapacity;
@@ -174,7 +193,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.upgradeDomain = upgradeDomain;
}
- /** Network location name */
+ /** Network location name. */
@Override
public String getName() {
return getXferAddr();
@@ -191,8 +210,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
/** The used space by the data node. */
public long getNonDfsUsed() {
- long nonDFSUsed = capacity - dfsUsed - remaining;
- return nonDFSUsed < 0 ? 0 : nonDFSUsed;
+ return nonDfsUsed;
}
/** The used space by the data node as percentage of present capacity */
@@ -282,6 +300,11 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.dfsUsed = dfsUsed;
}
+ /** Sets the nondfs-used space for the datanode. */
+ public void setNonDfsUsed(long nonDfsUsed) {
+ this.nonDfsUsed = nonDfsUsed;
+ }
+
/** Sets raw free space. */
public void setRemaining(long remaining) {
this.remaining = remaining;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 7df68e7..246b5a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -289,6 +289,7 @@ public class PBHelperClient {
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
.setDfsUsed(info.getDfsUsed())
+ .setNonDfsUsed(info.getNonDfsUsed())
.setRemaining(info.getRemaining())
.setBlockPoolUsed(info.getBlockPoolUsed())
.setCacheCapacity(info.getCacheCapacity())
@@ -581,15 +582,24 @@ public class PBHelperClient {
}
static public DatanodeInfo convert(DatanodeInfoProto di) {
- if (di == null) return null;
- return new DatanodeInfo(
- convert(di.getId()),
- di.hasLocation() ? di.getLocation() : null,
- di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
- di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
- di.getLastUpdate(), di.getLastUpdateMonotonic(),
- di.getXceiverCount(), convert(di.getAdminState()),
+ if (di == null) {
+ return null;
+ }
+ DatanodeInfo dinfo = new DatanodeInfo(convert(di.getId()),
+ di.hasLocation() ? di.getLocation() : null, di.getCapacity(),
+ di.getDfsUsed(), di.getRemaining(), di.getBlockPoolUsed(),
+ di.getCacheCapacity(), di.getCacheUsed(), di.getLastUpdate(),
+ di.getLastUpdateMonotonic(), di.getXceiverCount(),
+ convert(di.getAdminState()),
di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
+ if (di.hasNonDfsUsed()) {
+ dinfo.setNonDfsUsed(di.getNonDfsUsed());
+ } else {
+ // use the legacy way for older datanodes
+ long nonDFSUsed = di.getCapacity() - di.getDfsUsed() - di.getRemaining();
+ dinfo.setNonDfsUsed(nonDFSUsed < 0 ? 0 : nonDFSUsed);
+ }
+ return dinfo;
}
public static StorageType[] convertStorageTypes(
@@ -1556,12 +1566,12 @@ public class PBHelperClient {
}
public static StorageReport convert(StorageReportProto p) {
- return new StorageReport(
- p.hasStorage() ?
- convert(p.getStorage()) :
- new DatanodeStorage(p.getStorageUuid()),
- p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
- p.getBlockPoolUsed());
+ long nonDfsUsed = p.hasNonDfsUsed() ? p.getNonDfsUsed() : p.getCapacity()
+ - p.getDfsUsed() - p.getRemaining();
+ return new StorageReport(p.hasStorage() ? convert(p.getStorage())
+ : new DatanodeStorage(p.getStorageUuid()), p.getFailed(),
+ p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
+ p.getBlockPoolUsed(), nonDfsUsed);
}
public static DatanodeStorage convert(DatanodeStorageProto s) {
@@ -2136,7 +2146,8 @@ public class PBHelperClient {
.setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
.setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
.setStorageUuid(r.getStorage().getStorageID())
- .setStorage(convert(r.getStorage()));
+ .setStorage(convert(r.getStorage()))
+ .setNonDfsUsed(r.getNonDfsUsed());
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
index 3fbc424..7042b58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
@@ -25,17 +25,19 @@ public class StorageReport {
private final boolean failed;
private final long capacity;
private final long dfsUsed;
+ private final long nonDfsUsed;
private final long remaining;
private final long blockPoolUsed;
public static final StorageReport[] EMPTY_ARRAY = {};
- public StorageReport(DatanodeStorage storage, boolean failed,
- long capacity, long dfsUsed, long remaining, long bpUsed) {
+ public StorageReport(DatanodeStorage storage, boolean failed, long capacity,
+ long dfsUsed, long remaining, long bpUsed, long nonDfsUsed) {
this.storage = storage;
this.failed = failed;
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = bpUsed;
}
@@ -56,6 +58,10 @@ public class StorageReport {
return dfsUsed;
}
+ public long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
public long getRemaining() {
return remaining;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 7de8550..1141fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -101,6 +101,7 @@ message DatanodeInfoProto {
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
optional string upgradeDomain = 14;
+ optional uint64 nonDfsUsed = 15;
}
/**
@@ -125,6 +126,7 @@ message StorageReportProto {
optional uint64 remaining = 5 [ default = 0 ];
optional uint64 blockPoolUsed = 6 [ default = 0 ];
optional DatanodeStorageProto storage = 7; // supersedes StorageUuid
+ optional uint64 nonDfsUsed = 8;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 1646129..c74d7c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -417,6 +417,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
long totalRemaining = 0;
long totalBlockPoolUsed = 0;
long totalDfsUsed = 0;
+ long totalNonDfsUsed = 0;
Set<DatanodeStorageInfo> failedStorageInfos = null;
// Decide if we should check for any missing StorageReport and mark it as
@@ -477,6 +478,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
totalRemaining += report.getRemaining();
totalBlockPoolUsed += report.getBlockPoolUsed();
totalDfsUsed += report.getDfsUsed();
+ totalNonDfsUsed += report.getNonDfsUsed();
}
rollBlocksScheduled(getLastUpdateMonotonic());
@@ -485,6 +487,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
setRemaining(totalRemaining);
setBlockPoolUsed(totalBlockPoolUsed);
setDfsUsed(totalDfsUsed);
+ setNonDfsUsed(totalNonDfsUsed);
if (checkFailedStorages) {
updateFailedStorage(failedStorageInfos);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
index 0d4e235..8386b27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
@@ -35,6 +35,7 @@ class DatanodeStats {
private final StorageTypeStatsMap statsMap = new StorageTypeStatsMap();
private long capacityTotal = 0L;
private long capacityUsed = 0L;
+ private long capacityUsedNonDfs = 0L;
private long capacityRemaining = 0L;
private long blockPoolUsed = 0L;
private int xceiverCount = 0;
@@ -49,6 +50,7 @@ class DatanodeStats {
xceiverCount += node.getXceiverCount();
if (node.isInService()) {
capacityUsed += node.getDfsUsed();
+ capacityUsedNonDfs += node.getNonDfsUsed();
blockPoolUsed += node.getBlockPoolUsed();
nodesInService++;
nodesInServiceXceiverCount += node.getXceiverCount();
@@ -77,6 +79,7 @@ class DatanodeStats {
xceiverCount -= node.getXceiverCount();
if (node.isInService()) {
capacityUsed -= node.getDfsUsed();
+ capacityUsedNonDfs -= node.getNonDfsUsed();
blockPoolUsed -= node.getBlockPoolUsed();
nodesInService--;
nodesInServiceXceiverCount -= node.getXceiverCount();
@@ -159,8 +162,7 @@ class DatanodeStats {
}
synchronized long getCapacityUsedNonDFS() {
- final long nonDFSUsed = capacityTotal - capacityRemaining - capacityUsed;
- return nonDFSUsed < 0L? 0L : nonDFSUsed;
+ return capacityUsedNonDfs;
}
synchronized float getCapacityUsedPercent() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 843a8d5..d98a2c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -93,6 +93,7 @@ public class DatanodeStorageInfo {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private volatile long remaining;
private long blockPoolUsed;
@@ -202,6 +203,10 @@ public class DatanodeStorageInfo {
return dfsUsed;
}
+ long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
long getRemaining() {
return remaining;
}
@@ -283,6 +288,7 @@ public class DatanodeStorageInfo {
void updateState(StorageReport r) {
capacity = r.getCapacity();
dfsUsed = r.getDfsUsed();
+ nonDfsUsed = r.getNonDfsUsed();
remaining = r.getRemaining();
blockPoolUsed = r.getBlockPoolUsed();
}
@@ -332,7 +338,7 @@ public class DatanodeStorageInfo {
StorageReport toStorageReport() {
return new StorageReport(
new DatanodeStorage(storageID, state, storageType),
- false, capacity, dfsUsed, remaining, blockPoolUsed);
+ false, capacity, dfsUsed, remaining, blockPoolUsed, nonDfsUsed);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index e0d2baf..ffd2f8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -164,7 +164,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volume.getCapacity(),
volume.getDfsUsed(),
volume.getAvailable(),
- volume.getBlockPoolUsed(bpid));
+ volume.getBlockPoolUsed(bpid),
+ volume.getNonDfsUsed());
reports.add(sr);
} catch (ClosedChannelException e) {
continue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 59c9ed4..afcc5dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -385,15 +385,46 @@ public class FsVolumeImpl implements FsVolumeSpi {
*/
@Override
public long getAvailable() throws IOException {
- long remaining = getCapacity() - getDfsUsed() - reservedForReplicas.get();
- long available = usage.getAvailable() - reserved
- - reservedForReplicas.get();
+ long remaining = getCapacity() - getDfsUsed() - getReservedForReplicas();
+ long available = usage.getAvailable() - getRemainingReserved()
+ - getReservedForReplicas();
if (remaining > available) {
remaining = available;
}
return (remaining > 0) ? remaining : 0;
}
+ long getActualNonDfsUsed() throws IOException {
+ return usage.getUsed() - getDfsUsed();
+ }
+
+ private long getRemainingReserved() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return reserved - actualNonDfsUsed;
+ }
+ return 0L;
+ }
+
+ /**
+ * Unplanned Non-DFS usage, i.e. Extra usage beyond reserved.
+ *
+ * @return
+ * @throws IOException
+ */
+ public long getNonDfsUsed() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return 0L;
+ }
+ return actualNonDfsUsed - reserved;
+ }
+
+ @VisibleForTesting
+ long getDfAvailable() {
+ return usage.getAvailable();
+ }
+
@VisibleForTesting
public long getReservedForReplicas() {
return reservedForReplicas.get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 45cd4ea..af17756 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -720,6 +720,28 @@ public class TestPBHelper {
assertBlockECRecoveryInfoEquals(blkECRecoveryInfo1, iterator.next());
}
+ @Test
+ public void testDataNodeInfoPBHelper() {
+ DatanodeID id = DFSTestUtil.getLocalDatanodeID();
+ DatanodeInfo dnInfos0 = new DatanodeInfo(id);
+ dnInfos0.setCapacity(3500L);
+ dnInfos0.setDfsUsed(1000L);
+ dnInfos0.setNonDfsUsed(2000L);
+ dnInfos0.setRemaining(500L);
+ HdfsProtos.DatanodeInfoProto dnproto = PBHelperClient.convert(dnInfos0);
+ DatanodeInfo dnInfos1 = PBHelperClient.convert(dnproto);
+ compare(dnInfos0, dnInfos1);
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos1.getNonDfsUsed());
+
+ //Testing without nonDfs field
+ HdfsProtos.DatanodeInfoProto.Builder b =
+ HdfsProtos.DatanodeInfoProto.newBuilder();
+ b.setId(PBHelperClient.convert(id)).setCapacity(3500L).setDfsUsed(1000L)
+ .setRemaining(500L);
+ DatanodeInfo dnInfos3 = PBHelperClient.convert(b.build());
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos3.getNonDfsUsed());
+ }
+
private void assertBlockECRecoveryInfoEquals(
BlockECReconstructionInfo blkECRecoveryInfo1,
BlockECReconstructionInfo blkECRecoveryInfo2) {
@@ -768,4 +790,6 @@ public class TestPBHelper {
compare(dnInfos1[i], dnInfos2[i]);
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 1b565ee..5fb3b25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -294,7 +294,7 @@ public class BlockManagerTestUtil {
StorageReport report = new StorageReport(
dns ,false, storage.getCapacity(),
storage.getDfsUsed(), storage.getRemaining(),
- storage.getBlockPoolUsed());
+ storage.getBlockPoolUsed(), 0);
reports.add(report);
}
return reports.toArray(StorageReport.EMPTY_ARRAY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 1f58f99..942569a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1200,7 +1200,7 @@ public class TestBlockManager {
StorageReport report = new StorageReport(
dns, true, storageInfo.getCapacity(),
storageInfo.getDfsUsed(), storageInfo.getRemaining(),
- storageInfo.getBlockPoolUsed());
+ storageInfo.getBlockPoolUsed(), 0L);
reports.add(report);
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index c0c76ad..a082fbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -441,7 +441,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
synchronized StorageReport getStorageReport(String bpid) {
return new StorageReport(dnStorage,
false, getCapacity(), getUsed(), getFree(),
- map.get(bpid).getUsed());
+ map.get(bpid).getUsed(), 0L);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index b8564bb..fcd960a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -72,7 +72,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
@Override
public StorageReport[] getStorageReports(String bpid) throws IOException {
StorageReport[] result = new StorageReport[1];
- result[0] = new StorageReport(storage, false, 0, 0, 0, 0);
+ result[0] = new StorageReport(storage, false, 0, 0, 0, 0, 0);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 796d249..3d4c38c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
@@ -177,4 +178,43 @@ public class TestFsVolumeList {
conf, StorageType.DEFAULT);
assertEquals("", 100L, volume4.getReserved());
}
+
+ @Test
+ public void testNonDfsUsedMetricForVolume() throws Exception {
+ File volDir = new File(baseDir, "volume-0");
+ volDir.mkdirs();
+ /*
+ * Lets have the example.
+ * Capacity - 1000
+ * Reserved - 100
+ * DfsUsed - 200
+ * Actual Non-DfsUsed - 300 -->(expected)
+ * ReservedForReplicas - 50
+ */
+ long diskCapacity = 1000L;
+ long duReserved = 100L;
+ long dfsUsage = 200L;
+ long actualNonDfsUsage = 300L;
+ long reservedForReplicas = 50L;
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, duReserved);
+ FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf,
+ StorageType.DEFAULT);
+ FsVolumeImpl spyVolume = Mockito.spy(volume);
+ // Set Capacity for testing
+ long testCapacity = diskCapacity - duReserved;
+ spyVolume.setCapacityForTesting(testCapacity);
+ // Mock volume.getDfAvailable()
+ long dfAvailable = diskCapacity - dfsUsage - actualNonDfsUsage;
+ Mockito.doReturn(dfAvailable).when(spyVolume).getDfAvailable();
+ // Mock dfsUsage
+ Mockito.doReturn(dfsUsage).when(spyVolume).getDfsUsed();
+ // Mock reservedForReplcas
+ Mockito.doReturn(reservedForReplicas).when(spyVolume)
+ .getReservedForReplicas();
+ Mockito.doReturn(actualNonDfsUsage).when(spyVolume)
+ .getActualNonDfsUsed();
+ long expectedNonDfsUsage =
+ actualNonDfsUsage - duReserved;
+ assertEquals(expectedNonDfsUsage, spyVolume.getNonDfsUsed());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index be2a678..ad336ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -949,7 +949,7 @@ public class NNThroughputBenchmark implements Tool {
// register datanode
// TODO:FEDERATION currently a single block pool is supported
StorageReport[] rep = { new StorageReport(storage, false,
- DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0L) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration, rep,
0L, 0L, 0, 0, 0, null, true).getCommands();
if(cmds != null) {
@@ -998,7 +998,7 @@ public class NNThroughputBenchmark implements Tool {
int replicateBlocks() throws IOException {
// register datanode
StorageReport[] rep = { new StorageReport(storage,
- false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration,
rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
if (cmds != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index 341a149..14b14b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -128,7 +128,7 @@ public class TestDeadDatanode {
// that asks datanode to register again
StorageReport[] rep = { new StorageReport(
new DatanodeStorage(reg.getDatanodeUuid()),
- false, 0, 0, 0, 0) };
+ false, 0, 0, 0, 0, 0) };
DatanodeCommand[] cmd =
dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
assertEquals(1, cmd.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
index 99323f5..38ec9f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
@@ -71,7 +71,6 @@ public class TestNamenodeCapacityReport {
try {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
-
final FSNamesystem namesystem = cluster.getNamesystem();
final DatanodeManager dm = cluster.getNamesystem().getBlockManager(
).getDatanodeManager();
@@ -100,8 +99,9 @@ public class TestNamenodeCapacityReport {
+ " used " + used + " non DFS used " + nonDFSUsed
+ " remaining " + remaining + " perentUsed " + percentUsed
+ " percentRemaining " + percentRemaining);
-
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
configCapacity));
assertTrue(percentRemaining == DFSUtilClient.getPercentRemaining(
@@ -148,7 +148,9 @@ public class TestNamenodeCapacityReport {
assertTrue(configCapacity == diskCapacity - reserved);
// Ensure new total capacity reported excludes the reserved space
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
@@ -160,9 +162,33 @@ public class TestNamenodeCapacityReport {
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentRemaining == ((float)remaining * 100.0f)/(float)configCapacity);
+
+ //Adding testcase for non-dfs used where we need to consider
+ // reserved replica also.
+ final int fileCount = 5;
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ // create streams and hsync to force datastreamers to start
+ DFSOutputStream[] streams = new DFSOutputStream[fileCount];
+ for (int i=0; i < fileCount; i++) {
+ streams[i] = (DFSOutputStream)fs.create(new Path("/f"+i))
+ .getWrappedStream();
+ streams[i].write("1".getBytes());
+ streams[i].hsync();
+ }
+ triggerHeartbeats(cluster.getDataNodes());
+ assertTrue(configCapacity > (namesystem.getCapacityUsed() + namesystem
+ .getCapacityRemaining() + namesystem.getNonDfsUsedSpace()));
+ // There is a chance that nonDFS usage might have slightly due to
+ // testlogs, So assume 1MB other files used within this gap
+ assertTrue(
+ (namesystem.getCapacityUsed() + namesystem.getCapacityRemaining()
+ + namesystem.getNonDfsUsedSpace() + fileCount * fs
+ .getDefaultBlockSize()) - configCapacity < 1 * 1024);
}
finally {
- if (cluster != null) {cluster.shutdown();}
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index c4a9837..9e95650 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -166,7 +166,9 @@ public class TestNameNodeMetrics {
MetricsAsserts.getLongGauge("CapacityRemaining", rb);
long capacityUsedNonDFS =
MetricsAsserts.getLongGauge("CapacityUsedNonDFS", rb);
- assert(capacityUsed + capacityRemaining + capacityUsedNonDFS ==
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assert (capacityUsed + capacityRemaining + capacityUsedNonDFS <=
capacityTotal);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org