You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ms...@apache.org on 2018/05/18 08:39:39 UTC
hadoop git commit: HDDS-76. Modify SCMStorageReportProto to include
the data dir paths as well as the StorageType info. Contributed by Shashikant
Banerjee.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3159bffce -> 6e996867f
HDDS-76. Modify SCMStorageReportProto to include the data dir paths as well as the StorageType info. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e996867
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e996867
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e996867
Branch: refs/heads/trunk
Commit: 6e996867f641b297e3f88068f6d185b709d509b0
Parents: 3159bff
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Fri May 18 14:08:46 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Fri May 18 14:08:46 2018 +0530
----------------------------------------------------------------------
.../impl/ContainerLocationManagerImpl.java | 18 +-
.../common/impl/ContainerManagerImpl.java | 9 +-
.../common/impl/ContainerStorageLocation.java | 9 +
.../common/impl/StorageLocationReport.java | 235 ++++++++++++++++++-
.../StorageContainerDatanodeProtocol.proto | 21 +-
5 files changed, 275 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e996867/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
index e0e826c..5f5b81f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
@@ -108,12 +108,14 @@ public class ContainerLocationManagerImpl implements ContainerLocationManager,
@Override
public StorageLocationReport[] getLocationReport() throws IOException {
+ boolean failed;
StorageLocationReport[] reports =
new StorageLocationReport[dataLocations.size()];
for (int idx = 0; idx < dataLocations.size(); idx++) {
ContainerStorageLocation loc = dataLocations.get(idx);
long scmUsed = 0;
long remaining = 0;
+ failed = false;
try {
scmUsed = loc.getScmUsed();
remaining = loc.getAvailable();
@@ -123,13 +125,19 @@ public class ContainerLocationManagerImpl implements ContainerLocationManager,
// reset scmUsed and remaining if df/du failed.
scmUsed = 0;
remaining = 0;
+ failed = true;
}
- // TODO: handle failed storage
- // For now, include storage report for location that failed to get df/du.
- StorageLocationReport r = new StorageLocationReport(
- loc.getStorageUuId(), false, loc.getCapacity(),
- scmUsed, remaining);
+ StorageLocationReport.Builder builder =
+ StorageLocationReport.newBuilder();
+ builder.setStorageLocation(loc.getStorageLocation())
+ .setId(loc.getStorageUuId())
+ .setFailed(failed)
+ .setCapacity(loc.getCapacity())
+ .setRemaining(remaining)
+ .setScmUsed(scmUsed)
+ .setStorageType(loc.getStorageType());
+ StorageLocationReport r = builder.build();
reports[idx] = r;
}
return reports;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e996867/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index 240beba..039b4c3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
@@ -38,6 +39,8 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMStorageReport;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.StorageTypeProto;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
@@ -818,11 +821,7 @@ public class ContainerManagerImpl implements ContainerManager {
SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
for (int i = 0; i < reports.length; i++) {
SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
- nrb.addStorageReport(i, srb.setStorageUuid(reports[i].getId())
- .setCapacity(reports[i].getCapacity())
- .setScmUsed(reports[i].getScmUsed())
- .setRemaining(reports[i].getRemaining())
- .build());
+ nrb.addStorageReport(reports[i].getProtoBufMessage());
}
return nrb.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e996867/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
index 7293895..7431baa 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CachingGetSpaceUsed;
import org.apache.hadoop.fs.DF;
import org.apache.hadoop.fs.GetSpaceUsed;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.io.IOUtils;
@@ -121,6 +122,14 @@ public class ContainerStorageLocation {
return scmUsage.getUsed();
}
+ public String getStorageLocation() {
+ return getNormalizedUri().getRawPath();
+ }
+
+ public StorageType getStorageType() {
+ return dataLocation.getStorageType();
+ }
+
public void shutdown() {
saveScmUsed();
scmUsedSaved = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e996867/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
index 7ef91a9..a5ad6c2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
@@ -18,26 +18,38 @@
package org.apache.hadoop.ozone.container.common.impl;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.protocol.proto.
+ StorageContainerDatanodeProtocolProtos.SCMStorageReport;
+import org.apache.hadoop.hdds.protocol.proto.
+ StorageContainerDatanodeProtocolProtos.StorageTypeProto;
+
+import java.io.IOException;
+
/**
* Storage location stats of datanodes that provide back store for containers.
*
*/
public class StorageLocationReport {
- public static final StorageLocationReport[] EMPTY_ARRAY = {};
private final String id;
private final boolean failed;
private final long capacity;
private final long scmUsed;
private final long remaining;
+ private final StorageType storageType;
+ private final String storageLocation;
- public StorageLocationReport(String id, boolean failed,
- long capacity, long scmUsed, long remaining) {
+ private StorageLocationReport(String id, boolean failed, long capacity,
+ long scmUsed, long remaining, StorageType storageType,
+ String storageLocation) {
this.id = id;
this.failed = failed;
this.capacity = capacity;
this.scmUsed = scmUsed;
this.remaining = remaining;
+ this.storageType = storageType;
+ this.storageLocation = storageLocation;
}
public String getId() {
@@ -60,4 +72,221 @@ public class StorageLocationReport {
return remaining;
}
+ public String getStorageLocation() {
+ return storageLocation;
+ }
+
+ public StorageType getStorageType() {
+ return storageType;
+ }
+
+
+ private StorageTypeProto getStorageTypeProto() throws
+ IOException {
+ StorageTypeProto storageTypeProto;
+ switch (getStorageType()) {
+ case SSD:
+ storageTypeProto = StorageTypeProto.SSD;
+ break;
+ case DISK:
+ storageTypeProto = StorageTypeProto.DISK;
+ break;
+ case ARCHIVE:
+ storageTypeProto = StorageTypeProto.ARCHIVE;
+ break;
+ case PROVIDED:
+ storageTypeProto = StorageTypeProto.PROVIDED;
+ break;
+ case RAM_DISK:
+ storageTypeProto = StorageTypeProto.RAM_DISK;
+ break;
+ default:
+ throw new IOException("Illegal Storage Type specified");
+ }
+ return storageTypeProto;
+ }
+
+ private static StorageType getStorageType(StorageTypeProto proto) throws
+ IOException {
+ StorageType storageType;
+ switch (proto) {
+ case SSD:
+ storageType = StorageType.SSD;
+ break;
+ case DISK:
+ storageType = StorageType.DISK;
+ break;
+ case ARCHIVE:
+ storageType = StorageType.ARCHIVE;
+ break;
+ case PROVIDED:
+ storageType = StorageType.PROVIDED;
+ break;
+ case RAM_DISK:
+ storageType = StorageType.RAM_DISK;
+ break;
+ default:
+ throw new IOException("Illegal Storage Type specified");
+ }
+ return storageType;
+ }
+
+ /**
+ * Returns the SCMStorageReport protoBuf message for the Storage Location
+ * report.
+ * @return SCMStorageReport
+ * @throws IOException In case, the storage type specified is invalid.
+ */
+ public SCMStorageReport getProtoBufMessage() throws IOException{
+ SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
+ return srb.setStorageUuid(getId())
+ .setCapacity(getCapacity())
+ .setScmUsed(getScmUsed())
+ .setRemaining(getRemaining())
+ .setStorageType(getStorageTypeProto())
+ .setStorageLocation(getStorageLocation())
+ .setFailed(isFailed())
+ .build();
+ }
+
+ /**
+ * Returns the StorageLocationReport from the protoBuf message.
+ * @param report SCMStorageReport
+ * @return StorageLocationReport
+ * @throws IOException in case of invalid storage type
+ */
+
+ public static StorageLocationReport getFromProtobuf(SCMStorageReport report)
+ throws IOException {
+ StorageLocationReport.Builder builder = StorageLocationReport.newBuilder();
+ builder.setId(report.getStorageUuid())
+ .setStorageLocation(report.getStorageLocation());
+ if (report.hasCapacity()) {
+ builder.setCapacity(report.getCapacity());
+ }
+ if (report.hasScmUsed()) {
+ builder.setScmUsed(report.getScmUsed());
+ }
+ if (report.hasStorageType()) {
+ builder.setStorageType(getStorageType(report.getStorageType()));
+ }
+ if (report.hasRemaining()) {
+ builder.setRemaining(report.getRemaining());
+ }
+
+ if (report.hasFailed()) {
+ builder.setFailed(report.getFailed());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns StorageLocation.Builder instance.
+ *
+ * @return StorageLocation.Builder
+ */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /**
+ * Builder class for building StorageLocationReport.
+ */
+ public static class Builder {
+ private String id;
+ private boolean failed;
+ private long capacity;
+ private long scmUsed;
+ private long remaining;
+ private StorageType storageType;
+ private String storageLocation;
+
+ /**
+ * Sets the storageId.
+ *
+ * @param id storageId
+ * @return StorageLocationReport.Builder
+ */
+ public Builder setId(String id) {
+ this.id = id;
+ return this;
+ }
+
+ /**
+ * Sets whether the volume failed or not.
+ *
+ * @param failed whether volume failed or not
+ * @return StorageLocationReport.Builder
+ */
+ public Builder setFailed(boolean failed) {
+ this.failed = failed;
+ return this;
+ }
+
+ /**
+ * Sets the capacity of volume.
+ *
+ * @param capacity capacity
+ * @return StorageLocationReport.Builder
+ */
+ public Builder setCapacity(long capacity) {
+ this.capacity = capacity;
+ return this;
+ }
+ /**
+ * Sets the scmUsed Value.
+ *
+ * @param scmUsed storage space used by scm
+ * @return StorageLocationReport.Builder
+ */
+ public Builder setScmUsed(long scmUsed) {
+ this.scmUsed = scmUsed;
+ return this;
+ }
+
+ /**
+ * Sets the remaining free space value.
+ *
+ * @param remaining remaining free space
+ * @return StorageLocationReport.Builder
+ */
+ public Builder setRemaining(long remaining) {
+ this.remaining = remaining;
+ return this;
+ }
+
+ /**
+ * Sets the storageType.
+ *
+ * @param storageType type of the storage used
+ * @return StorageLocationReport.Builder
+ */
+ public Builder setStorageType(StorageType storageType) {
+ this.storageType = storageType;
+ return this;
+ }
+
+ /**
+ * Sets the storageLocation.
+ *
+ * @param storageLocation location of the volume
+ * @return StorageLocationReport.Builder
+ */
+ public Builder setStorageLocation(String storageLocation) {
+ this.storageLocation = storageLocation;
+ return this;
+ }
+
+ /**
+ * Builds and returns StorageLocationReport instance.
+ *
+ * @return StorageLocationReport
+ */
+ public StorageLocationReport build() {
+ return new StorageLocationReport(id, failed, capacity, scmUsed,
+ remaining, storageType, storageLocation);
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e996867/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index f44abc9..2b34d11 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -136,12 +136,25 @@ message SCMNodeReport {
repeated SCMStorageReport storageReport = 1;
}
+/**
+ * Types of recognized storage media.
+ */
+enum StorageTypeProto {
+ DISK = 1;
+ SSD = 2;
+ ARCHIVE = 3;
+ RAM_DISK = 4;
+ PROVIDED = 5;
+}
+
message SCMStorageReport {
required string storageUuid = 1;
- optional uint64 capacity = 2 [default = 0];
- optional uint64 scmUsed = 3 [default = 0];
- optional uint64 remaining = 4 [default = 0];
- //optional hadoop.hdfs.StorageTypeProto storageType = 5 [default = DISK];
+ required string storageLocation = 2;
+ optional uint64 capacity = 3 [default = 0];
+ optional uint64 scmUsed = 4 [default = 0];
+ optional uint64 remaining = 5 [default = 0];
+ optional StorageTypeProto storageType = 6 [default = DISK];
+ optional bool failed = 7 [default = false];
}
message SCMRegisterRequestProto {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org