You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/05/17 15:32:09 UTC
hadoop git commit: HDDS-77. Key replication factor and type should be
stored per key by Ozone Manager. Contributed by Mukul Kumar Singh.
Repository: hadoop
Updated Branches:
refs/heads/trunk 0ce6290de -> 41ae5c500
HDDS-77. Key replication factor and type should be stored per key by Ozone Manager. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41ae5c50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41ae5c50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41ae5c50
Branch: refs/heads/trunk
Commit: 41ae5c50024e3682b61df6f391d0fe63591ac92b
Parents: 0ce6290
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu May 17 08:31:44 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu May 17 08:31:44 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 36 ++++++++++++++++++--
...ceManagerProtocolClientSideTranslatorPB.java | 2 --
.../main/proto/KeySpaceManagerProtocol.proto | 10 +++---
.../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 19 +++--------
...ceManagerProtocolServerSideTranslatorPB.java | 6 ----
5 files changed, 44 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
index 678ce92..5d6e633 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.ksm.helpers;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.util.Time;
@@ -39,10 +40,13 @@ public final class KsmKeyInfo {
private List<KsmKeyLocationInfoGroup> keyLocationVersions;
private final long creationTime;
private long modificationTime;
+ private HddsProtos.ReplicationType type;
+ private HddsProtos.ReplicationFactor factor;
private KsmKeyInfo(String volumeName, String bucketName, String keyName,
List<KsmKeyLocationInfoGroup> versions, long dataSize,
- long creationTime, long modificationTime) {
+ long creationTime, long modificationTime, HddsProtos.ReplicationType type,
+ HddsProtos.ReplicationFactor factor) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.keyName = keyName;
@@ -61,6 +65,8 @@ public final class KsmKeyInfo {
this.keyLocationVersions = versions;
this.creationTime = creationTime;
this.modificationTime = modificationTime;
+ this.factor = factor;
+ this.type = type;
}
public String getVolumeName() {
@@ -71,6 +77,14 @@ public final class KsmKeyInfo {
return bucketName;
}
+ public HddsProtos.ReplicationType getType() {
+ return type;
+ }
+
+ public HddsProtos.ReplicationFactor getFactor() {
+ return factor;
+ }
+
public String getKeyName() {
return keyName;
}
@@ -170,6 +184,8 @@ public final class KsmKeyInfo {
private List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups;
private long creationTime;
private long modificationTime;
+ private HddsProtos.ReplicationType type;
+ private HddsProtos.ReplicationFactor factor;
public Builder setVolumeName(String volume) {
this.volumeName = volume;
@@ -207,10 +223,20 @@ public final class KsmKeyInfo {
return this;
}
+ public Builder setReplicationFactor(HddsProtos.ReplicationFactor factor) {
+ this.factor = factor;
+ return this;
+ }
+
+ public Builder setReplicationType(HddsProtos.ReplicationType type) {
+ this.type = type;
+ return this;
+ }
+
public KsmKeyInfo build() {
return new KsmKeyInfo(
volumeName, bucketName, keyName, ksmKeyLocationInfoGroups,
- dataSize, creationTime, modificationTime);
+ dataSize, creationTime, modificationTime, type, factor);
}
}
@@ -222,6 +248,8 @@ public final class KsmKeyInfo {
.setBucketName(bucketName)
.setKeyName(keyName)
.setDataSize(dataSize)
+ .setFactor(factor)
+ .setType(type)
.addAllKeyLocationList(keyLocationVersions.stream()
.map(KsmKeyLocationInfoGroup::getProtobuf)
.collect(Collectors.toList()))
@@ -241,7 +269,9 @@ public final class KsmKeyInfo {
.collect(Collectors.toList()),
keyInfo.getDataSize(),
keyInfo.getCreationTime(),
- keyInfo.getModificationTime());
+ keyInfo.getModificationTime(),
+ keyInfo.getType(),
+ keyInfo.getFactor());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
index 854c688..0f38169 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
@@ -560,8 +560,6 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
- .setFactor(args.getFactor())
- .setType(args.getType())
.setDataSize(args.getDataSize()).build();
req.setKeyArgs(keyArgs);
req.setClientID(clientID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
index 7b70330..d3d1de6 100644
--- a/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto
@@ -249,10 +249,12 @@ message KeyInfo {
required string bucketName = 2;
required string keyName = 3;
required uint64 dataSize = 4;
- repeated KeyLocationList keyLocationList = 5;
- required uint64 creationTime = 6;
- required uint64 modificationTime = 7;
- optional uint64 latestVersion = 8;
+ required hadoop.hdds.ReplicationType type = 5;
+ required hadoop.hdds.ReplicationFactor factor = 6;
+ repeated KeyLocationList keyLocationList = 7;
+ required uint64 creationTime = 8;
+ required uint64 modificationTime = 9;
+ optional uint64 latestVersion = 10;
}
message LocateKeyRequest {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
index 6409a73..0d4cfda 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
@@ -173,18 +173,6 @@ public class KeyManagerImpl implements KeyManager {
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
- ReplicationFactor factor = args.getFactor();
- ReplicationType type = args.getType();
-
- // If user does not specify a replication strategy or
- // replication factor, KSM will use defaults.
- if(factor == null) {
- factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE;
- }
-
- if(type == null) {
- type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE;
- }
try {
validateBucket(volumeName, bucketName);
@@ -198,10 +186,11 @@ public class KeyManagerImpl implements KeyManager {
throw new KSMException("Open Key not found",
KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
}
- AllocatedBlock allocatedBlock =
- scmBlockClient.allocateBlock(scmBlockSize, type, factor, ksmId);
KsmKeyInfo keyInfo =
KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
+ AllocatedBlock allocatedBlock =
+ scmBlockClient.allocateBlock(scmBlockSize, keyInfo.getType(),
+ keyInfo.getFactor(), ksmId);
KsmKeyLocationInfo info = new KsmKeyLocationInfo.Builder()
.setBlockID(allocatedBlock.getBlockID())
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
@@ -293,6 +282,8 @@ public class KeyManagerImpl implements KeyManager {
.setCreationTime(currentTime)
.setModificationTime(currentTime)
.setDataSize(size)
+ .setReplicationType(type)
+ .setReplicationFactor(factor)
.build();
openVersion = 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae5c50/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
index 536f95a..38e7797 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
@@ -527,16 +527,10 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
AllocateBlockResponse.newBuilder();
try {
KeyArgs keyArgs = request.getKeyArgs();
- HddsProtos.ReplicationType type =
- keyArgs.hasType()? keyArgs.getType() : null;
- HddsProtos.ReplicationFactor factor =
- keyArgs.hasFactor()? keyArgs.getFactor() : null;
KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
.setVolumeName(keyArgs.getVolumeName())
.setBucketName(keyArgs.getBucketName())
.setKeyName(keyArgs.getKeyName())
- .setType(type)
- .setFactor(factor)
.build();
int id = request.getClientID();
KsmKeyLocationInfo newLocation = impl.allocateBlock(ksmKeyArgs, id);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org