You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2020/07/06 10:45:05 UTC
[hadoop-ozone] branch master updated: HDDS-3604. Use Ozone version
of Hadoop Security/Token classes (#933)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a84d5bc HDDS-3604. Use Ozone version of Hadoop Security/Token classes (#933)
a84d5bc is described below
commit a84d5bc425c9bc2b1fcfd9c7c88e160c22fef253
Author: Vinayakumar B <vi...@apache.org>
AuthorDate: Mon Jul 6 16:14:53 2020 +0530
HDDS-3604. Use Ozone version of Hadoop Security/Token classes (#933)
---
.../RandomContainerDeletionChoosingPolicy.java | 7 +--
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 6 ++-
...OzoneManagerProtocolClientSideTranslatorPB.java | 6 +--
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 2 +-
.../hadoop/ozone/protocolPB/OzonePBHelper.java | 60 +++++++++++++++++++++-
.../dev-support/findbugsExcludeFile.xml | 2 +-
.../interface-client/src/main/proto/Security.proto | 3 +-
.../interface-client/src/main/proto/proto.lock | 2 +-
.../security/OMCancelDelegationTokenRequest.java | 4 +-
.../security/OMGetDelegationTokenRequest.java | 6 +--
.../security/OMRenewDelegationTokenRequest.java | 5 +-
.../security/TestOMGetDelegationTokenRequest.java | 3 +-
.../security/TestOMGetDelegationTokenResponse.java | 2 +-
13 files changed, 86 insertions(+), 22 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
index 4dde3d6..9f2e9e8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.ozone.container.common.impl;
import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.container.common.interfaces
.ContainerDeletionChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
@@ -50,8 +50,9 @@ public class RandomContainerDeletionChoosingPolicy
List<ContainerData> result = new LinkedList<>();
ContainerData[] values = new ContainerData[candidateContainers.size()];
// to get a shuffle list
- for (ContainerData entry : DFSUtil.shuffle(
- candidateContainers.values().toArray(values))) {
+ ContainerData[] shuffled = candidateContainers.values().toArray(values);
+ ArrayUtils.shuffle(shuffled);
+ for (ContainerData entry : shuffled) {
if (currentCount < count) {
result.add(entry);
currentCount++;
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
index b81fcd0..b9a2920 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.UnknownPipelineStateException;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
+import org.apache.hadoop.ozone.protocolPB.OzonePBHelper;
import org.apache.hadoop.security.token.Token;
import java.util.Objects;
@@ -162,7 +163,7 @@ public final class OmKeyLocationInfo {
.setOffset(offset)
.setCreateVersion(createVersion);
if (this.token != null) {
- builder.setToken(this.token.toTokenProto());
+ builder.setToken(OzonePBHelper.protoFromToken(token));
}
try {
builder.setPipeline(pipeline.getProtobufMessage());
@@ -188,7 +189,8 @@ public final class OmKeyLocationInfo {
keyLocation.getLength(),
keyLocation.getOffset());
if(keyLocation.hasToken()) {
- info.token = new Token<>(keyLocation.getToken());
+ info.token = (Token<OzoneBlockTokenIdentifier>)
+ OzonePBHelper.tokenFromProto(keyLocation.getToken());
}
info.setCreateVersion(keyLocation.getCreateVersion());
return info;
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 3c676eb..ae2c622 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -132,9 +132,9 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeI
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
-import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
-import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.security.token.Token;
import com.google.common.annotations.VisibleForTesting;
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
index 4ff5f6a..2ff2dc8 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.FileEncryptionInfoProto;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
/**
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OzonePBHelper.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OzonePBHelper.java
index 8361bac..5939ca6 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OzonePBHelper.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OzonePBHelper.java
@@ -17,14 +17,72 @@
*/
package org.apache.hadoop.ozone.protocolPB;
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+import java.util.concurrent.ConcurrentHashMap;
+
/**
* Helper class for converting protobuf objects.
*/
public final class OzonePBHelper {
-
private OzonePBHelper() {
/** Hidden constructor */
}
+ // Borrowed from ProtobuHelper.java in hadoop-common involving protobuf
+ // messages to avoid breakage due to shading of protobuf in Hadoop-3.3+.
+ /**
+ * Map used to cache fixed strings to ByteStrings. Since there is no
+ * automatic expiration policy, only use this for strings from a fixed, small
+ * set.
+ * <p/>
+ * This map should not be accessed directly. Used the getFixedByteString
+ * methods instead.
+ */
+ private final static ConcurrentHashMap<Object, ByteString>
+ FIXED_BYTESTRING_CACHE = new ConcurrentHashMap<>();
+
+ /**
+ * Get the ByteString for frequently used fixed and small set strings.
+ *
+ * @param key string
+ * @return
+ */
+ public static ByteString getFixedByteString(Text key) {
+ ByteString value = FIXED_BYTESTRING_CACHE.get(key);
+ if (value == null) {
+ value = ByteString.copyFromUtf8(key.toString());
+ ByteString oldValue = FIXED_BYTESTRING_CACHE.putIfAbsent(key, value);
+ return oldValue != null ? oldValue : value;
+ }
+ return value;
+ }
+
+ public static ByteString getByteString(byte[] bytes) {
+ // return singleton to reduce object allocation
+ return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
+ }
+ public static Token<? extends TokenIdentifier> tokenFromProto(
+ TokenProto tokenProto) {
+ Token<? extends TokenIdentifier> token = new Token<>(
+ tokenProto.getIdentifier().toByteArray(),
+ tokenProto.getPassword().toByteArray(),
+ new Text(tokenProto.getKind()),
+ new Text(tokenProto.getService()));
+ return token;
+ }
+
+ public static TokenProto protoFromToken(Token<?> tok) {
+ TokenProto.Builder builder = TokenProto.newBuilder().
+ setIdentifier(getByteString(tok.getIdentifier())).
+ setPassword(getByteString(tok.getPassword())).
+ setKindBytes(getFixedByteString(tok.getKind())).
+ setServiceBytes(getFixedByteString(tok.getService()));
+ return builder.build();
+ }
}
diff --git a/hadoop-ozone/interface-client/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/interface-client/dev-support/findbugsExcludeFile.xml
index 9d0ce5d..ed27981 100644
--- a/hadoop-ozone/interface-client/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-ozone/interface-client/dev-support/findbugsExcludeFile.xml
@@ -19,6 +19,6 @@
<Package name="org.apache.hadoop.ozone.protocol.proto"/>
</Match>
<Match>
- <Package name="org.apache.hadoop.security.proto"/>
+ <Package name="org.apache.hadoop.ozone.security.proto"/>
</Match>
</FindBugsFilter>
diff --git a/hadoop-ozone/interface-client/src/main/proto/Security.proto b/hadoop-ozone/interface-client/src/main/proto/Security.proto
index a3ce739..2adf58e 100644
--- a/hadoop-ozone/interface-client/src/main/proto/Security.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/Security.proto
@@ -22,7 +22,8 @@
* for what changes are allowed for a *stable* .proto interface.
*/
-option java_package = "org.apache.hadoop.security.proto";
+//Use ozone specific security proto until start using hadoop-thirdparty shaded protobuf everywhere.
+option java_package = "org.apache.hadoop.ozone.security.proto";
option java_outer_classname = "SecurityProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
diff --git a/hadoop-ozone/interface-client/src/main/proto/proto.lock b/hadoop-ozone/interface-client/src/main/proto/proto.lock
index 7e41dfa..0331ff1 100644
--- a/hadoop-ozone/interface-client/src/main/proto/proto.lock
+++ b/hadoop-ozone/interface-client/src/main/proto/proto.lock
@@ -3404,7 +3404,7 @@
"options": [
{
"name": "java_package",
- "value": "org.apache.hadoop.security.proto"
+ "value": "org.apache.hadoop.ozone.security.proto"
},
{
"name": "java_outer_classname",
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java
index 48451f5..e931735 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRespo
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CancelDelegationTokenResponseProto;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.security.proto.SecurityProtos;
-import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java
index a3809be..4d2a6b4 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java
@@ -33,11 +33,11 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRespo
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UpdateGetDelegationTokenRequest;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.security.proto.SecurityProtos;
-import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.token.Token;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java
index 859b24b..360ca4f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenewDe
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UpdateRenewDelegationTokenRequest;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
@@ -67,7 +67,8 @@ public class OMRenewDelegationTokenRequest extends OMClientRequest {
RenewDelegationTokenResponseProto.Builder renewResponse =
RenewDelegationTokenResponseProto.newBuilder();
- renewResponse.setResponse(org.apache.hadoop.security.proto.SecurityProtos
+ renewResponse.setResponse(
+ org.apache.hadoop.ozone.security.proto.SecurityProtos
.RenewDelegationTokenResponseProto.newBuilder()
.setNewExpiryTime(renewTime));
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java
index df0fcb9..dfbb044 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java
@@ -30,8 +30,9 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRequest;
import static org.apache.hadoop.ozone.security.OzoneTokenIdentifier.KIND_NAME;
+
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.io.Text;
import org.mockito.Mockito;
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java
index df90d7e..032b1a7 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.ozone.om.response.security;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.om.request.security.OMGetDelegationTokenRequest;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -29,6 +28,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UpdateGetDelegationTokenRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import java.io.IOException;
import java.util.UUID;
import org.junit.Assert;
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org