You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/10/26 19:29:35 UTC
[2/2] git commit: YARN-2743. Fixed a bug in ResourceManager that was
causing RMDelegationToken identifiers to be tampered and thus causing app
submission failures in secure mode. Contributed by Jian He.
YARN-2743. Fixed a bug in ResourceManager that was causing RMDelegationToken identifiers to be tampered and thus causing app submission failures in secure mode. Contributed by Jian He.
(cherry picked from commit 018664550507981297fd9f91e29408e6b7801ea9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/259db7a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/259db7a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/259db7a7
Branch: refs/heads/branch-2.6
Commit: 259db7a726e2a571a682531d3099f2710a41134a
Parents: 7bb5071
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Sun Oct 26 11:14:34 2014 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sun Oct 26 11:29:06 2014 -0700
----------------------------------------------------------------------
.../AbstractDelegationTokenIdentifier.java | 2 +-
.../AbstractDelegationTokenSecretManager.java | 15 +-
hadoop-yarn-project/CHANGES.txt | 4 +
.../client/YARNDelegationTokenIdentifier.java | 198 ++++---------------
.../main/proto/server/yarn_security_token.proto | 3 +-
.../yarn/security/TestYARNTokenIdentifier.java | 5 -
.../hadoop-yarn-server-resourcemanager/pom.xml | 1 +
.../recovery/FileSystemRMStateStore.java | 32 +--
.../recovery/ZKRMStateStore.java | 31 +--
.../RMDelegationTokenIdentifierData.java | 61 ++++++
.../yarn_server_resourcemanager_recovery.proto | 6 +
.../RMDelegationTokenIdentifierForTest.java | 156 ++++-----------
.../recovery/RMStateStoreTestBase.java | 6 +
13 files changed, 193 insertions(+), 327 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
index 168a77b..5a9d076 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
@@ -159,7 +159,7 @@ extends TokenIdentifier {
return masterKeyId;
}
- static boolean isEqual(Object a, Object b) {
+ protected static boolean isEqual(Object a, Object b) {
return a == null ? b == null : a.equals(b);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index ac399ec..52e6a01 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
@@ -18,15 +18,8 @@
package org.apache.hadoop.security.token.delegation;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.MD5Hash;
-import org.apache.hadoop.io.Text;
-
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
@@ -39,10 +32,13 @@ import javax.crypto.SecretKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.HadoopKerberosName;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.Time;
@@ -386,7 +382,8 @@ extends AbstractDelegationTokenIdentifier>
identifier.setMaxDate(now + tokenMaxLifetime);
identifier.setMasterKeyId(currentKey.getKeyId());
identifier.setSequenceNumber(sequenceNum);
- LOG.info("Creating password for identifier: [" + MD5Hash.digest(identifier.getBytes()) + ", " + currentKey.getKeyId() + "]");
+ LOG.info("Creating password for identifier: " + identifier
+ + ", currentKey: " + currentKey.getKeyId());
byte[] password = createPassword(identifier.getBytes(), currentKey.getKey());
DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now
+ tokenRenewInterval, password, getTrackingIdIfEnabled(identifier));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eee45ba..30f87e2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -574,6 +574,10 @@ Release 2.6.0 - UNRELEASED
YARN-2715. Fixed ResourceManager to respect common configurations for proxy
users/groups beyond just the YARN level config. (Zhijie Shen via vinodkv)
+ YARN-2743. Fixed a bug in ResourceManager that was causing RMDelegationToken
+ identifiers to be tampered and thus causing app submission failures in
+ secure mode. (Jian He via vinodkv)
+
BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
YARN-1707. Introduce APIs to add/remove/resize queues in the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
index 3c5fa867..7ccb923 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
@@ -22,193 +22,61 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.HadoopKerberosName;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto;
+@Private
public abstract class YARNDelegationTokenIdentifier extends
AbstractDelegationTokenIdentifier {
-
- YARNDelegationTokenIdentifierProto.Builder builder =
+
+ YARNDelegationTokenIdentifierProto.Builder builder =
YARNDelegationTokenIdentifierProto.newBuilder();
- public YARNDelegationTokenIdentifier() {}
+ public YARNDelegationTokenIdentifier() {
+ }
public YARNDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
- setOwner(owner);
- setRenewer(renewer);
- setRealUser(realUser);
- }
-
- /**
- * Get the username encoded in the token identifier
- *
- * @return the username or owner
- */
- @Override
- public UserGroupInformation getUser() {
- String owner = getOwner() == null ? null : getOwner().toString();
- String realUser = getRealUser() == null ? null: getRealUser().toString();
- if ( (owner == null) || (owner.toString().isEmpty())) {
- return null;
- }
- final UserGroupInformation realUgi;
- final UserGroupInformation ugi;
- if ((realUser == null) || (realUser.toString().isEmpty())
- || realUser.equals(owner)) {
- ugi = realUgi = UserGroupInformation.createRemoteUser(owner.toString());
- } else {
- realUgi = UserGroupInformation.createRemoteUser(realUser.toString());
- ugi = UserGroupInformation.createProxyUser(owner.toString(), realUgi);
- }
- realUgi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
- return ugi;
+ super(owner, renewer, realUser);
}
- public Text getOwner() {
- String owner = builder.getOwner();
- if (owner == null) {
- return null;
- } else {
- return new Text(owner);
- }
+ public YARNDelegationTokenIdentifier(
+ YARNDelegationTokenIdentifierProto.Builder builder) {
+ this.builder = builder;
}
@Override
- public void setOwner(Text owner) {
- if (builder != null && owner != null) {
- builder.setOwner(owner.toString());
+ public synchronized void readFields(DataInput in) throws IOException {
+ builder.mergeFrom((DataInputStream) in);
+ if (builder.getOwner() != null) {
+ setOwner(new Text(builder.getOwner()));
}
- }
-
- public Text getRenewer() {
- String renewer = builder.getRenewer();
- if (renewer == null) {
- return null;
- } else {
- return new Text(renewer);
+ if (builder.getRenewer() != null) {
+ setRenewer(new Text(builder.getRenewer()));
}
- }
-
- @Override
- public void setRenewer(Text renewer) {
- if (builder != null && renewer != null) {
- HadoopKerberosName renewerKrbName = new HadoopKerberosName(renewer.toString());
- try {
- builder.setRenewer(renewerKrbName.getShortName());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- public Text getRealUser() {
- String realUser = builder.getRealUser();
- if (realUser == null) {
- return null;
- } else {
- return new Text(realUser);
+ if (builder.getRealUser() != null) {
+ setRealUser(new Text(builder.getRealUser()));
}
+ setIssueDate(builder.getIssueDate());
+ setMaxDate(builder.getMaxDate());
+ setSequenceNumber(builder.getSequenceNumber());
+ setMasterKeyId(builder.getMasterKeyId());
}
@Override
- public void setRealUser(Text realUser) {
- if (builder != null && realUser != null) {
- builder.setRealUser(realUser.toString());
- }
+ public synchronized void write(DataOutput out) throws IOException {
+ builder.setOwner(getOwner().toString());
+ builder.setRenewer(getRenewer().toString());
+ builder.setRealUser(getRealUser().toString());
+ builder.setIssueDate(getIssueDate());
+ builder.setMaxDate(getMaxDate());
+ builder.setSequenceNumber(getSequenceNumber());
+ builder.setMasterKeyId(getMasterKeyId());
+ builder.build().writeTo((DataOutputStream) out);
}
- public void setIssueDate(long issueDate) {
- builder.setIssueDate(issueDate);
- }
-
- public long getIssueDate() {
- return builder.getIssueDate();
+ public YARNDelegationTokenIdentifierProto getProto() {
+ return builder.build();
}
-
-
- public void setRenewDate(long renewDate) {
- builder.setRenewDate(renewDate);
- }
-
- public long getRenewDate() {
- return builder.getRenewDate();
- }
-
- public void setMaxDate(long maxDate) {
- builder.setMaxDate(maxDate);
- }
-
- public long getMaxDate() {
- return builder.getMaxDate();
- }
-
- public void setSequenceNumber(int seqNum) {
- builder.setSequenceNumber(seqNum);
- }
-
- public int getSequenceNumber() {
- return builder.getSequenceNumber();
- }
-
- public void setMasterKeyId(int newId) {
- builder.setMasterKeyId(newId);
- }
-
- public int getMasterKeyId() {
- return builder.getMasterKeyId();
- }
-
- protected static boolean isEqual(Object a, Object b) {
- return a == null ? b == null : a.equals(b);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (obj instanceof YARNDelegationTokenIdentifier) {
- YARNDelegationTokenIdentifier that = (YARNDelegationTokenIdentifier) obj;
- return this.getSequenceNumber() == that.getSequenceNumber()
- && this.getIssueDate() == that.getIssueDate()
- && this.getMaxDate() == that.getMaxDate()
- && this.getMasterKeyId() == that.getMasterKeyId()
- && isEqual(this.getOwner(), that.getOwner())
- && isEqual(this.getRenewer(), that.getRenewer())
- && isEqual(this.getRealUser(), that.getRealUser());
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return this.getSequenceNumber();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- builder.mergeFrom((DataInputStream) in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- builder.build().writeTo((DataOutputStream)out);
- }
-
- @Override
- public String toString() {
- StringBuilder buffer = new StringBuilder();
- buffer
- .append("owner=" + getOwner() + ", renewer=" + getRenewer() + ", realUser="
- + getRealUser() + ", issueDate=" + getIssueDate()
- + ", maxDate=" + getMaxDate() + ", sequenceNumber="
- + getSequenceNumber() + ", masterKeyId="
- + getMasterKeyId());
- return buffer.toString();
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
index 60c7fcc..317032d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
@@ -63,7 +63,6 @@ message YARNDelegationTokenIdentifierProto {
optional int64 issueDate = 4;
optional int64 maxDate = 5;
optional int32 sequenceNumber = 6;
- optional int32 masterKeyId = 7 [default = -1];
- optional int64 renewDate = 8;
+ optional int32 masterKeyId = 7;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
index 2e6255a..2052c23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
@@ -257,7 +257,6 @@ public class TestYARNTokenIdentifier {
long maxDate = 2;
int sequenceNumber = 3;
int masterKeyId = 4;
- long renewDate = 5;
TimelineDelegationTokenIdentifier token =
new TimelineDelegationTokenIdentifier(owner, renewer, realUser);
@@ -265,7 +264,6 @@ public class TestYARNTokenIdentifier {
token.setMaxDate(maxDate);
token.setSequenceNumber(sequenceNumber);
token.setMasterKeyId(masterKeyId);
- token.setRenewDate(renewDate);
TimelineDelegationTokenIdentifier anotherToken =
new TimelineDelegationTokenIdentifier();
@@ -299,9 +297,6 @@ public class TestYARNTokenIdentifier {
Assert.assertEquals("masterKeyId from proto is not the same with original token",
anotherToken.getMasterKeyId(), masterKeyId);
-
- Assert.assertEquals("renewDate from proto is not the same with original token",
- anotherToken.getRenewDate(), renewDate);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 8e4e852..6dd0b85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -279,6 +279,7 @@
<imports>
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>
+ <param>${basedir}/../../hadoop-yarn-common/src/main/proto/server/</param>
<param>${basedir}/../hadoop-yarn-server-common/src/main/proto</param>
<param>${basedir}/src/main/proto</param>
</imports>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index d434e07..2bbc5c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -47,9 +47,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.EpochProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.EpochProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenS
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@@ -369,12 +370,24 @@ public class FileSystemRMStateStore extends RMStateStore {
DelegationKey key = new DelegationKey();
key.readFields(fsIn);
rmState.rmSecretManagerState.masterKeyState.add(key);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loaded delegation key: keyId=" + key.getKeyId()
+ + ", expirationDate=" + key.getExpiryDate());
+ }
} else if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) {
- RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier();
- identifier.readFields(fsIn);
- long renewDate = identifier.getRenewDate();
+ RMDelegationTokenIdentifierData identifierData =
+ new RMDelegationTokenIdentifierData();
+ identifierData.readFields(fsIn);
+ RMDelegationTokenIdentifier identifier =
+ identifierData.getTokenIdentifier();
+ long renewDate = identifierData.getRenewDate();
+
rmState.rmSecretManagerState.delegationTokenState.put(identifier,
renewDate);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier
+ + " renewDate=" + renewDate);
+ }
} else {
LOG.warn("Unknown file for recovering RMDelegationTokenSecretManager");
}
@@ -503,18 +516,15 @@ public class FileSystemRMStateStore extends RMStateStore {
Path nodeCreatePath =
getNodePath(rmDTSecretManagerRoot,
DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber());
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- DataOutputStream fsOut = new DataOutputStream(os);
- identifier.setRenewDate(renewDate);
- identifier.write(fsOut);
+ RMDelegationTokenIdentifierData identifierData =
+ new RMDelegationTokenIdentifierData(identifier, renewDate);
if (isUpdate) {
LOG.info("Updating RMDelegationToken_" + identifier.getSequenceNumber());
- updateFile(nodeCreatePath, os.toByteArray());
+ updateFile(nodeCreatePath, identifierData.toByteArray());
} else {
LOG.info("Storing RMDelegationToken_" + identifier.getSequenceNumber());
- writeFile(nodeCreatePath, os.toByteArray());
+ writeFile(nodeCreatePath, identifierData.toByteArray());
}
- fsOut.close();
// store sequence number
Path latestSequenceNumberPath = getNodePath(rmDTSecretManagerRoot,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index fdbf125..ab048ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -50,13 +50,14 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.Appl
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.EpochProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@@ -488,6 +489,10 @@ public class ZKRMStateStore extends RMStateStore {
DelegationKey key = new DelegationKey();
key.readFields(fsIn);
rmState.rmSecretManagerState.masterKeyState.add(key);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loaded delegation key: keyId=" + key.getKeyId()
+ + ", expirationDate=" + key.getExpiryDate());
+ }
}
} finally {
is.close();
@@ -527,12 +532,18 @@ public class ZKRMStateStore extends RMStateStore {
try {
if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) {
+ RMDelegationTokenIdentifierData identifierData =
+ new RMDelegationTokenIdentifierData();
+ identifierData.readFields(fsIn);
RMDelegationTokenIdentifier identifier =
- new RMDelegationTokenIdentifier();
- identifier.readFields(fsIn);
- long renewDate = identifier.getRenewDate();
+ identifierData.getTokenIdentifier();
+ long renewDate = identifierData.getRenewDate();
rmState.rmSecretManagerState.delegationTokenState.put(identifier,
renewDate);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier
+ + " renewDate=" + renewDate);
+ }
}
} finally {
is.close();
@@ -770,23 +781,20 @@ public class ZKRMStateStore extends RMStateStore {
String nodeCreatePath =
getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
+ rmDTIdentifier.getSequenceNumber());
- ByteArrayOutputStream tokenOs = new ByteArrayOutputStream();
- DataOutputStream tokenOut = new DataOutputStream(tokenOs);
ByteArrayOutputStream seqOs = new ByteArrayOutputStream();
DataOutputStream seqOut = new DataOutputStream(seqOs);
-
+ RMDelegationTokenIdentifierData identifierData =
+ new RMDelegationTokenIdentifierData(rmDTIdentifier, renewDate);
try {
- rmDTIdentifier.setRenewDate(renewDate);
- rmDTIdentifier.write(tokenOut);
if (LOG.isDebugEnabled()) {
LOG.debug((isUpdate ? "Storing " : "Updating ") + "RMDelegationToken_" +
rmDTIdentifier.getSequenceNumber());
}
if (isUpdate) {
- opList.add(Op.setData(nodeCreatePath, tokenOs.toByteArray(), -1));
+ opList.add(Op.setData(nodeCreatePath, identifierData.toByteArray(), -1));
} else {
- opList.add(Op.create(nodeCreatePath, tokenOs.toByteArray(), zkAcl,
+ opList.add(Op.create(nodeCreatePath, identifierData.toByteArray(), zkAcl,
CreateMode.PERSISTENT));
}
@@ -799,7 +807,6 @@ public class ZKRMStateStore extends RMStateStore {
opList.add(Op.setData(dtSequenceNumberPath, seqOs.toByteArray(), -1));
} finally {
- tokenOs.close();
seqOs.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/RMDelegationTokenIdentifierData.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/RMDelegationTokenIdentifierData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/RMDelegationTokenIdentifierData.java
new file mode 100644
index 0000000..97b5c1c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/RMDelegationTokenIdentifierData.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.recovery.records;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.RMDelegationTokenIdentifierDataProto;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
+
+public class RMDelegationTokenIdentifierData {
+ RMDelegationTokenIdentifierDataProto.Builder builder =
+ RMDelegationTokenIdentifierDataProto.newBuilder();
+
+ public RMDelegationTokenIdentifierData() {}
+
+ public RMDelegationTokenIdentifierData(
+ YARNDelegationTokenIdentifier identifier, long renewdate) {
+ builder.setTokenIdentifier(identifier.getProto());
+ builder.setRenewDate(renewdate);
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ builder.mergeFrom((DataInputStream) in);
+ }
+
+ public byte[] toByteArray() throws IOException {
+ return builder.build().toByteArray();
+ }
+
+ public RMDelegationTokenIdentifier getTokenIdentifier() throws IOException {
+ ByteArrayInputStream in =
+ new ByteArrayInputStream(builder.getTokenIdentifier().toByteArray());
+ RMDelegationTokenIdentifier identifer = new RMDelegationTokenIdentifier();
+ identifer.readFields(new DataInputStream(in));
+ return identifer;
+ }
+
+ public long getRenewDate() {
+ return builder.getRenewDate();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 4d29153..3c8ac34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -24,6 +24,7 @@ package hadoop.yarn;
import "yarn_server_common_protos.proto";
import "yarn_protos.proto";
+import "yarn_security_token.proto";
////////////////////////////////////////////////////////////////////////
////// RM recovery related records /////////////////////////////////////
@@ -91,3 +92,8 @@ message AMRMTokenSecretManagerStateProto {
optional MasterKeyProto current_master_key = 1;
optional MasterKeyProto next_master_key = 2;
}
+
+message RMDelegationTokenIdentifierDataProto {
+ optional YARNDelegationTokenIdentifierProto token_identifier = 1;
+ optional int64 renewDate = 2;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java
index 5e1baf7..29b4a0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMDelegationTokenIdentifierForTest.java
@@ -20,155 +20,73 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.IOException;
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
-import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.proto.YarnSecurityTestClientAMTokenProtos.RMDelegationTokenIdentifierForTestProto;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
public class RMDelegationTokenIdentifierForTest extends
RMDelegationTokenIdentifier {
- private RMDelegationTokenIdentifierForTestProto proto;
- private RMDelegationTokenIdentifierForTestProto.Builder builder;
+ private RMDelegationTokenIdentifierForTestProto.Builder builder =
+ RMDelegationTokenIdentifierForTestProto.newBuilder();
public RMDelegationTokenIdentifierForTest() {
}
- public RMDelegationTokenIdentifierForTest(
- RMDelegationTokenIdentifier token, String message) {
- builder = RMDelegationTokenIdentifierForTestProto.newBuilder();
+ public RMDelegationTokenIdentifierForTest(RMDelegationTokenIdentifier token,
+ String message) {
if (token.getOwner() != null) {
- builder.setOwner(token.getOwner().toString());
+ setOwner(new Text(token.getOwner()));
}
if (token.getRenewer() != null) {
- builder.setRenewer(token.getRenewer().toString());
+ setRenewer(new Text(token.getRenewer()));
}
if (token.getRealUser() != null) {
- builder.setRealUser(token.getRealUser().toString());
+ setRealUser(new Text(token.getRealUser()));
}
- builder.setIssueDate(token.getIssueDate());
- builder.setMaxDate(token.getMaxDate());
- builder.setSequenceNumber(token.getSequenceNumber());
- builder.setMasterKeyId(token.getMasterKeyId());
+ setIssueDate(token.getIssueDate());
+ setMaxDate(token.getMaxDate());
+ setSequenceNumber(token.getSequenceNumber());
+ setMasterKeyId(token.getMasterKeyId());
builder.setMessage(message);
- proto = builder.build();
- builder = null;
}
@Override
public void write(DataOutput out) throws IOException {
- out.write(proto.toByteArray());
+ builder.setOwner(getOwner().toString());
+ builder.setRenewer(getRenewer().toString());
+ builder.setRealUser(getRealUser().toString());
+ builder.setIssueDate(getIssueDate());
+ builder.setMaxDate(getMaxDate());
+ builder.setSequenceNumber(getSequenceNumber());
+ builder.setMasterKeyId(getMasterKeyId());
+ builder.setMessage(getMessage());
+ builder.build().writeTo((DataOutputStream) out);
}
@Override
public void readFields(DataInput in) throws IOException {
- DataInputStream dis = (DataInputStream)in;
- byte[] buffer = IOUtils.toByteArray(dis);
- proto = RMDelegationTokenIdentifierForTestProto.parseFrom(buffer);
- }
-
- /**
- * Get the username encoded in the token identifier
- *
- * @return the username or owner
- */
- @Override
- public UserGroupInformation getUser() {
- String owner = getOwner().toString();
- String realUser = getRealUser().toString();
- if ( (owner == null) || (owner.toString().isEmpty())) {
- return null;
+ builder.mergeFrom((DataInputStream) in);
+ if (builder.getOwner() != null) {
+ setOwner(new Text(builder.getOwner()));
}
- final UserGroupInformation realUgi;
- final UserGroupInformation ugi;
- if ((realUser == null) || (realUser.toString().isEmpty())
- || realUser.equals(owner)) {
- ugi = realUgi = UserGroupInformation.createRemoteUser(owner.toString());
- } else {
- realUgi = UserGroupInformation.createRemoteUser(realUser.toString());
- ugi = UserGroupInformation.createProxyUser(owner.toString(), realUgi);
+ if (builder.getRenewer() != null) {
+ setRenewer(new Text(builder.getRenewer()));
}
- realUgi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
- return ugi;
- }
-
- public Text getOwner() {
- String owner = proto.getOwner();
- if (owner == null) {
- return null;
- } else {
- return new Text(owner);
+ if (builder.getRealUser() != null) {
+ setRealUser(new Text(builder.getRealUser()));
}
- }
-
- public Text getRenewer() {
- String renewer = proto.getRenewer();
- if (renewer == null) {
- return null;
- } else {
- return new Text(renewer);
- }
- }
-
- public Text getRealUser() {
- String realUser = proto.getRealUser();
- if (realUser == null) {
- return null;
- } else {
- return new Text(realUser);
- }
- }
-
- public void setIssueDate(long issueDate) {
- RMDelegationTokenIdentifierForTestProto.Builder builder =
- RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
- builder.setIssueDate(issueDate);
- proto = builder.build();
- }
-
- public long getIssueDate() {
- return proto.getIssueDate();
- }
-
- public void setMaxDate(long maxDate) {
- RMDelegationTokenIdentifierForTestProto.Builder builder =
- RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
- builder.setMaxDate(maxDate);
- proto = builder.build();
- }
-
- public long getMaxDate() {
- return proto.getMaxDate();
- }
-
- public void setSequenceNumber(int seqNum) {
- RMDelegationTokenIdentifierForTestProto.Builder builder =
- RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
- builder.setSequenceNumber(seqNum);
- proto = builder.build();
- }
-
- public int getSequenceNumber() {
- return proto.getSequenceNumber();
- }
-
- public void setMasterKeyId(int newId) {
- RMDelegationTokenIdentifierForTestProto.Builder builder =
- RMDelegationTokenIdentifierForTestProto.newBuilder(proto);
- builder.setMasterKeyId(newId);
- proto = builder.build();
- }
-
- public int getMasterKeyId() {
- return proto.getMasterKeyId();
+ setIssueDate(builder.getIssueDate());
+ setMaxDate(builder.getMaxDate());
+ setSequenceNumber(builder.getSequenceNumber());
+ setMasterKeyId(builder.getMasterKeyId());
}
public String getMessage() {
- return proto.getMessage();
+ return builder.getMessage();
}
@Override
@@ -189,10 +107,4 @@ public class RMDelegationTokenIdentifierForTest extends
}
return false;
}
-
- @Override
- public int hashCode() {
- return this.getSequenceNumber();
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259db7a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 85022d9..00b60d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.when;
import static org.mockito.Mockito.spy;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -402,6 +403,7 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
RMDelegationTokenIdentifier dtId1 =
new RMDelegationTokenIdentifier(new Text("owner1"),
new Text("renewer1"), new Text("realuser1"));
+ byte[] tokenBeforeStore = dtId1.getBytes();
Long renewDate1 = new Long(System.currentTimeMillis());
int sequenceNumber = 1111;
store.storeRMDelegationTokenAndSequenceNumber(dtId1, renewDate1,
@@ -423,6 +425,10 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
Assert.assertEquals(keySet, secretManagerState.getMasterKeyState());
Assert.assertEquals(sequenceNumber,
secretManagerState.getDTSequenceNumber());
+ RMDelegationTokenIdentifier tokenAfterStore =
+ secretManagerState.getTokenState().keySet().iterator().next();
+ Assert.assertTrue(Arrays.equals(tokenBeforeStore,
+ tokenAfterStore.getBytes()));
// update RM delegation token;
renewDate1 = new Long(System.currentTimeMillis());