You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by na...@apache.org on 2017/09/03 23:55:38 UTC
[58/77] [abbrv] hadoop git commit: HDFS-12300. Audit-log delegation
token related operations.
HDFS-12300. Audit-log delegation token related operations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b3b9938
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b3b9938
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b3b9938
Branch: refs/heads/yarn-3409
Commit: 1b3b9938cf663c71d2e5d9032fdfb1460bae0d3f
Parents: 36f33a1
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Aug 31 23:17:16 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Aug 31 23:20:01 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 22 +++++++++
.../hdfs/server/namenode/FSNamesystem.java | 35 ++++++++++---
.../namenode/TestAuditLoggerWithCommands.java | 52 ++++++++++++++++++++
3 files changed, 103 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3b9938/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 47e1c0d..7776dc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -36,6 +36,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetAddress;
@@ -70,6 +72,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.http.HttpConfig;
@@ -80,6 +83,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ToolRunner;
import com.google.common.annotations.VisibleForTesting;
@@ -1570,4 +1574,22 @@ public class DFSUtil {
.createKeyProviderCryptoExtension(keyProvider);
return cryptoProvider;
}
+
+ /**
+ * Decodes an HDFS delegation token to its identifier.
+ *
+ * @param token the token
+ * @return the decoded identifier.
+ * @throws IOException
+ */
+ public static DelegationTokenIdentifier decodeDelegationToken(
+ final Token<DelegationTokenIdentifier> token) throws IOException {
+ final DelegationTokenIdentifier id = new DelegationTokenIdentifier();
+ final ByteArrayInputStream buf =
+ new ByteArrayInputStream(token.getIdentifier());
+ try (DataInputStream in = new DataInputStream(buf)) {
+ id.readFields(in);
+ }
+ return id;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3b9938/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 12d96d8..346f046 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -101,9 +101,7 @@ import static org.apache.hadoop.util.Time.monotonicNow;
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
import java.io.BufferedWriter;
-import java.io.ByteArrayInputStream;
import java.io.DataInput;
-import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
@@ -5399,6 +5397,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException {
+ final String operationName = "getDelegationToken";
+ final boolean success;
+ final String tokenId;
Token<DelegationTokenIdentifier> token;
checkOperation(OperationCategory.WRITE);
writeLock();
@@ -5427,10 +5428,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
dtId, dtSecretManager);
long expiryTime = dtSecretManager.getTokenExpiryTime(dtId);
getEditLog().logGetDelegationToken(dtId, expiryTime);
+ tokenId = dtId.toStringStable();
+ success = true;
} finally {
writeUnlock("getDelegationToken");
}
getEditLog().logSync();
+ logAuditEvent(success, operationName, tokenId);
return token;
}
@@ -5443,6 +5447,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
long renewDelegationToken(Token<DelegationTokenIdentifier> token)
throws InvalidToken, IOException {
+ final String operationName = "renewDelegationToken";
+ boolean success = false;
+ String tokenId;
long expiryTime;
checkOperation(OperationCategory.WRITE);
writeLock();
@@ -5456,15 +5463,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
String renewer = getRemoteUser().getShortUserName();
expiryTime = dtSecretManager.renewToken(token, renewer);
- DelegationTokenIdentifier id = new DelegationTokenIdentifier();
- ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
- DataInputStream in = new DataInputStream(buf);
- id.readFields(in);
+ final DelegationTokenIdentifier id = DFSUtil.decodeDelegationToken(token);
getEditLog().logRenewDelegationToken(id, expiryTime);
+ tokenId = id.toStringStable();
+ success = true;
+ } catch (AccessControlException ace) {
+ final DelegationTokenIdentifier id = DFSUtil.decodeDelegationToken(token);
+ tokenId = id.toStringStable();
+ logAuditEvent(success, operationName, tokenId);
+ throw ace;
} finally {
writeUnlock("renewDelegationToken");
}
getEditLog().logSync();
+ logAuditEvent(success, operationName, tokenId);
return expiryTime;
}
@@ -5475,6 +5487,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
throws IOException {
+ final String operationName = "cancelDelegationToken";
+ boolean success = false;
+ String tokenId;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -5485,10 +5500,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
DelegationTokenIdentifier id = dtSecretManager
.cancelToken(token, canceller);
getEditLog().logCancelDelegationToken(id);
+ tokenId = id.toStringStable();
+ success = true;
+ } catch (AccessControlException ace) {
+ final DelegationTokenIdentifier id = DFSUtil.decodeDelegationToken(token);
+ tokenId = id.toStringStable();
+ logAuditEvent(success, operationName, tokenId);
+ throw ace;
} finally {
writeUnlock("cancelDelegationToken");
}
getEditLog().logSync();
+ logAuditEvent(success, operationName, tokenId);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3b9938/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
index 8b06b0b..2adf470 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
@@ -34,13 +34,16 @@ import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import org.junit.BeforeClass;
import org.junit.AfterClass;
import org.junit.Ignore;
import org.junit.Test;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.mockito.Mock;
@@ -68,6 +71,7 @@ public class TestAuditLoggerWithCommands {
conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
+ conf.setBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
cluster.waitActive();
@@ -566,6 +570,54 @@ public class TestAuditLoggerWithCommands {
cluster.getNamesystem().setFSDirectory(dir);
}
+ @Test
+ public void testDelegationTokens() throws Exception {
+ Token dt = fs.getDelegationToken("foo");
+ final String getDT =
+ ".*src=HDFS_DELEGATION_TOKEN token 1.*with renewer foo.*";
+ verifyAuditLogs(true, ".*cmd=getDelegationToken" + getDT);
+
+ // renew
+ final UserGroupInformation foo =
+ UserGroupInformation.createUserForTesting("foo", new String[] {});
+ foo.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ dt.renew(conf);
+ return null;
+ }
+ });
+ verifyAuditLogs(true, ".*cmd=renewDelegationToken" + getDT);
+ try {
+ dt.renew(conf);
+ fail("Renewing a token with non-renewer should fail");
+ } catch (AccessControlException expected) {
+ }
+ verifyAuditLogs(false, ".*cmd=renewDelegationToken" + getDT);
+
+ // cancel
+ final UserGroupInformation bar =
+ UserGroupInformation.createUserForTesting("bar", new String[] {});
+ try {
+ bar.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ dt.cancel(conf);
+ return null;
+ }
+ });
+ fail("Canceling a token with non-renewer should fail");
+ } catch (AccessControlException expected) {
+ }
+ verifyAuditLogs(false, ".*cmd=cancelDelegationToken" + getDT);
+ dt.cancel(conf);
+ verifyAuditLogs(true, ".*cmd=cancelDelegationToken" + getDT);
+ }
+
+ private int verifyAuditLogs(final boolean allowed, final String pattern) {
+ return verifyAuditLogs(".*allowed=" + allowed + pattern);
+ }
+
private int verifyAuditLogs(String pattern) {
int length = auditlog.getOutput().split("\n").length;
String lastAudit = auditlog.getOutput().split("\n")[length - 1];
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org