You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2012/08/31 06:12:38 UTC
svn commit: r1379281 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project: ./ hadoop-hdfs/
hadoop-hdfs/src/main/java/
hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/
hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenod...
Author: eli
Date: Fri Aug 31 04:12:37 2012
New Revision: 1379281
URL: http://svn.apache.org/viewvc?rev=1379281&view=rev
Log:
HDFS-3733. Audit logs should include WebHDFS access. Contributed by Andy Isaacson
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1379278
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1379278
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1379281&r1=1379280&r2=1379281&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Aug 31 04:12:37 2012
@@ -527,6 +527,8 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3837. Fix DataNode.recoverBlock findbugs warning. (eli)
+ HDFS-3733. Audit logs should include WebHDFS access. (Andy Isaacson via eli)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1379278
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1379281&r1=1379280&r2=1379281&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Aug 31 04:12:37 2012
@@ -169,6 +169,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
@@ -1056,7 +1057,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setPermission", src, null, null);
}
throw e;
@@ -1085,7 +1086,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setPermission", src, null, resultingStat);
}
}
@@ -1102,7 +1103,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setOwner", src, null, null);
}
throw e;
@@ -1140,7 +1141,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setOwner", src, null, resultingStat);
}
}
@@ -1175,7 +1176,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"open", src, null, null);
}
throw e;
@@ -1201,7 +1202,7 @@ public class FSNamesystem implements Nam
offset, length, doAccessTime, needBlockToken);
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"open", src, null, null);
}
if (checkSafeMode && isInSafeMode()) {
@@ -1286,7 +1287,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getLoginUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"concat", Arrays.toString(srcs), target, null);
}
throw e;
@@ -1336,7 +1337,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getLoginUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"concat", Arrays.toString(srcs), target, resultingStat);
}
}
@@ -1453,7 +1454,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setTimes", src, null, null);
}
throw e;
@@ -1480,7 +1481,7 @@ public class FSNamesystem implements Nam
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
final HdfsFileStatus stat = dir.getFileInfo(src, false);
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setTimes", src, null, stat);
}
} else {
@@ -1502,7 +1503,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"createSymlink", link, target, null);
}
throw e;
@@ -1530,7 +1531,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"createSymlink", link, target, resultingStat);
}
}
@@ -1586,7 +1587,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setReplication", src, null, null);
}
throw e;
@@ -1622,7 +1623,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
if (isFile && auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"setReplication", src, null, null);
}
return isFile;
@@ -1679,7 +1680,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"create", src, null, null);
}
throw e;
@@ -1704,7 +1705,7 @@ public class FSNamesystem implements Nam
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
final HdfsFileStatus stat = dir.getFileInfo(src, false);
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"create", src, null, stat);
}
}
@@ -2002,7 +2003,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"append", src, null, null);
}
throw e;
@@ -2040,7 +2041,7 @@ public class FSNamesystem implements Nam
}
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"append", src, null, null);
}
return lb;
@@ -2506,7 +2507,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"rename", src, dst, null);
}
throw e;
@@ -2535,7 +2536,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
if (status && auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"rename", src, dst, resultingStat);
}
return status;
@@ -2595,7 +2596,7 @@ public class FSNamesystem implements Nam
for (Rename option : options) {
cmd.append(option.value()).append(" ");
}
- logAuditEvent(UserGroupInformation.getCurrentUser(), Server.getRemoteIp(),
+ logAuditEvent(UserGroupInformation.getCurrentUser(), getRemoteIp(),
cmd.toString(), src, dst, resultingStat);
}
}
@@ -2633,7 +2634,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"delete", src, null, null);
}
throw e;
@@ -2649,7 +2650,7 @@ public class FSNamesystem implements Nam
boolean status = deleteInternal(src, recursive, true);
if (status && auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"delete", src, null, null);
}
return status;
@@ -2787,8 +2788,11 @@ public class FSNamesystem implements Nam
*/
HdfsFileStatus getFileInfo(String src, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException,
- StandbyException {
+ StandbyException, IOException {
+ HdfsFileStatus stat = null;
+
readLock();
+
try {
checkOperation(OperationCategory.READ);
@@ -2798,10 +2802,23 @@ public class FSNamesystem implements Nam
if (isPermissionEnabled) {
checkTraverse(src);
}
- return dir.getFileInfo(src, resolveLink);
+ stat = dir.getFileInfo(src, resolveLink);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ getRemoteIp(),
+ "getfileinfo", src, null, null);
+ }
+ throw e;
} finally {
readUnlock();
}
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(UserGroupInformation.getCurrentUser(),
+ getRemoteIp(),
+ "getfileinfo", src, null, null);
+ }
+ return stat;
}
/**
@@ -2814,7 +2831,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"mkdirs", src, null, null);
}
throw e;
@@ -2839,7 +2856,7 @@ public class FSNamesystem implements Nam
if (status && auditLog.isInfoEnabled() && isExternalInvocation()) {
final HdfsFileStatus stat = dir.getFileInfo(src, false);
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"mkdirs", src, null, stat);
}
return status;
@@ -3280,7 +3297,7 @@ public class FSNamesystem implements Nam
} catch (AccessControlException e) {
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(false, UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"listStatus", src, null, null);
}
throw e;
@@ -3304,7 +3321,7 @@ public class FSNamesystem implements Nam
}
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(UserGroupInformation.getCurrentUser(),
- Server.getRemoteIp(),
+ getRemoteIp(),
"listStatus", src, null, null);
}
dl = dir.getListing(src, startAfter, needLocation);
@@ -5235,7 +5252,15 @@ public class FSNamesystem implements Nam
* RPC call context even if the client exits.
*/
private boolean isExternalInvocation() {
- return Server.isRpcInvocation();
+ return Server.isRpcInvocation() || NamenodeWebHdfsMethods.isWebHdfsInvocation();
+ }
+
+ private static InetAddress getRemoteIp() {
+ InetAddress ip = Server.getRemoteIp();
+ if (ip != null) {
+ return ip;
+ }
+ return NamenodeWebHdfsMethods.getRemoteIp();
}
/**
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1379281&r1=1379280&r2=1379281&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Fri Aug 31 04:12:37 2012
@@ -21,6 +21,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
@@ -92,6 +93,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
@@ -116,9 +118,20 @@ public class NamenodeWebHdfsMethods {
return REMOTE_ADDRESS.get();
}
- /** Set the remote client address. */
- static void setRemoteAddress(String remoteAddress) {
- REMOTE_ADDRESS.set(remoteAddress);
+ public static InetAddress getRemoteIp() {
+ try {
+ return InetAddress.getByName(getRemoteAddress());
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Returns true if a WebHdfs request is in progress. Akin to
+ * {@link Server#isRpcInvocation()}.
+ */
+ public static boolean isWebHdfsInvocation() {
+ return getRemoteAddress() != null;
}
private @Context ServletContext context;
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1379278
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1379278
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1379278
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1379278
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1379278
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1379281&r1=1379280&r2=1379281&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Fri Aug 31 04:12:37 2012
@@ -61,7 +61,7 @@ public class NameNodeAdapter {
public static HdfsFileStatus getFileInfo(NameNode namenode, String src,
boolean resolveLink) throws AccessControlException, UnresolvedLinkException,
- StandbyException {
+ StandbyException, IOException {
return namenode.getNamesystem().getFileInfo(src, resolveLink);
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java?rev=1379281&r1=1379280&r2=1379281&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java Fri Aug 31 04:12:37 2012
@@ -32,13 +32,17 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.HftpFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -83,6 +87,7 @@ public class TestAuditLogs {
final long precision = 1L;
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
+ conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
util = new DFSTestUtil.Builder().setName("TestAuditAllowed").
setNumFiles(20).build();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
@@ -115,6 +120,18 @@ public class TestAuditLogs {
assertTrue("failed to read from file", val > 0);
}
+ /** test that allowed stat puts proper entry in audit log */
+ @Test
+ public void testAuditAllowedStat() throws Exception {
+ final Path file = new Path(fnames[0]);
+ FileSystem userfs = DFSTestUtil.getFileSystemAs(userGroupInfo, conf);
+
+ setupAuditLogs();
+ FileStatus st = userfs.getFileStatus(file);
+ verifyAuditLogs(true);
+ assertTrue("failed to stat file", st != null && st.isFile());
+ }
+
/** test that denied operation puts proper entry in audit log */
@Test
public void testAuditDenied() throws Exception {
@@ -135,6 +152,85 @@ public class TestAuditLogs {
verifyAuditLogs(false);
}
+ /** test that access via webhdfs puts proper entry in audit log */
+ @Test
+ public void testAuditWebHdfs() throws Exception {
+ final Path file = new Path(fnames[0]);
+
+ fs.setPermission(file, new FsPermission((short)0644));
+ fs.setOwner(file, "root", null);
+
+ setupAuditLogs();
+
+ WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf);
+ InputStream istream = webfs.open(file);
+ int val = istream.read();
+ istream.close();
+
+ verifyAuditLogsRepeat(true, 3);
+ assertTrue("failed to read from file", val > 0);
+ }
+
+ /** test that stat via webhdfs puts proper entry in audit log */
+ @Test
+ public void testAuditWebHdfsStat() throws Exception {
+ final Path file = new Path(fnames[0]);
+
+ fs.setPermission(file, new FsPermission((short)0644));
+ fs.setOwner(file, "root", null);
+
+ setupAuditLogs();
+
+ WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf);
+ FileStatus st = webfs.getFileStatus(file);
+
+ verifyAuditLogs(true);
+ assertTrue("failed to stat file", st != null && st.isFile());
+ }
+
+ /** test that access via Hftp puts proper entry in audit log */
+ @Test
+ public void testAuditHftp() throws Exception {
+ final Path file = new Path(fnames[0]);
+
+ final String hftpUri =
+ "hftp://" + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+
+ HftpFileSystem hftpFs = null;
+
+ setupAuditLogs();
+ try {
+ hftpFs = (HftpFileSystem) new Path(hftpUri).getFileSystem(conf);
+ InputStream istream = hftpFs.open(file);
+ int val = istream.read();
+ istream.close();
+
+ verifyAuditLogs(true);
+ } finally {
+ if (hftpFs != null) hftpFs.close();
+ }
+ }
+
+ /** test that denied access via webhdfs puts proper entry in audit log */
+ @Test
+ public void testAuditWebHdfsDenied() throws Exception {
+ final Path file = new Path(fnames[0]);
+
+ fs.setPermission(file, new FsPermission((short)0600));
+ fs.setOwner(file, "root", null);
+
+ setupAuditLogs();
+ try {
+ WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf);
+ InputStream istream = webfs.open(file);
+ int val = istream.read();
+ fail("open+read must not succeed, got " + val);
+ } catch(AccessControlException E) {
+ System.out.println("got access denied, as expected.");
+ }
+ verifyAuditLogsRepeat(false, 2);
+ }
+
/** Sets up log4j logger for auditlogs */
private void setupAuditLogs() throws IOException {
File file = new File(auditLogFile);
@@ -148,19 +244,34 @@ public class TestAuditLogs {
logger.addAppender(appender);
}
+ // Ensure audit log has only one entry
private void verifyAuditLogs(boolean expectSuccess) throws IOException {
+ verifyAuditLogsRepeat(expectSuccess, 1);
+ }
+
+ // Ensure audit log has exactly N entries
+ private void verifyAuditLogsRepeat(boolean expectSuccess, int ndupe)
+ throws IOException {
// Turn off the logs
Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger();
logger.setLevel(Level.OFF);
- // Ensure audit log has only one entry
BufferedReader reader = new BufferedReader(new FileReader(auditLogFile));
- String line = reader.readLine();
- assertNotNull(line);
- assertTrue("Expected audit event not found in audit log",
- auditPattern.matcher(line).matches());
- assertTrue("Expected success=" + expectSuccess,
- successPattern.matcher(line).matches() == expectSuccess);
- assertNull("Unexpected event in audit log", reader.readLine());
+ String line = null;
+ boolean ret = true;
+
+ try {
+ for (int i = 0; i < ndupe; i++) {
+ line = reader.readLine();
+ assertNotNull(line);
+ assertTrue("Expected audit event not found in audit log",
+ auditPattern.matcher(line).matches());
+ ret &= successPattern.matcher(line).matches();
+ }
+ assertNull("Unexpected event in audit log", reader.readLine());
+ assertTrue("Expected success=" + expectSuccess, ret == expectSuccess);
+ } finally {
+ reader.close();
+ }
}
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1379281&r1=1379280&r2=1379281&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java Fri Aug 31 04:12:37 2012
@@ -95,6 +95,12 @@ public class TestFsck {
"ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
"cmd=fsck\\ssrc=\\/\\sdst=null\\s" +
"perm=null");
+ static final Pattern getfileinfoPattern = Pattern.compile(
+ "allowed=.*?\\s" +
+ "ugi=.*?\\s" +
+ "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+ "cmd=getfileinfo\\ssrc=\\/\\sdst=null\\s" +
+ "perm=null");
static final Pattern numCorruptBlocksPattern = Pattern.compile(
".*Corrupt blocks:\t\t([0123456789]*).*");
@@ -180,10 +186,14 @@ public class TestFsck {
Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger();
logger.setLevel(Level.OFF);
- // Ensure audit log has only one for FSCK
+ // Audit log should contain one getfileinfo and one fsck
BufferedReader reader = new BufferedReader(new FileReader(auditLogFile));
String line = reader.readLine();
assertNotNull(line);
+ assertTrue("Expected getfileinfo event not found in audit log",
+ getfileinfoPattern.matcher(line).matches());
+ line = reader.readLine();
+ assertNotNull(line);
assertTrue("Expected fsck event not found in audit log",
fsckPattern.matcher(line).matches());
assertNull("Unexpected event in audit log", reader.readLine());
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java?rev=1379281&r1=1379280&r2=1379281&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java Fri Aug 31 04:12:37 2012
@@ -89,7 +89,6 @@ public class TestWebHdfsDataLocality {
//set client address to a particular datanode
final DataNode dn = cluster.getDataNodes().get(i);
final String ipAddr = dm.getDatanode(dn.getDatanodeId()).getIpAddr();
- NamenodeWebHdfsMethods.setRemoteAddress(ipAddr);
//The chosen datanode must be the same as the client address
final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(