You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/09/07 03:26:21 UTC
svn commit: r1165950 - in
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server:
hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
hadoop-yarn-server-nodemanager/src/test/java/org/apache/hado...
Author: acmurthy
Date: Wed Sep 7 01:26:21 2011
New Revision: 1165950
URL: http://svn.apache.org/viewvc?rev=1165950&view=rev
Log:
Adding new files for MAPREDUCE-2655.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java Wed Sep 7 01:26:21 2011
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.net.InetAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/**
+ * Manages NodeManager audit logs.
+ *
+ * Audit log format is written as key=value pairs. Tab separated.
+ */
+public class NMAuditLogger {
+ private static final Log LOG = LogFactory.getLog(NMAuditLogger.class);
+
+ static enum Keys {USER, OPERATION, TARGET, RESULT, IP,
+ DESCRIPTION, APPID, CONTAINERID}
+
+ public static class AuditConstants {
+ static final String SUCCESS = "SUCCESS";
+ static final String FAILURE = "FAILURE";
+ static final String KEY_VAL_SEPARATOR = "=";
+ static final char PAIR_SEPARATOR = '\t';
+
+ // Some commonly used descriptions
+ public static final String START_CONTAINER = "Start Container Request";
+ public static final String STOP_CONTAINER = "Stop Container Request";
+ public static final String FINISH_SUCCESS_CONTAINER = "Container Finished - Succeeded";
+ public static final String FINISH_FAILED_CONTAINER = "Container Finished - Failed";
+ public static final String FINISH_KILLED_CONTAINER = "Container Finished - Killed";
+ }
+
+ /**
+ * A helper api for creating an audit log for a successful event.
+ */
+ static String createSuccessLog(String user, String operation, String target,
+ ApplicationId appId, ContainerId containerId) {
+ StringBuilder b = new StringBuilder();
+ start(Keys.USER, user, b);
+ addRemoteIP(b);
+ add(Keys.OPERATION, operation, b);
+ add(Keys.TARGET, target ,b);
+ add(Keys.RESULT, AuditConstants.SUCCESS, b);
+ if (appId != null) {
+ add(Keys.APPID, appId.toString(), b);
+ }
+ if (containerId != null) {
+ add(Keys.CONTAINERID, containerId.toString(), b);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user
+ * @param target The target on which the operation is being performed.
+ * @param appId Application Id in which operation was performed.
+ * @param containerId Container Id in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, ContainerId containerId) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, containerId));
+ }
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user
+ * @param target The target on which the operation is being performed.
+ *
+ * <br><br>
+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, null, null));
+ }
+ }
+
+ /**
+ * A helper api for creating an audit log for a failure event.
+ * This is factored out for testing purpose.
+ */
+ static String createFailureLog(String user, String operation, String target,
+ String description, ApplicationId appId, ContainerId containerId) {
+ StringBuilder b = new StringBuilder();
+ start(Keys.USER, user, b);
+ addRemoteIP(b);
+ add(Keys.OPERATION, operation, b);
+ add(Keys.TARGET, target ,b);
+ add(Keys.RESULT, AuditConstants.FAILURE, b);
+ add(Keys.DESCRIPTION, description, b);
+ if (appId != null) {
+ add(Keys.APPID, appId.toString(), b);
+ }
+ if (containerId != null) {
+ add(Keys.CONTAINERID, containerId.toString(), b);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param description Some additional information as to why the operation
+ * failed.
+ * @param appId ApplicationId in which operation was performed.
+ * @param containerId Container Id in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String target,
+ String description, ApplicationId appId, ContainerId containerId) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, target, description, appId, containerId));
+ }
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param description Some additional information as to why the operation
+ * failed.
+ *
+ * <br><br>
+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation,
+ String target, String description) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, target, description, null, null));
+ }
+ }
+
+ /**
+ * A helper api to add remote IP address
+ */
+ static void addRemoteIP(StringBuilder b) {
+ InetAddress ip = Server.getRemoteIp();
+ // ip address can be null for testcases
+ if (ip != null) {
+ add(Keys.IP, ip.getHostAddress(), b);
+ }
+ }
+
+ /**
+ * Adds the first key-val pair to the passed builder in the following format
+ * key=value
+ */
+ static void start(Keys key, String value, StringBuilder b) {
+ b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+ }
+
+ /**
+ * Appends the key-val pair to the passed builder in the following format
+ * <pair-delim>key=value
+ */
+ static void add(Keys key, String value, StringBuilder b) {
+ b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
+ .append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java Wed Sep 7 01:26:21 2011
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.TestRPC.TestImpl;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
+
+import org.apache.hadoop.net.NetUtils;
+
+import static org.mockito.Mockito.*;
+import static junit.framework.Assert.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Tests {@link NMAuditLogger}.
+ */
+public class TestNMAuditLogger {
+ private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class);
+ private static final String USER = "test";
+ private static final String OPERATION = "oper";
+ private static final String TARGET = "tgt";
+ private static final String DESC = "description of an audit log";
+
+ private static final ApplicationId APPID = mock(ApplicationId.class);
+ private static final ContainerId CONTAINERID = mock(ContainerId.class);
+
+ @Before
+ public void setUp() throws Exception {
+ when(APPID.toString()).thenReturn("app_1");
+ when(CONTAINERID.toString()).thenReturn("container_1");
+ }
+
+
+ /**
+ * Test the AuditLog format with key-val pair.
+ */
+ @Test
+ public void testKeyValLogFormat() throws Exception {
+ StringBuilder actLog = new StringBuilder();
+ StringBuilder expLog = new StringBuilder();
+ // add the first k=v pair and check
+ NMAuditLogger.start(Keys.USER, USER, actLog);
+ expLog.append("USER=test");
+ assertEquals(expLog.toString(), actLog.toString());
+
+ // append another k1=v1 pair to already added k=v and test
+ NMAuditLogger.add(Keys.OPERATION, OPERATION, actLog);
+ expLog.append("\tOPERATION=oper");
+ assertEquals(expLog.toString(), actLog.toString());
+
+ // append another k1=null pair and test
+ NMAuditLogger.add(Keys.APPID, (String)null, actLog);
+ expLog.append("\tAPPID=null");
+ assertEquals(expLog.toString(), actLog.toString());
+
+ // now add the target and check of the final string
+ NMAuditLogger.add(Keys.TARGET, TARGET, actLog);
+ expLog.append("\tTARGET=tgt");
+ assertEquals(expLog.toString(), actLog.toString());
+ }
+
+
+ /**
+ * Test the AuditLog format for successful events.
+ */
+ private void testSuccessLogFormatHelper(boolean checkIP,
+ ApplicationId appId, ContainerId containerId) {
+ // check without the IP
+ String sLog = NMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
+ appId, containerId);
+ StringBuilder expLog = new StringBuilder();
+ expLog.append("USER=test\t");
+ if (checkIP) {
+ InetAddress ip = Server.getRemoteIp();
+ expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+ }
+ expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
+ if (appId != null) {
+ expLog.append("\tAPPID=app_1");
+ }
+ if (containerId != null) {
+ expLog.append("\tCONTAINERID=container_1");
+ }
+ assertEquals(expLog.toString(), sLog);
+ }
+
+ /**
+ * Test the AuditLog format for successful events passing nulls.
+ */
+ private void testSuccessLogNulls(boolean checkIP) {
+ String sLog = NMAuditLogger.createSuccessLog(null, null, null,
+ null, null);
+ StringBuilder expLog = new StringBuilder();
+ expLog.append("USER=null\t");
+ if (checkIP) {
+ InetAddress ip = Server.getRemoteIp();
+ expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+ }
+ expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS");
+ assertEquals(expLog.toString(), sLog);
+ }
+
+ /**
+ * Test the AuditLog format for successful events with the various
+ * parameters.
+ */
+ private void testSuccessLogFormat(boolean checkIP) {
+ testSuccessLogFormatHelper(checkIP, null, null);
+ testSuccessLogFormatHelper(checkIP, APPID, null);
+ testSuccessLogFormatHelper(checkIP, null, CONTAINERID);
+ testSuccessLogFormatHelper(checkIP, APPID, CONTAINERID);
+ testSuccessLogNulls(checkIP);
+ }
+
+
+ /**
+ * Test the AuditLog format for failure events.
+ */
+ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
+ ContainerId containerId) {
+ String fLog =
+ NMAuditLogger.createFailureLog(USER, OPERATION, TARGET, DESC, appId,
+ containerId);
+ StringBuilder expLog = new StringBuilder();
+ expLog.append("USER=test\t");
+ if (checkIP) {
+ InetAddress ip = Server.getRemoteIp();
+ expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+ }
+ expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
+ expLog.append("DESCRIPTION=description of an audit log");
+
+ if (appId != null) {
+ expLog.append("\tAPPID=app_1");
+ }
+ if (containerId != null) {
+ expLog.append("\tCONTAINERID=container_1");
+ }
+ assertEquals(expLog.toString(), fLog);
+ }
+
+ /**
+ * Test the AuditLog format for failure events with the various
+ * parameters.
+ */
+ private void testFailureLogFormat(boolean checkIP) {
+ testFailureLogFormatHelper(checkIP, null, null);
+ testFailureLogFormatHelper(checkIP, APPID, null);
+ testFailureLogFormatHelper(checkIP, null, CONTAINERID);
+ testFailureLogFormatHelper(checkIP, APPID, CONTAINERID);
+ }
+
+ /**
+ * Test {@link NMAuditLogger} without IP set.
+ */
+ @Test
+ public void testNMAuditLoggerWithoutIP() throws Exception {
+ // test without ip
+ testSuccessLogFormat(false);
+ testFailureLogFormat(false);
+ }
+
+ /**
+ * A special extension of {@link TestImpl} RPC server with
+ * {@link TestImpl#ping()} testing the audit logs.
+ */
+ private class MyTestRPCServer extends TestImpl {
+ @Override
+ public void ping() {
+ // test with ip set
+ testSuccessLogFormat(true);
+ testFailureLogFormat(true);
+ }
+ }
+
+ /**
+ * Test {@link NMAuditLogger} with IP set.
+ */
+ @Test
+ public void testNMAuditLoggerWithIP() throws Exception {
+ Configuration conf = new Configuration();
+ // start the IPC server
+ Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
+ server.start();
+
+ InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
+ // Make a client connection and test the audit log
+ TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ // Start the testcase
+ proxy.ping();
+
+ server.stop();
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java Wed Sep 7 01:26:21 2011
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.net.InetAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/**
+ * Manages ResourceManager audit logs.
+ *
+ * Audit log format is written as key=value pairs. Tab separated.
+ */
+public class RMAuditLogger {
+ private static final Log LOG = LogFactory.getLog(RMAuditLogger.class);
+
+ static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
+ DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID}
+
+ public static class AuditConstants {
+ static final String SUCCESS = "SUCCESS";
+ static final String FAILURE = "FAILURE";
+ static final String KEY_VAL_SEPARATOR = "=";
+ static final char PAIR_SEPARATOR = '\t';
+
+ public static final String KILL_APP_REQUEST = "Kill Application Request";
+ public static final String SUBMIT_APP_REQUEST = "Submit Application Request";
+ public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded";
+ public static final String FINISH_FAILED_APP = "Application Finished - Failed";
+ public static final String FINISH_KILLED_APP = "Application Finished - Killed";
+ public static final String REGISTER_AM = "Register App Master";
+ public static final String ALLOC_CONTAINER = "AM Allocated Container";
+ public static final String RELEASE_CONTAINER = "AM Released Container";
+
+ // Some commonly used descriptions
+ public static final String UNAUTHORIZED_USER = "Unauthorized user";
+ }
+
+ /**
+ * A helper api for creating an audit log for a successful event.
+ */
+ static String createSuccessLog(String user, String operation, String target,
+ ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) {
+ StringBuilder b = new StringBuilder();
+ start(Keys.USER, user, b);
+ addRemoteIP(b);
+ add(Keys.OPERATION, operation, b);
+ add(Keys.TARGET, target ,b);
+ add(Keys.RESULT, AuditConstants.SUCCESS, b);
+ if (appId != null) {
+ add(Keys.APPID, appId.toString(), b);
+ }
+ if (attemptId != null) {
+ add(Keys.APPATTEMPTID, attemptId.toString(), b);
+ }
+ if (containerId != null) {
+ add(Keys.CONTAINERID, containerId.toString(), b);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request to the ResourceManager
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param appId Application Id in which operation was performed.
+ * @param containerId Container Id in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, ContainerId containerId) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, null,
+ containerId));
+ }
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request to the ResourceManager.
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param appId Application Id in which operation was performed.
+ * @param attemptId Application Attempt Id in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, ApplicationAttemptId attemptId) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, attemptId,
+ null));
+ }
+ }
+
+
+ /**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request to the ResourceManager.
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param appId Application Id in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, null, null));
+ }
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, null, null, null));
+ }
+ }
+
+ /**
+ * A helper api for creating an audit log for a failure event.
+ */
+ static String createFailureLog(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId) {
+ StringBuilder b = new StringBuilder();
+ start(Keys.USER, user, b);
+ addRemoteIP(b);
+ add(Keys.OPERATION, operation, b);
+ add(Keys.TARGET, target ,b);
+ add(Keys.RESULT, AuditConstants.FAILURE, b);
+ add(Keys.DESCRIPTION, description, b);
+ add(Keys.PERMISSIONS, perm, b);
+ if (appId != null) {
+ add(Keys.APPID, appId.toString(), b);
+ }
+ if (attemptId != null) {
+ add(Keys.APPATTEMPTID, attemptId.toString(), b);
+ }
+ if (containerId != null) {
+ add(Keys.CONTAINERID, containerId.toString(), b);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param perm Target permissions.
+ * @param target The target on which the operation is being performed.
+ * @param description Some additional information as to why the operation
+ * failed.
+ * @param appId Application Id in which operation was performed.
+ * @param containerId Container Id in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ ContainerId containerId) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ appId, null, containerId));
+ }
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param perm Target permissions.
+ * @param target The target on which the operation is being performed.
+ * @param description Some additional information as to why the operation
+ * failed.
+ * @param appId ApplicationId in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ ApplicationAttemptId attemptId) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ appId, attemptId, null));
+ }
+ }
+
+
+ /**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param perm Target permissions.
+ * @param target The target on which the operation is being performed.
+ * @param description Some additional information as to why the operation
+ * failed.
+ * @param appId ApplicationId in which operation was performed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ApplicationId appId) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ appId, null, null));
+ }
+ }
+
+ /**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param perm Target permissions.
+ * @param target The target on which the operation is being performed.
+ * @param description Some additional information as to why the operation
+ * failed.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ null, null, null));
+ }
+ }
+
+ /**
+ * A helper api to add remote IP address
+ */
+ static void addRemoteIP(StringBuilder b) {
+ InetAddress ip = Server.getRemoteIp();
+ // ip address can be null for testcases
+ if (ip != null) {
+ add(Keys.IP, ip.getHostAddress(), b);
+ }
+ }
+
+ /**
+ * Adds the first key-val pair to the passed builder in the following format
+ * key=value
+ */
+ static void start(Keys key, String value, StringBuilder b) {
+ b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+ }
+
+ /**
+ * Appends the key-val pair to the passed builder in the following format
+ * <pair-delim>key=value
+ */
+ static void add(Keys key, String value, StringBuilder b) {
+ b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
+ .append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java Wed Sep 7 01:26:21 2011
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.TestRPC.TestImpl;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
+
+import org.apache.hadoop.net.NetUtils;
+
+import static org.mockito.Mockito.*;
+import static junit.framework.Assert.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Tests {@link RMAuditLogger}.
+ */
+public class TestRMAuditLogger {
+ private static final Log LOG = LogFactory.getLog(TestRMAuditLogger.class);
+ private static final String USER = "test";
+ private static final String OPERATION = "oper";
+ private static final String TARGET = "tgt";
+ private static final String PERM = "admin group";
+ private static final String DESC = "description of an audit log";
+ private static final ApplicationId APPID = mock(ApplicationId.class);
+ private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class);
+ private static final ContainerId CONTAINERID = mock(ContainerId.class);
+
+ @Before
+ public void setUp() throws Exception {
+ when(APPID.toString()).thenReturn("app_1");
+ when(ATTEMPTID.toString()).thenReturn("app_attempt_1");
+ when(CONTAINERID.toString()).thenReturn("container_1");
+ }
+
+
+ /**
+ * Test the AuditLog format with key-val pair.
+ */
+ @Test
+ public void testKeyValLogFormat() throws Exception {
+ StringBuilder actLog = new StringBuilder();
+ StringBuilder expLog = new StringBuilder();
+ // add the first k=v pair and check
+ RMAuditLogger.start(Keys.USER, USER, actLog);
+ expLog.append("USER=test");
+ assertEquals(expLog.toString(), actLog.toString());
+
+ // append another k1=v1 pair to already added k=v and test
+ RMAuditLogger.add(Keys.OPERATION, OPERATION, actLog);
+ expLog.append("\tOPERATION=oper");
+ assertEquals(expLog.toString(), actLog.toString());
+
+ // append another k1=null pair and test
+ RMAuditLogger.add(Keys.APPID, (String)null, actLog);
+ expLog.append("\tAPPID=null");
+ assertEquals(expLog.toString(), actLog.toString());
+
+ // now add the target and check of the final string
+ RMAuditLogger.add(Keys.TARGET, TARGET, actLog);
+ expLog.append("\tTARGET=tgt");
+ assertEquals(expLog.toString(), actLog.toString());
+ }
+
+
+ /**
+ * Test the AuditLog format for successful events.
+ */
+ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId) {
+ String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
+ appId, attemptId, containerId);
+ StringBuilder expLog = new StringBuilder();
+ expLog.append("USER=test\t");
+ if (checkIP) {
+ InetAddress ip = Server.getRemoteIp();
+ expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+ }
+ expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
+
+ if (appId != null) {
+ expLog.append("\tAPPID=app_1");
+ }
+ if (attemptId != null) {
+ expLog.append("\tAPPATTEMPTID=app_attempt_1");
+ }
+ if (containerId != null) {
+ expLog.append("\tCONTAINERID=container_1");
+ }
+ assertEquals(expLog.toString(), sLog);
+ }
+
+ /**
+ * Test the AuditLog format for successful events passing nulls.
+ */
+ private void testSuccessLogNulls(boolean checkIP) {
+ String sLog = RMAuditLogger.createSuccessLog(null, null, null, null,
+ null, null);
+ StringBuilder expLog = new StringBuilder();
+ expLog.append("USER=null\t");
+ if (checkIP) {
+ InetAddress ip = Server.getRemoteIp();
+ expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+ }
+ expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS");
+ assertEquals(expLog.toString(), sLog);
+ }
+
+ /**
+ * Test the AuditLog format for successful events with the various
+ * parameters.
+ */
+ private void testSuccessLogFormat(boolean checkIP) {
+ testSuccessLogFormatHelper(checkIP, null, null, null);
+ testSuccessLogFormatHelper(checkIP, APPID, null, null);
+ testSuccessLogFormatHelper(checkIP, null, null, CONTAINERID);
+ testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, null);
+ testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, null);
+ testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID);
+ testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
+ testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
+ testSuccessLogNulls(checkIP);
+ }
+
+
+ /**
+ * Test the AuditLog format for failure events.
+ */
+ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId) {
+ String fLog =
+ RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
+ appId, attemptId, containerId);
+ StringBuilder expLog = new StringBuilder();
+ expLog.append("USER=test\t");
+ if (checkIP) {
+ InetAddress ip = Server.getRemoteIp();
+ expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+ }
+ expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
+ expLog.append("DESCRIPTION=description of an audit log");
+ expLog.append("\tPERMISSIONS=admin group");
+ if (appId != null) {
+ expLog.append("\tAPPID=app_1");
+ }
+ if (attemptId != null) {
+ expLog.append("\tAPPATTEMPTID=app_attempt_1");
+ }
+ if (containerId != null) {
+ expLog.append("\tCONTAINERID=container_1");
+ }
+ assertEquals(expLog.toString(), fLog);
+ }
+
+ /**
+ * Test the AuditLog format for failure events with the various
+ * parameters.
+ */
+ private void testFailureLogFormat(boolean checkIP) {
+ testFailureLogFormatHelper(checkIP, null, null, null);
+ testFailureLogFormatHelper(checkIP, APPID, null, null);
+ testFailureLogFormatHelper(checkIP, null, null, CONTAINERID);
+ testFailureLogFormatHelper(checkIP, null, ATTEMPTID, null);
+ testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, null);
+ testFailureLogFormatHelper(checkIP, APPID, null, CONTAINERID);
+ testFailureLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
+ testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
+ }
+
+ /**
+ * Test {@link RMAuditLogger} without IP set.
+ */
+ @Test
+ public void testRMAuditLoggerWithoutIP() throws Exception {
+ // test without ip
+ testSuccessLogFormat(false);
+ testFailureLogFormat(false);
+ }
+
+ /**
+ * A special extension of {@link TestImpl} RPC server with
+ * {@link TestImpl#ping()} testing the audit logs.
+ */
+ private class MyTestRPCServer extends TestImpl {
+ @Override
+ public void ping() {
+ // test with ip set
+ testSuccessLogFormat(true);
+ testFailureLogFormat(true);
+ }
+ }
+
+ /**
+ * Test {@link RMAuditLogger} with IP set.
+ */
+ @Test
+ public void testRMAuditLoggerWithIP() throws Exception {
+ Configuration conf = new Configuration();
+ // start the IPC server
+ Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
+ server.start();
+
+ InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
+ // Make a client connection and test the audit log
+ TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ // Start the testcase
+ proxy.ping();
+
+ server.stop();
+ }
+}