You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/09/23 20:14:07 UTC
[hadoop] branch branch-3.2 updated: YARN-9762. Add submission
context label to audit logs. Contributed by Manoj Kumar
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new a1fa9a8 YARN-9762. Add submission context label to audit logs. Contributed by Manoj Kumar
a1fa9a8 is described below
commit a1fa9a8a7f79a1a711cd881b526724b502e03456
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Mon Sep 23 11:42:41 2019 -0700
YARN-9762. Add submission context label to audit logs. Contributed by Manoj Kumar
(cherry picked from commit 3d78b1223d3fdc29d500803cefd2931b54f44928)
---
.../server/resourcemanager/ClientRMService.java | 6 ++-
.../yarn/server/resourcemanager/RMAuditLogger.java | 53 ++++++++++++++++++++--
.../server/resourcemanager/TestRMAuditLogger.java | 21 +++++++--
3 files changed, 69 insertions(+), 11 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 3d1f01d..571add2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -694,13 +694,15 @@ public class ClientRMService extends AbstractService implements
" submitted by user " + user);
RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST,
"ClientRMService", applicationId, callerContext,
- submissionContext.getQueue());
+ submissionContext.getQueue(),
+ submissionContext.getNodeLabelExpression());
} catch (YarnException e) {
LOG.info("Exception in submitting " + applicationId, e);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
e.getMessage(), "ClientRMService",
"Exception in submitting application", applicationId, callerContext,
- submissionContext.getQueue());
+ submissionContext.getQueue(),
+ submissionContext.getNodeLabelExpression());
throw e;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index 292aa8b..06ac64c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -275,6 +275,16 @@ public class RMAuditLogger {
}
}
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, CallerContext callerContext, String queueName,
+ String partition) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(
+ createSuccessLog(user, operation, target, appId, null, null, null,
+ callerContext, Server.getRemoteIp(), queueName, partition));
+ }
+ }
+
/**
* Create a readable and parseable audit log string for a successful event.
*
@@ -395,7 +405,8 @@ public class RMAuditLogger {
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
- Resource resource, CallerContext callerContext, String queueName) {
+ Resource resource, CallerContext callerContext, String queueName,
+ String partition) {
StringBuilder b = createStringBuilderForFailureLog(user,
operation, target, description, perm);
if (appId != null) {
@@ -414,6 +425,10 @@ public class RMAuditLogger {
if (queueName != null) {
add(Keys.QUEUENAME, queueName, b);
}
+ if (partition != null) {
+ add(Keys.NODELABEL, partition, b);
+ }
+
return b.toString();
}
@@ -424,7 +439,7 @@ public class RMAuditLogger {
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) {
return createFailureLog(user, operation, perm, target, description, appId,
- attemptId, containerId, resource, null, null);
+ attemptId, containerId, resource, null, null, null);
}
/**
@@ -496,7 +511,7 @@ public class RMAuditLogger {
CallerContext callerContext) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
- appId, null, null, null, callerContext, null));
+ appId, null, null, null, callerContext, null, null));
}
}
@@ -505,7 +520,7 @@ public class RMAuditLogger {
CallerContext callerContext, String queueName) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
- appId, null, null, null, callerContext, queueName));
+ appId, null, null, null, callerContext, queueName, null));
}
}
@@ -537,7 +552,7 @@ public class RMAuditLogger {
String queueName) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
- appId, null, null, null, null, queueName));
+ appId, null, null, null, null, queueName, null));
}
}
@@ -586,6 +601,34 @@ public class RMAuditLogger {
}
/**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param perm Target permissions.
+ * @param target The target on which the operation is being performed.
+ * @param description Some additional information as to why the operation
+ * failed.
+ * @param appId ApplicationId in which operation was performed.
+ * @param callerContext Caller context
+ * @param queueName Name of queue.
+ * @param partition Name of labeled partition.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ CallerContext callerContext, String queueName, String partition) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(
+ createFailureLog(user, operation, perm, target, description, appId,
+ null, null, null, callerContext, queueName, partition));
+ }
+ }
+
+ /**
* A helper api to add remote IP address.
*/
static void addRemoteIP(StringBuilder b) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index 7d8eeab..84ea2d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -293,16 +293,17 @@ public class TestRMAuditLogger {
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource) {
testFailureLogFormatHelper(checkIP, appId, attemptId, containerId,
- callerContext, resource, null);
+ callerContext, resource, null, null, null);
}
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource,
- RMAuditLogger.ArgsBuilder args) {
+ String queueName, String partition, RMAuditLogger.ArgsBuilder args) {
String fLog = args == null ?
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
- appId, attemptId, containerId, resource, callerContext, null) :
+ appId, attemptId, containerId, resource, callerContext,
+ queueName, partition) :
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
args);
StringBuilder expLog = new StringBuilder();
@@ -334,6 +335,12 @@ public class TestRMAuditLogger {
expLog.append("\tCALLERSIGNATURE=signature");
}
}
+ if (queueName != null) {
+ expLog.append("\tQUEUENAME=" + QUEUE);
+ }
+ if (partition != null) {
+ expLog.append("\tNODELABEL=" + PARTITION);
+ }
if (args != null) {
expLog.append("\tQUEUENAME=root");
expLog.append("\tRECURSIVE=true");
@@ -364,10 +371,16 @@ public class TestRMAuditLogger {
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
.build(), RESOURCE);
+ testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
+ new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
+ .build(), RESOURCE, QUEUE, null, null);
+ testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
+ new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
+ .build(), RESOURCE, QUEUE, PARTITION, null);
RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder()
.append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
testFailureLogFormatHelper(checkIP, null, null, null, null, null,
- args);
+ null, null, args);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org