You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2015/04/10 02:34:09 UTC
oozie git commit: OOZIE-2140 Audit Log should be shown in Oozie UI
Repository: oozie
Updated Branches:
refs/heads/master f7f7628b8 -> 3fe9c2a89
OOZIE-2140 Audit Log should be shown in Oozie UI
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3fe9c2a8
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3fe9c2a8
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3fe9c2a8
Branch: refs/heads/master
Commit: 3fe9c2a897d29cbe6272c53caa02a43065b5db18
Parents: f7f7628
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Thu Apr 9 17:33:58 2015 -0700
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Thu Apr 9 17:33:58 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/oozie/cli/OozieCLI.java | 14 ++-
.../org/apache/oozie/client/OozieClient.java | 18 ++++
.../apache/oozie/client/rest/RestConstants.java | 2 +
.../main/java/org/apache/oozie/BaseEngine.java | 41 ++++++++
.../java/org/apache/oozie/BundleEngine.java | 51 ++++++----
.../org/apache/oozie/CoordinatorEngine.java | 38 ++++---
.../main/java/org/apache/oozie/DagEngine.java | 49 ++++++---
.../oozie/service/DagXLogInfoService.java | 2 +
.../org/apache/oozie/service/XLogService.java | 37 ++++++-
.../oozie/service/XLogStreamingService.java | 25 ++++-
.../oozie/service/ZKXLogStreamingService.java | 35 ++++++-
.../apache/oozie/servlet/BaseJobServlet.java | 8 ++
.../apache/oozie/servlet/JsonRestServlet.java | 8 +-
.../org/apache/oozie/servlet/V0JobServlet.java | 5 +
.../org/apache/oozie/servlet/V1JobServlet.java | 6 +-
.../org/apache/oozie/servlet/V2JobServlet.java | 18 ++++
.../oozie/util/TimestampedMessageParser.java | 5 +-
.../java/org/apache/oozie/util/XLogFilter.java | 7 +-
.../oozie/service/TestXLogStreamingService.java | 86 +++++++++++++---
.../resources/test-no-dash-log4j.properties | 13 +++
docs/src/site/twiki/DG_CommandLineTool.twiki | 16 +++
docs/src/site/twiki/WebServicesAPI.twiki | 23 +++++
release-log.txt | 1 +
webapp/src/main/webapp/oozie-console.js | 100 ++++++++++++++++++-
24 files changed, 523 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
index 22e1021..aa7ad6c 100644
--- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
+++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
@@ -117,6 +117,7 @@ public class OozieCLI {
public static final String INFO_OPTION = "info";
public static final String LOG_OPTION = "log";
public static final String ERROR_LOG_OPTION = "errorlog";
+ public static final String AUDIT_LOG_OPTION = "auditlog";
public static final String ACTION_OPTION = "action";
public static final String DEFINITION_OPTION = "definition";
@@ -339,7 +340,7 @@ public class OozieCLI {
"use time zone with the specified ID (default GMT).\nSee 'oozie info -timezones' for a list");
Option log = new Option(LOG_OPTION, true, "job log");
Option errorlog = new Option(ERROR_LOG_OPTION, true, "job error log");
-
+ Option auditlog = new Option(AUDIT_LOG_OPTION, true, "job audit log");
Option logFilter = new Option(
RestConstants.LOG_FILTER_OPTION, true,
"job log search parameter. Can be specified as -logfilter opt1=val1;opt2=val1;opt3=val1. "
@@ -396,6 +397,7 @@ public class OozieCLI {
actions.addOption(rerun);
actions.addOption(log);
actions.addOption(errorlog);
+ actions.addOption(auditlog);
actions.addOption(definition);
actions.addOption(config_content);
actions.addOption(ignore);
@@ -1219,7 +1221,15 @@ public class OozieCLI {
ps.close();
}
}
-
+ else if (options.contains(AUDIT_LOG_OPTION)) {
+ PrintStream ps = System.out;
+ try {
+ wc.getJobAuditLog(commandLine.getOptionValue(AUDIT_LOG_OPTION), ps);
+ }
+ finally {
+ ps.close();
+ }
+ }
else if (options.contains(DEFINITION_OPTION)) {
System.out.println(wc.getJobDefinition(commandLine.getOptionValue(DEFINITION_OPTION)));
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/client/src/main/java/org/apache/oozie/client/OozieClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java
index ceb193a..db21e9e 100644
--- a/client/src/main/java/org/apache/oozie/client/OozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java
@@ -1061,6 +1061,17 @@ public class OozieClient {
}
/**
+ * Get the audit log of a job.
+ *
+ * @param jobId
+ * @param ps
+ * @throws OozieClientException
+ */
+ public void getJobAuditLog(String jobId, PrintStream ps) throws OozieClientException {
+ new JobAuditLog(jobId, ps).call();
+ }
+
+ /**
* Get the log of a job.
*
* @param jobId job Id.
@@ -1115,6 +1126,13 @@ public class OozieClient {
}
}
+ private class JobAuditLog extends JobMetadata {
+ JobAuditLog(String jobId, PrintStream ps) {
+ super(jobId, RestConstants.JOB_SHOW_AUDIT_LOG, ps);
+ }
+ }
+
+
/**
* Gets the JMS topic name for a particular job
* @param jobId given jobId
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
index 4c75d2a..2434b19 100644
--- a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
+++ b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
@@ -89,6 +89,8 @@ public interface RestConstants {
public static final String JOB_SHOW_ERROR_LOG = "errorlog";
+ public static final String JOB_SHOW_AUDIT_LOG = "auditlog";
+
public static final String JOB_SHOW_DEFINITION = "definition";
public static final String JOB_SHOW_GRAPH = "graph";
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/BaseEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/BaseEngine.java b/core/src/main/java/org/apache/oozie/BaseEngine.java
index 44074ea..4ea87c0 100644
--- a/core/src/main/java/org/apache/oozie/BaseEngine.java
+++ b/core/src/main/java/org/apache/oozie/BaseEngine.java
@@ -20,6 +20,7 @@ package org.apache.oozie;
import java.io.IOException;
import java.io.Writer;
+import java.util.Date;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -28,10 +29,16 @@ import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.executor.jpa.JPAExecutorException;
import org.apache.oozie.service.JMSTopicService;
import org.apache.oozie.service.Services;
+import org.apache.oozie.service.XLogStreamingService;
+import org.apache.oozie.util.XLogFilter;
public abstract class BaseEngine {
public static final String USE_XCOMMAND = "oozie.useXCommand";
+ public enum LOG_TYPE {
+ LOG, ERROR_LOG, AUDIT_LOG
+ }
+
protected String user;
/**
@@ -182,6 +189,19 @@ public abstract class BaseEngine {
*/
public abstract void streamErrorLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
BaseEngineException;
+ /**
+ * Stream Audit log of a job.
+ *
+ * @param jobId job Id.
+ * @param writer writer to stream the log to.
+ * @param params additional parameters from the request
+ * @throws IOException thrown if the log cannot be streamed.
+ * @throws BaseEngineException thrown if there is error in getting the Workflow/Coordinator Job Information for
+ * jobId.
+ */
+ public abstract void streamAuditLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
+ BaseEngineException;
+
/**
* Return the workflow Job ID for an external ID.
@@ -279,4 +299,25 @@ public abstract class BaseEngine {
public abstract void changeSLA(String id, String actions, String dates, String childIds, String newParams)
throws BaseEngineException;
+ protected void fetchLog(XLogFilter filter, Date startTime, Date endTime, Writer writer,
+ Map<String, String[]> params, LOG_TYPE logType) throws IOException {
+
+ switch (logType) {
+ case LOG:
+ Services.get().get(XLogStreamingService.class).streamLog(filter, startTime, endTime, writer, params);
+ break;
+ case ERROR_LOG:
+ Services.get().get(XLogStreamingService.class)
+ .streamErrorLog(filter, startTime, endTime, writer, params);
+ break;
+ case AUDIT_LOG:
+ Services.get().get(XLogStreamingService.class)
+ .streamAuditLog(filter, startTime, endTime, writer, params);
+ break;
+ default:
+ throw new IOException("Unsupported log Type");
+ }
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/BundleEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/BundleEngine.java b/core/src/main/java/org/apache/oozie/BundleEngine.java
index e191184..bfb29e6 100644
--- a/core/src/main/java/org/apache/oozie/BundleEngine.java
+++ b/core/src/main/java/org/apache/oozie/BundleEngine.java
@@ -24,17 +24,14 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.BaseEngine.LOG_TYPE;
import org.apache.oozie.client.CoordinatorAction;
import org.apache.oozie.client.CoordinatorJob;
-import org.apache.oozie.client.Job;
-import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.client.rest.BulkResponseImpl;
import org.apache.oozie.command.BulkJobsXCommand;
@@ -56,18 +53,16 @@ import org.apache.oozie.command.OperationType;
import org.apache.oozie.executor.jpa.BundleJobQueryExecutor;
import org.apache.oozie.executor.jpa.JPAExecutorException;
import org.apache.oozie.service.DagXLogInfoService;
-import org.apache.oozie.service.Services;
-import org.apache.oozie.service.XLogStreamingService;
import org.apache.oozie.util.DateUtils;
import org.apache.oozie.util.JobsFilterUtils;
import org.apache.oozie.util.JobUtils;
+import org.apache.oozie.util.XLogAuditFilter;
import org.apache.oozie.util.XLogFilter;
import org.apache.oozie.util.XLogUserFilterParam;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.util.XLog;
import com.google.common.annotations.VisibleForTesting;
-import sun.reflect.generics.tree.ReturnType;
import javax.servlet.ServletException;
@@ -251,23 +246,43 @@ public class BundleEngine extends BaseEngine {
}
}
-
+ @Override
public void streamLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
BundleEngineException {
- streamJobLog(jobId, writer, params, false);
+ streamJobLog(jobId, writer, params, LOG_TYPE.LOG);
}
+ @Override
public void streamErrorLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
BundleEngineException {
- streamJobLog(jobId, writer, params, true);
+ streamJobLog(jobId, writer, params, LOG_TYPE.ERROR_LOG);
}
- private void streamJobLog(String jobId, Writer writer, Map<String, String[]> params, boolean isErrorLog) throws IOException,
+ @Override
+ public void streamAuditLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
BundleEngineException {
+ try {
+ streamJobLog(new XLogAuditFilter(new XLogUserFilterParam(params)), jobId, writer, params, LOG_TYPE.AUDIT_LOG);
+ }
+ catch (CommandException e) {
+ throw new IOException(e);
+ }
+ }
- BundleJobBean job;
+ private void streamJobLog(String jobId, Writer writer, Map<String, String[]> params, LOG_TYPE logType)
+ throws IOException, BundleEngineException {
try {
- XLogFilter filter = new XLogFilter(new XLogUserFilterParam(params));
+ streamJobLog(new XLogFilter(new XLogUserFilterParam(params)), jobId, writer, params, logType);
+ }
+ catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void streamJobLog(XLogFilter filter, String jobId, Writer writer, Map<String, String[]> params, LOG_TYPE logType)
+ throws IOException, BundleEngineException {
+ try {
+ BundleJobBean job;
filter.setParameter(DagXLogInfoService.JOB, jobId);
job = new BundleJobXCommand(jobId).call();
Date lastTime = null;
@@ -277,15 +292,7 @@ public class BundleEngine extends BaseEngine {
if (lastTime == null) {
lastTime = new Date();
}
- if (isErrorLog) {
- Services.get().get(XLogStreamingService.class)
- .streamErrorLog(filter, job.getCreatedTime(), lastTime, writer, params);
- }
- else {
- Services.get().get(XLogStreamingService.class)
- .streamLog(filter, job.getCreatedTime(), lastTime, writer, params);
-
- }
+ fetchLog(filter, job.getStartTime(), lastTime, writer, params, logType);
}
catch (Exception ex) {
throw new IOException(ex);
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/CoordinatorEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java
index 1b21d87..cb67be0 100644
--- a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java
+++ b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.BaseEngine.LOG_TYPE;
import org.apache.oozie.client.CoordinatorAction;
import org.apache.oozie.client.CoordinatorJob;
import org.apache.oozie.client.OozieClient;
@@ -59,6 +60,7 @@ import org.apache.oozie.util.JobUtils;
import org.apache.oozie.util.Pair;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.util.XLog;
+import org.apache.oozie.util.XLogAuditFilter;
import org.apache.oozie.util.XLogFilter;
import org.apache.oozie.util.XLogUserFilterParam;
@@ -303,20 +305,39 @@ public class CoordinatorEngine extends BaseEngine {
@Override
public void streamLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
BaseEngineException {
- streamJobLog(jobId, writer, params, false);
+ streamJobLog(jobId, writer, params, LOG_TYPE.LOG);
}
@Override
public void streamErrorLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
BaseEngineException {
- streamJobLog(jobId, writer, params, true);
+ streamJobLog(jobId, writer, params, LOG_TYPE.ERROR_LOG);
}
- private void streamJobLog(String jobId, Writer writer, Map<String, String[]> params, boolean isErrorLog)
+ @Override
+ public void streamAuditLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
+ BaseEngineException {
+ try {
+ streamJobLog(new XLogAuditFilter(new XLogUserFilterParam(params)), jobId, writer, params, LOG_TYPE.AUDIT_LOG);
+ }
+ catch (CommandException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void streamJobLog(String jobId, Writer writer, Map<String, String[]> params, LOG_TYPE logType)
throws IOException, BaseEngineException {
+ try {
+ streamJobLog(new XLogFilter(new XLogUserFilterParam(params)), jobId, writer, params, logType);
+ }
+ catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ private void streamJobLog(XLogFilter filter, String jobId, Writer writer, Map<String, String[]> params, LOG_TYPE logType)
+ throws IOException, BaseEngineException {
try {
- XLogFilter filter = new XLogFilter(new XLogUserFilterParam(params));
filter.setParameter(DagXLogInfoService.JOB, jobId);
Date lastTime = null;
CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId);
@@ -326,14 +347,7 @@ public class CoordinatorEngine extends BaseEngine {
if (lastTime == null) {
lastTime = new Date();
}
- if (isErrorLog) {
- Services.get().get(XLogStreamingService.class)
- .streamErrorLog(filter, job.getCreatedTime(), lastTime, writer, params);
- }
- else {
- Services.get().get(XLogStreamingService.class)
- .streamLog(filter, job.getCreatedTime(), lastTime, writer, params);
- }
+ fetchLog(filter, job.getStartTime(), lastTime, writer, params, logType);
}
catch (Exception e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/DagEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/DagEngine.java b/core/src/main/java/org/apache/oozie/DagEngine.java
index faf2d64..08a126c 100644
--- a/core/src/main/java/org/apache/oozie/DagEngine.java
+++ b/core/src/main/java/org/apache/oozie/DagEngine.java
@@ -49,6 +49,7 @@ import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.CallableQueueService;
+import org.apache.oozie.util.XLogAuditFilter;
import org.apache.oozie.util.XLogFilter;
import org.apache.oozie.util.XLogUserFilterParam;
import org.apache.oozie.util.ParamChecker;
@@ -400,7 +401,7 @@ public class DagEngine extends BaseEngine {
@Override
public void streamLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
DagEngineException {
- streamJobLog(jobId, writer, params, false);
+ streamJobLog(jobId, writer, params, LOG_TYPE.LOG);
}
/**
@@ -412,30 +413,52 @@ public class DagEngine extends BaseEngine {
* @throws IOException thrown if the log cannot be streamed.
* @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId.
*/
+ @Override
public void streamErrorLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
DagEngineException {
- streamJobLog(jobId, writer, params, true);
+ streamJobLog(jobId, writer, params, LOG_TYPE.ERROR_LOG);
}
- public void streamJobLog(String jobId, Writer writer, Map<String, String[]> params, boolean isErrorLog)
+ /**
+ * Stream the audit log of a job.
+ *
+ * @param jobId job Id.
+ * @param writer writer to stream the log to.
+ * @param params additional parameters from the request
+ * @throws IOException thrown if the log cannot be streamed.
+ * @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId.
+ */
+ @Override
+ public void streamAuditLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
+ DagEngineException {
+ try {
+ streamJobLog(new XLogAuditFilter(new XLogUserFilterParam(params)),jobId, writer, params, LOG_TYPE.AUDIT_LOG);
+ }
+ catch (CommandException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void streamJobLog(String jobId, Writer writer, Map<String, String[]> params, LOG_TYPE logType)
+ throws IOException, DagEngineException {
+ try {
+ streamJobLog(new XLogFilter(new XLogUserFilterParam(params)), jobId, writer, params, logType);
+ }
+ catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void streamJobLog(XLogFilter filter, String jobId, Writer writer, Map<String, String[]> params, LOG_TYPE logType)
throws IOException, DagEngineException {
try {
- XLogFilter filter = new XLogFilter(new XLogUserFilterParam(params));
filter.setParameter(DagXLogInfoService.JOB, jobId);
WorkflowJob job = getJob(jobId);
Date lastTime = job.getEndTime();
if (lastTime == null) {
lastTime = job.getLastModifiedTime();
}
- if (isErrorLog) {
- Services.get().get(XLogStreamingService.class)
- .streamErrorLog(filter, job.getCreatedTime(), lastTime, writer, params);
- }
- else {
- Services.get().get(XLogStreamingService.class)
- .streamLog(filter, job.getCreatedTime(), lastTime, writer, params);
- }
-
+ fetchLog(filter, job.getStartTime(), lastTime, writer, params, logType);
}
catch (Exception e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/service/DagXLogInfoService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/DagXLogInfoService.java b/core/src/main/java/org/apache/oozie/service/DagXLogInfoService.java
index 7970546..33e41ed 100644
--- a/core/src/main/java/org/apache/oozie/service/DagXLogInfoService.java
+++ b/core/src/main/java/org/apache/oozie/service/DagXLogInfoService.java
@@ -43,6 +43,8 @@ public class DagXLogInfoService implements Service {
*/
public static final String JOB = "JOB";
+ public static final String AUDIT_JOBID = "JOBID";
+
/**
* Action log info constant.
*/
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/service/XLogService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/XLogService.java b/core/src/main/java/org/apache/oozie/service/XLogService.java
index c291fa8..3feab4c 100644
--- a/core/src/main/java/org/apache/oozie/service/XLogService.java
+++ b/core/src/main/java/org/apache/oozie/service/XLogService.java
@@ -92,7 +92,9 @@ public class XLogService implements Service, Instrumentable {
private boolean fromClasspath;
private String log4jFileName;
private boolean logOverWS = true;
- private boolean errorLogEnable = true;
+ private boolean errorLogEnabled = true;
+ private boolean auditLogEnabled = true;
+
private static final String STARTUP_MESSAGE = "{E}"
+ " ******************************************************************************* {E}"
@@ -104,8 +106,12 @@ public class XLogService implements Service, Instrumentable {
private String oozieLogName;
private String oozieErrorLogPath;
private String oozieErrorLogName;
+ private String oozieAuditLogPath;
+ private String oozieAuditLogName;
private int oozieLogRotation = -1;
private int oozieErrorLogRotation = -1;
+ private int oozieAuditLogRotation = -1;
+
public XLogService() {
@@ -217,10 +223,17 @@ public class XLogService implements Service, Instrumentable {
oozieLogName = logUtil.getLogFileName() == null ? oozieLogName : logUtil.getLogFileName();
logUtil = new XLogUtil(conf, "oozieError");
- errorLogEnable = logUtil.isLogOverEnable();
+ errorLogEnabled = logUtil.isLogOverEnable();
oozieErrorLogRotation = logUtil.getLogRotation() == 0 ? oozieErrorLogRotation : logUtil.getLogRotation();
oozieErrorLogPath = logUtil.getLogPath() == null ? oozieErrorLogPath : logUtil.getLogPath();
oozieErrorLogName = logUtil.getLogFileName() == null ? oozieErrorLogName : logUtil.getLogFileName();
+
+ logUtil = new XLogUtil(conf, "oozieaudit");
+ auditLogEnabled = logUtil.isLogOverEnable();
+ oozieAuditLogRotation = logUtil.getLogRotation() == 0 ? oozieAuditLogRotation : logUtil.getLogRotation();
+ oozieAuditLogPath = logUtil.getLogPath() == null ? oozieAuditLogPath : logUtil.getLogPath();
+ oozieAuditLogName = logUtil.getLogFileName() == null ? oozieAuditLogName : logUtil.getLogFileName();
+
}
/**
@@ -286,17 +299,33 @@ public class XLogService implements Service, Instrumentable {
return logOverWS;
}
- boolean isErrorLogEnable(){
- return errorLogEnable;
+ boolean isErrorLogEnabled(){
+ return errorLogEnabled;
}
int getOozieLogRotation() {
return oozieLogRotation;
}
+
int getOozieErrorLogRotation() {
return oozieErrorLogRotation;
}
+ int getOozieAuditLogRotation() {
+ return oozieAuditLogRotation;
+ }
+
+ public String getOozieAuditLogPath() {
+ return oozieAuditLogPath;
+ }
+
+ public String getOozieAuditLogName() {
+ return oozieAuditLogName;
+ }
+
+ boolean isAuditLogEnabled() {
+ return auditLogEnabled;
+ }
String getLog4jProperties() {
return log4jFileName;
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/service/XLogStreamingService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/XLogStreamingService.java b/core/src/main/java/org/apache/oozie/service/XLogStreamingService.java
index 9066c7a..c15c4c1 100644
--- a/core/src/main/java/org/apache/oozie/service/XLogStreamingService.java
+++ b/core/src/main/java/org/apache/oozie/service/XLogStreamingService.java
@@ -106,7 +106,7 @@ public class XLogStreamingService implements Service, Instrumentable {
public void streamErrorLog(XLogFilter filter, Date startTime, Date endTime, Writer writer, Map<String, String[]> params)
throws IOException {
XLogService xLogService = Services.get().get(XLogService.class);
- if (xLogService.isErrorLogEnable()) {
+ if (xLogService.isErrorLogEnabled()) {
new XLogStreamer(filter, xLogService.getOozieErrorLogPath(), xLogService.getOozieErrorLogName(),
xLogService.getOozieErrorLogRotation()).streamLog(writer, startTime, endTime, bufferLen);
}
@@ -115,6 +115,29 @@ public class XLogStreamingService implements Service, Instrumentable {
}
}
+ /**
+ * Stream the audit log of a job.
+ *
+ * @param filter log streamer filter.
+ * @param startTime start time for log events to filter.
+ * @param endTime end time for log events to filter.
+ * @param writer writer to stream the log to.
+ * @param params additional parameters from the request
+ * @throws IOException thrown if the log cannot be streamed.
+ */
+ public void streamAuditLog(XLogFilter filter, Date startTime, Date endTime, Writer writer, Map<String, String[]> params)
+ throws IOException {
+ XLogService xLogService = Services.get().get(XLogService.class);
+ if (xLogService.isAuditLogEnabled()) {
+ new XLogStreamer(filter, xLogService.getOozieAuditLogPath(), xLogService.getOozieAuditLogName(),
+ xLogService.getOozieAuditLogRotation()).streamLog(writer, startTime, endTime, bufferLen);
+ }
+ else {
+ writer.write("Audit Log is disabled!!");
+ }
+ }
+
+
public int getBufferLen() {
return bufferLen;
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/service/ZKXLogStreamingService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ZKXLogStreamingService.java b/core/src/main/java/org/apache/oozie/service/ZKXLogStreamingService.java
index b28a591..834dbd8 100644
--- a/core/src/main/java/org/apache/oozie/service/ZKXLogStreamingService.java
+++ b/core/src/main/java/org/apache/oozie/service/ZKXLogStreamingService.java
@@ -135,7 +135,7 @@ public class ZKXLogStreamingService extends XLogStreamingService implements Serv
public void streamErrorLog(XLogFilter filter, Date startTime, Date endTime, Writer writer, Map<String, String[]> params)
throws IOException {
XLogService xLogService = Services.get().get(XLogService.class);
- if (xLogService.isErrorLogEnable()) {
+ if (xLogService.isErrorLogEnabled()) {
// If ALL_SERVERS_PARAM is set to false, then only stream our log
if (!Services.get().get(JobsConcurrencyService.class).isAllServerRequest(params)) {
new XLogStreamer(filter, xLogService.getOozieErrorLogPath(), xLogService.getOozieErrorLogName(),
@@ -153,6 +153,39 @@ public class ZKXLogStreamingService extends XLogStreamingService implements Serv
}
}
+ /**
+ * Stream the audit log of a job. It contacts any other running Oozie servers to collate relevant audit logs while streaming.
+ *
+ * @param filter log streamer filter.
+ * @param startTime start time for log events to filter.
+ * @param endTime end time for log events to filter.
+ * @param writer writer to stream the log to.
+ * @param params additional parameters from the request
+ * @throws IOException thrown if the log cannot be streamed.
+ */
+ @Override
+ public void streamAuditLog(XLogFilter filter, Date startTime, Date endTime, Writer writer, Map<String, String[]> params)
+ throws IOException {
+ XLogService xLogService = Services.get().get(XLogService.class);
+ if (xLogService.isAuditLogEnabled()) {
+ // If ALL_SERVERS_PARAM is set to false, then only stream our log
+ if (!Services.get().get(JobsConcurrencyService.class).isAllServerRequest(params)) {
+ new XLogStreamer(filter, xLogService.getOozieAuditLogPath(), xLogService.getOozieAuditLogName(),
+ xLogService.getOozieAuditLogRotation()).streamLog(writer, startTime, endTime, bufferLen);
+ }
+ // Otherwise, we have to go collate relevant logs from the other Oozie servers
+ else {
+ collateLogs(filter, startTime, endTime, writer, params, xLogService.getOozieAuditLogPath(),
+ xLogService.getOozieAuditLogName(), xLogService.getOozieAuditLogRotation(),
+ RestConstants.JOB_SHOW_AUDIT_LOG);
+ }
+ }
+ else {
+ writer.write("Audit Log streaming disabled!!");
+ }
+ }
+
+
/**
* Contacts each of the other Oozie servers, gets their logs for the job, collates them, and sends them to the user via the
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java
index bd9e998..9be4866 100644
--- a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java
@@ -329,6 +329,11 @@ public abstract class BaseJobServlet extends JsonRestServlet {
response.setContentType(TEXT_UTF8);
streamJobErrorLog(request, response);
}
+ else if (show.equals(RestConstants.JOB_SHOW_AUDIT_LOG)) {
+ response.setContentType(TEXT_UTF8);
+ streamJobAuditLog(request, response);
+ }
+
else if (show.equals(RestConstants.JOB_SHOW_DEFINITION)) {
stopCron();
response.setContentType(XML_UTF8);
@@ -471,6 +476,9 @@ public abstract class BaseJobServlet extends JsonRestServlet {
IOException;
+ abstract void streamJobAuditLog(HttpServletRequest request, HttpServletResponse response) throws XServletException,
+ IOException;
+
/**
* abstract method to create and stream image for runtime DAG -- workflow only
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/servlet/JsonRestServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/JsonRestServlet.java b/core/src/main/java/org/apache/oozie/servlet/JsonRestServlet.java
index 4e9c224..a409c22 100644
--- a/core/src/main/java/org/apache/oozie/servlet/JsonRestServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/JsonRestServlet.java
@@ -252,10 +252,10 @@ public abstract class JsonRestServlet extends HttpServlet {
String errorMessage = (String) request.getAttribute(AUDIT_ERROR_MESSAGE);
String hostDetail = request.getRemoteAddr();
- auditLog.info(
- "IP [{0}], USER [{1}], GROUP [{2}], APP [{3}], JOBID [{4}], OPERATION [{5}], PARAMETER [{6}], STATUS [{7}],"
- + " HTTPCODE [{8}], ERRORCODE [{9}], ERRORMESSAGE [{10}]", hostDetail, user, group, app,
- jobId, operation, param, status, httpStatusCode, errorCode, errorMessage);
+ auditLog.info("IP [{0}], USER [{1}], GROUP [{2}], APP [{3}], " + DagXLogInfoService.AUDIT_JOBID
+ + " [{4}], OPERATION [{5}], PARAMETER [{6}], STATUS [{7}],"
+ + " HTTPCODE [{8}], ERRORCODE [{9}], ERRORMESSAGE [{10}]", hostDetail, user, group, app, jobId,
+ operation, param, status, httpStatusCode, errorCode, errorMessage);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java
index 3cb9168..86ff278 100644
--- a/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java
@@ -200,6 +200,11 @@ public class V0JobServlet extends BaseJobServlet {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0306);
}
+ @Override
+ protected void streamJobAuditLog(HttpServletRequest request, HttpServletResponse response) throws XServletException,
+ IOException {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0306);
+ }
/*
* Not implemented in v0
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java
index d4564c6..60f1029 100644
--- a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java
@@ -1103,7 +1103,11 @@ public class V1JobServlet extends BaseJobServlet {
IOException {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v1");
}
-
+ @Override
+ protected void streamJobAuditLog(HttpServletRequest request, HttpServletResponse response) throws XServletException,
+ IOException {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v1");
+ }
@Override
void slaEnableAlert(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v1");
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java
index 7100c98..662a7ff 100644
--- a/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java
@@ -278,9 +278,27 @@ public class V2JobServlet extends V1JobServlet {
catch (BaseEngineException e) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, e);
}
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void streamJobAuditLog(HttpServletRequest request, HttpServletResponse response) throws XServletException,
+ IOException {
+
+ String jobId = getResourceName(request);
+ try {
+ getBaseEngine(jobId, getUser(request)).streamAuditLog(jobId, response.getWriter(), request.getParameterMap());
+ }
+ catch (DagEngineException ex) {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
+ }
+ catch (BaseEngineException e) {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, e);
+ }
}
+
/**
* Gets the base engine based on jobId.
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/util/TimestampedMessageParser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/TimestampedMessageParser.java b/core/src/main/java/org/apache/oozie/util/TimestampedMessageParser.java
index d2aa02a..f691409 100644
--- a/core/src/main/java/org/apache/oozie/util/TimestampedMessageParser.java
+++ b/core/src/main/java/org/apache/oozie/util/TimestampedMessageParser.java
@@ -21,11 +21,7 @@ package org.apache.oozie.util;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Writer;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.XLogStreamingService;
@@ -65,6 +61,7 @@ public class TimestampedMessageParser {
filter.constructPattern();
}
+
/**
* Causes the next message and timestamp to be parsed from the BufferedReader.
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/main/java/org/apache/oozie/util/XLogFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/XLogFilter.java b/core/src/main/java/org/apache/oozie/util/XLogFilter.java
index 5c0d1f3..3b49f77 100644
--- a/core/src/main/java/org/apache/oozie/util/XLogFilter.java
+++ b/core/src/main/java/org/apache/oozie/util/XLogFilter.java
@@ -30,8 +30,6 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.oozie.service.ConfigurationService;
-import org.apache.oozie.service.Services;
-
import com.google.common.annotations.VisibleForTesting;
/**
@@ -192,7 +190,6 @@ public class XLogFilter {
parameters.clear();
}
- @VisibleForTesting
public final Map<String, String> getFilterParams() {
return filterParams;
}
@@ -345,4 +342,8 @@ public class XLogFilter {
return org.apache.commons.lang.time.DateUtils.addMinutes(date, offset);
}
+ public void setFilterPattern(Pattern filterPattern) {
+ this.filterPattern = filterPattern;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/test/java/org/apache/oozie/service/TestXLogStreamingService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestXLogStreamingService.java b/core/src/test/java/org/apache/oozie/service/TestXLogStreamingService.java
index aedd35c..bebb678 100644
--- a/core/src/test/java/org/apache/oozie/service/TestXLogStreamingService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestXLogStreamingService.java
@@ -19,7 +19,10 @@
package org.apache.oozie.service;
import org.apache.commons.logging.LogFactory;
+import org.apache.oozie.command.CommandException;
import org.apache.oozie.test.XTestCase;
+import org.apache.oozie.util.XLog;
+import org.apache.oozie.util.XLogAuditFilter;
import org.apache.oozie.util.XLogFilter;
import org.apache.oozie.util.XLogUserFilterParam;
@@ -42,6 +45,17 @@ public class TestXLogStreamingService extends XTestCase {
super.tearDown();
}
+ private void setupXLog() throws CommandException {
+ XLogFilter.reset();
+ XLogFilter.defineParameter("USER");
+ XLogFilter.defineParameter("GROUP");
+ XLogFilter.defineParameter("TOKEN");
+ XLogFilter.defineParameter("APP");
+ XLogFilter.defineParameter("JOB");
+ XLogFilter.defineParameter("ACTION");
+ }
+
+
public void testDisableLogOverWS() throws Exception {
Properties props = new Properties();
// Test missing logfile
@@ -218,15 +232,8 @@ public class TestXLogStreamingService extends XTestCase {
}
public void testNoDashInConversionPattern() throws Exception{
- XLogFilter.reset();
- XLogFilter.defineParameter("USER");
- XLogFilter.defineParameter("GROUP");
- XLogFilter.defineParameter("TOKEN");
- XLogFilter.defineParameter("APP");
- XLogFilter.defineParameter("JOB");
- XLogFilter.defineParameter("ACTION");
+ setupXLog();
XLogFilter xf = new XLogFilter(new XLogUserFilterParam(null));
-
xf.setParameter("USER", "oozie");
xf.setLogLevel("DEBUG|INFO");
// Previously, a dash ("-") was always required somewhere in a line in order for that line to pass the filter; this test
@@ -266,15 +273,8 @@ public class TestXLogStreamingService extends XTestCase {
}
public void testErrorLog() throws Exception{
- XLogFilter.reset();
- XLogFilter.defineParameter("USER");
- XLogFilter.defineParameter("GROUP");
- XLogFilter.defineParameter("TOKEN");
- XLogFilter.defineParameter("APP");
- XLogFilter.defineParameter("JOB");
- XLogFilter.defineParameter("ACTION");
+ setupXLog();
XLogFilter xf = new XLogFilter(new XLogUserFilterParam(null));
-
xf.setParameter("USER", "oozie");
xf.setLogLevel("DEBUG|INFO");
// Previously, a dash ("-") was always required somewhere in a line in order for that line to pass the filter; this test
@@ -337,6 +337,54 @@ public class TestXLogStreamingService extends XTestCase {
}
}
+ public void testAuditLog() throws Exception {
+ setupXLog();
+ XLogFilter xf = new XLogAuditFilter(new XLogUserFilterParam(null));
+ xf.setParameter("USER", "oozie");
+ xf.setLogLevel("DEBUG|INFO");
+ File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties");
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties");
+ Properties log4jProps = new Properties();
+ log4jProps.load(is);
+ // prevent conflicts with other tests by changing the log file location
+ log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log");
+ log4jProps.setProperty("log4j.appender.oozieaudit.File", getTestCaseDir() + "/oozie-audit.log");
+
+ log4jProps.store(new FileOutputStream(log4jFile), "");
+ setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName());
+ try {
+ new Services().init();
+ XLog auditLog = XLog.getLog("oozieaudit");
+ xf.setParameter(DagXLogInfoService.JOB, "0000000-150322000230582-oozie-puru-C");
+
+
+ auditLog.info("2015-03-22 00:04:35,494 INFO oozieaudit:520 - IP [127.0.0.1], USER [purushah], GROUP [null], "
+ + "APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [start], "
+ + "PARAMETER [null], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [L1]");
+ auditLog.info("2015-03-22 00:05:13,823 INFO oozieaudit:520 - IP [127.0.0.1], USER [purushah], GROUP [null], "
+ + "APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [suspend], "
+ + "PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], "
+ + "ERRORMESSAGE [L2]");
+ auditLog.info("2015-03-22 00:05:13,823 INFO oozieaudit:520 - IP [127.0.0.1], USER [purushah], GROUP [null], "
+ + "APP [-], JOBID [0000001-150322000230582-oozie-puru-C], OPERATION [suspend], "
+ + "PARAMETER [0000001-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], "
+ + "ERRORMESSAGE [L3]");
+
+
+ String out = doStreamAuditLog(xf);
+ String outArr[] = out.split("\n");
+ // Lines 2 and 4 are filtered out because they have the wrong user
+ assertEquals(2, outArr.length);
+ assertTrue(outArr[0].contains("L1"));
+ assertTrue(out.contains("L2"));
+ assertFalse(out.contains("L3"));
+
+ }
+ finally {
+ Services.get().destroy();
+ }
+ }
private boolean doStreamDisabledCheckWithServices() throws Exception {
boolean result = false;
@@ -366,6 +414,12 @@ public class TestXLogStreamingService extends XTestCase {
Services.get().get(XLogStreamingService.class).streamErrorLog(xf, null, null, w, new HashMap<String, String[]>());
return w.toString();
}
+ private String doStreamAuditLog(XLogFilter xf) throws Exception {
+ StringWriter w = new StringWriter();
+ Services.get().get(XLogStreamingService.class).streamAuditLog(xf, null, null, w, new HashMap<String, String[]>());
+ return w.toString();
+ }
+
private boolean doErrorStreamDisabledCheck() throws Exception {
XLogFilter xf = new XLogFilter(new XLogUserFilterParam(null));
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/core/src/test/resources/test-no-dash-log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/test-no-dash-log4j.properties b/core/src/test/resources/test-no-dash-log4j.properties
index 2ff353e..4242f8a 100644
--- a/core/src/test/resources/test-no-dash-log4j.properties
+++ b/core/src/test/resources/test-no-dash-log4j.properties
@@ -27,6 +27,18 @@ log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L %m%n
log4j.appender.oozie.RollingPolicy.FileNamePattern=${log4j.appender.oozie.File}-%d{yyyy-MM-dd-HH}
log4j.appender.oozie.RollingPolicy.MaxHistory=720
+log4j.appender.oozieaudit=org.apache.log4j.rolling.RollingFileAppender
+log4j.appender.oozieaudit.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy
+log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log
+log4j.appender.oozieaudit.Append=true
+log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout
+# The default conversion pattern is %d{ISO8601} %5p %c{1}:%L - %m%n
+# We're testing if log streaming works without the dash
+log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L %m%n
+log4j.appender.oozieaudit.RollingPolicy.FileNamePattern=${log4j.appender.oozieaudit.File}-%d{yyyy-MM-dd-HH}
+log4j.appender.oozieaudit.RollingPolicy.MaxHistory=720
+
+
log4j.appender.oozieError=org.apache.log4j.rolling.RollingFileAppender
log4j.appender.oozieError.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy
log4j.appender.oozieError.File=${oozie.log.dir}/oozie-error.log
@@ -48,3 +60,4 @@ log4j.appender.oozieError.filter.4 = org.apache.log4j.varia.DenyAllFilter
log4j.logger.a=ALL, oozie, oozieError
+log4j.logger.oozieaudit=ALL, oozieaudit
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/docs/src/site/twiki/DG_CommandLineTool.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki
index 372bfe6..9494b22 100644
--- a/docs/src/site/twiki/DG_CommandLineTool.twiki
+++ b/docs/src/site/twiki/DG_CommandLineTool.twiki
@@ -63,6 +63,8 @@ usage:
-localtime use local time (same as passing your time zone to -timezone).
Overrides -timezone option
-log <arg> job log
+ -errorlog <arg> job error log
+ -auditlog <arg> job audit log
-logfilter <arg> job log search parameter. Can be specified as -logfilter
opt1=val1;opt2=val1;opt3=val1. Supported options are recent,
start, end, loglevel, text, limit and debug
@@ -648,6 +650,20 @@ org.xml.sax.SAXParseException; lineNumber: 20; columnNumber: 22; cvc-complex-typ
at org.apache.xerces.jaxp.validation.ValidatorImpl.validate(Unknown Source)
</verbatim>
+
+---+++ Checking the audit logs of a Workflow, Coordinator or Bundle Job
+
+Example:
+
+<verbatim>
+$ oozie job -oozie http://localhost:11000/oozie -auditlog 0000000-150322000230582-oozie-puru-C
+2015-03-22 00:04:35,494 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [start], PARAMETER [null], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 00:05:13,823 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [suspend], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 00:06:59,561 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [suspend], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 23:22:20,012 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [suspend], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 23:28:48,218 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [resume], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+</verbatim>
+
---+++ Checking the server logs for particular actions of a Coordinator Job
Example:
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/docs/src/site/twiki/WebServicesAPI.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki
index f169ba7..3d97447 100644
--- a/docs/src/site/twiki/WebServicesAPI.twiki
+++ b/docs/src/site/twiki/WebServicesAPI.twiki
@@ -1454,6 +1454,29 @@ org.xml.sax.SAXParseException; lineNumber: 20; columnNumber: 22; cvc-complex-typ
</verbatim>
+---++++ Job Audit Log
+
+An HTTP GET request retrieves the job audit log.
+
+*Request:*
+
+<verbatim>
+GET /oozie/v2/job/0000000-150322000230582-oozie-puru-C?show=auditlog
+</verbatim>
+
+*Response:*
+
+<verbatim>
+HTTP/1.1 200 OK
+Content-Type: text/plain;charset=UTF-8
+2015-03-22 00:04:35,494 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [start], PARAMETER [null], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 00:05:13,823 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [suspend], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 00:06:59,561 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [suspend], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 23:22:20,012 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [suspend], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]
+2015-03-22 23:28:48,218 INFO oozieaudit:520 - IP [-], USER [purushah], GROUP [null], APP [-], JOBID [0000000-150322000230582-oozie-puru-C], OPERATION [resume], PARAMETER [0000000-150322000230582-oozie-puru-C], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [null], ERRORMESSAGE [null]</verbatim>
+</verbatim>
+
+
---++++ Filtering the server logs with logfilter options
User can provide multiple option to filter logs using -logfilter opt1=val1;opt2=val1;opt3=val1. This can be used to fetch only just logs of interest faster as fetching Oozie server logs is slow due to the overhead of pattern matching.
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index c7e0900..642cf31 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-2140 Audit Log should be shown in Oozie UI (puru)
OOZIE-2139 Coord update doesn't work for job which is submitted by bundle (puru)
OOZIE-1726 Oozie does not support _HOST when configuring kerberos security (venkatnrangan via bzhang)
OOZIE-2197 ooziedb.cmd command failed due to classpath being too long on windows (me.venkatr via bzhang)
http://git-wip-us.apache.org/repos/asf/oozie/blob/3fe9c2a8/webapp/src/main/webapp/oozie-console.js
----------------------------------------------------------------------
diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js
index e07680f..16b2528 100644
--- a/webapp/src/main/webapp/oozie-console.js
+++ b/webapp/src/main/webapp/oozie-console.js
@@ -414,6 +414,16 @@ function jobDetailsPopup(response, request) {
autoScroll: true,
emptyText: ""
});
+ var jobAuditLogArea = new Ext.form.TextArea({
+ fieldLabel: 'AuditLogs',
+ editable: false,
+ name: 'auditlogs',
+ width: 1035,
+ height: 400,
+ autoScroll: true,
+ emptyText: ""
+ });
+
function fetchDefinition(workflowId) {
Ext.Ajax.request({
@@ -429,6 +439,11 @@ function jobDetailsPopup(response, request) {
function fetchErrorLogs(workflowId, errorLogStatus) {
getLogs(getOozieBase() + 'job/' + workflowId + "?show=errorlog", null, errorLogStatus, jobErrorLogArea, false, null);
}
+
+ function fetchAuditLogs(workflowId, auditLogStatus) {
+ getLogs(getOozieBase() + 'job/' + workflowId + "?show=auditlog", null, auditLogStatus, jobAuditLogArea, false, null);
+ }
+
function fetchLogs(workflowId, logStatus) {
getLogs(getOozieBase() + 'job/' + workflowId + "?show=log", searchFilterBox.getValue(), logStatus, jobLogArea, false, null);
@@ -836,6 +851,11 @@ function jobDetailsPopup(response, request) {
text : 'Log Status : '
});
+ var auditLogStatus = new Ext.form.Label({
+ text : 'Log Status : '
+ });
+
+
var getLogButton = new Ext.Button({
text: 'Get Logs',
ctCls: 'x-btn-over',
@@ -846,6 +866,7 @@ function jobDetailsPopup(response, request) {
var isLoadedDAG = false;
var isErrorLogLoaded = false;
+ var isAuditLogLoaded = false;
var jobDetailsTab = new Ext.TabPanel({
activeTab: 0,
@@ -886,6 +907,17 @@ function jobDetailsPopup(response, request) {
}, {xtype: 'tbfill'}, errorLogStatus]
},
{
+ title: 'Job Audit Log',
+ items: jobAuditLogArea,
+ tbar: [ {
+ text: " ",
+ icon: 'ext-2.2/resources/images/default/grid/refresh.gif',
+ handler: function() {
+ fetchAuditLogs(workflowId, auditLogStatus);
+ }
+ }, {xtype: 'tbfill'}, auditLogStatus]
+ },
+ {
title: 'Job DAG',
items: imageContainer,
tbar: [{
@@ -911,7 +943,14 @@ function jobDetailsPopup(response, request) {
fetchErrorLogs(workflowId, errorLogStatus);
isErrorLogLoaded=true;
}
- } else if(selectedTab.title == 'Job DAG') {
+
+ }else if (selectedTab.title == 'Job Audit Log') {
+ if(!isAuditLogLoaded){
+ fetchAuditLogs(workflowId, auditLogStatus);
+ isAuditLogLoaded=true;
+ }
+ }
+ else if(selectedTab.title == 'Job DAG') {
if(!isLoadedDAG){
var dagImage= new Ext.ux.Image({
id: 'dagImage',
@@ -976,6 +1015,17 @@ function coordJobDetailsPopup(response, request) {
autoScroll: true,
emptyText: ""
});
+ var jobAuditLogArea = new Ext.form.TextArea({
+ fieldLabel: 'AuditLogs',
+ editable: false,
+ id: 'jobAuditLogAreaId',
+ name: 'auditlogs',
+ width: 1035,
+ height: 400,
+ autoScroll: true,
+ emptyText: ""
+ });
+
var getLogButton = new Ext.Button({
text: 'Get Logs',
ctCls: 'x-btn-over',
@@ -1008,9 +1058,15 @@ function coordJobDetailsPopup(response, request) {
var logStatus = new Ext.form.Label({
text : 'Log Status : '
});
+
var errorLogStatus = new Ext.form.Label({
text : 'Log Status : '
});
+
+ var auditLogStatus = new Ext.form.Label({
+ text : 'Log Status : '
+ });
+
function fetchDefinition(coordJobId) {
Ext.Ajax.request({
url: getOozieBase() + 'job/' + coordJobId + "?show=definition",
@@ -1036,6 +1092,10 @@ function coordJobDetailsPopup(response, request) {
null, errorLogStatus, jobErrorLogArea, true, null);
}
+ function fetchAuditLogs(coordJobId) {
+ getLogs(getOozieBase() + 'job/' + coordJobId + "?show=auditlog", null,
+ auditLogStatus, jobAuditLogArea, true, null);
+ }
var jobDetails = eval("(" + response.responseText + ")");
var coordJobId = jobDetails["coordJobId"];
@@ -1454,6 +1514,17 @@ function coordJobDetailsPopup(response, request) {
},{xtype: 'tbfill'}, errorLogStatus]
},
{
+ title: 'Coord Audit Log',
+ items: jobAuditLogArea,
+ tbar: [ {
+ text: " ",
+ icon: 'ext-2.2/resources/images/default/grid/refresh.gif',
+ handler: function() {
+ fetchAuditLogs(coordJobId);
+ }
+ },{xtype: 'tbfill'}, auditLogStatus]
+ },
+ {
title: 'Coord Action Reruns',
items: rerunsUnit,
tbar: [
@@ -1697,6 +1768,10 @@ function bundleJobDetailsPopup(response, request) {
text : 'Log Status : '
});
+ var auditLogStatus = new Ext.form.Label({
+ text : 'Log Status : '
+ });
+
var getLogButton = new Ext.Button({
text: 'Get Logs',
ctCls: 'x-btn-over',
@@ -1743,7 +1818,20 @@ function bundleJobDetailsPopup(response, request) {
false, null);
}
}, {xtype: 'tbfill'}, errorLogStatus]
- }
+ },
+ {
+ title: 'Bundle Audit Log',
+ items: jobAuditLogArea,
+ tbar: [ {
+ text: " ",
+ icon: 'ext-2.2/resources/images/default/grid/refresh.gif',
+ handler: function() {
+ getLogs(getOozieBase() + 'job/' + bundleJobId + "?show=auditlog", null, auditLogStatus, jobAuditLogArea,
+ false, null);
+ }
+ }, {xtype: 'tbfill'}, auditLogStatus]
+ }
+
]});
jobDetailsTab.addListener("tabchange", function(panel, selectedTab) {
@@ -1761,6 +1849,14 @@ function bundleJobDetailsPopup(response, request) {
isErrorLogLoaded=true;
}
}
+ else if (selectedTab.title == 'Bundle Audit Log') {
+ if(!isAuditLogLoaded){
+ getLogs(getOozieBase() + 'job/' + bundleJobId + "?show=auditlog", null, auditLogStatus, jobAuditLogArea,
+ false, null);
+ isAuditLogLoaded=true;
+ }
+ }
+
coord_jobs_grid.setVisible(false);
});