You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by mo...@apache.org on 2014/07/17 01:20:20 UTC
git commit: OOZIE-1933 SLACalculatorMemory HA changes assume
SLARegistrationBean exists for all jobs (mona)
Repository: oozie
Updated Branches:
refs/heads/master 619952f3f -> 7bc738989
OOZIE-1933 SLACalculatorMemory HA changes assume SLARegistrationBean exists for all jobs (mona)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/7bc73898
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/7bc73898
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/7bc73898
Branch: refs/heads/master
Commit: 7bc738989a5811fdd206b827f2c7acd428d8c4b5
Parents: 619952f
Author: mona <ch...@yahoo-inc.com>
Authored: Wed Jul 16 16:20:12 2014 -0700
Committer: mona <ch...@yahoo-inc.com>
Committed: Wed Jul 16 16:20:12 2014 -0700
----------------------------------------------------------------------
.../oozie/service/EventHandlerService.java | 28 ++-------
.../apache/oozie/service/ShareLibService.java | 2 +
.../oozie/service/WorkflowAppService.java | 2 +
.../org/apache/oozie/sla/SLACalcStatus.java | 10 +---
.../apache/oozie/sla/SLACalculatorMemory.java | 29 +++++++---
.../java/org/apache/oozie/util/LogUtils.java | 54 +++++++++++++++++-
.../oozie/service/TestEventHandlerService.java | 60 +++++++++++++++++++-
.../apache/oozie/service/TestHASLAService.java | 1 +
.../oozie/sla/TestSLACalculatorMemory.java | 38 +++++++++++--
.../org/apache/oozie/sla/TestSLAService.java | 2 +-
.../java/org/apache/oozie/test/XTestCase.java | 30 ++++++++++
release-log.txt | 4 +-
12 files changed, 210 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/service/EventHandlerService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/EventHandlerService.java b/core/src/main/java/org/apache/oozie/service/EventHandlerService.java
index 6c075ab..761537b 100644
--- a/core/src/main/java/org/apache/oozie/service/EventHandlerService.java
+++ b/core/src/main/java/org/apache/oozie/service/EventHandlerService.java
@@ -208,35 +208,19 @@ public class EventHandlerService implements Service {
return listenerMap.toString();
}
- public synchronized void queueEvent(Event event) {
- setLogPrefix(LOG, event);
+ public void queueEvent(Event event) {
+ LOG = LogUtils.setLogPrefix(LOG, event);
LOG.debug("Queueing event : {0}", event);
LOG.trace("Stack trace while queueing event : {0}", event, new Throwable());
eventQueue.add(event);
+ LogUtils.clearLogPrefix();
}
public EventQueue getEventQueue() {
return eventQueue;
}
- private void setLogPrefix(XLog logObj, Event event) {
- logObj = XLog.resetPrefix(logObj);
- if (event instanceof JobEvent) {
- JobEvent je = (JobEvent) event;
- LogUtils.setLogPrefix(je.getId(), je.getAppName(), new XLog.Info());
- }
- else if (event instanceof SLAEvent) {
- SLAEvent se = (SLAEvent) event;
- LogUtils.setLogPrefix(se.getId(), se.getAppName(), new XLog.Info());
- }
- }
-
public class EventWorker implements Runnable {
- private XLog workerLog;
-
- public EventWorker() {
- workerLog = XLog.getLog(getClass());
- }
@Override
public void run() {
@@ -247,10 +231,8 @@ public class EventHandlerService implements Service {
if (!eventQueue.isEmpty()) {
List<Event> work = eventQueue.pollBatch();
for (Event event : work) {
- synchronized (workerLog) {
- setLogPrefix(workerLog, event);
- LOG.debug("Processing event : {0}", event);
- }
+ LOG = LogUtils.setLogPrefix(LOG, event);
+ LOG.debug("Processing event : {0}", event);
MessageType msgType = event.getMsgType();
List<?> listeners = listenerMap.get(msgType);
if (listeners != null) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index 893b85b..7b8bc8c 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -472,6 +472,7 @@ public class ShareLibService implements Service, Instrumentable {
else {
Path shareLibpath = getLatestLibPath(services.get(WorkflowAppService.class).getSystemLibPath(),
SHARED_LIB_PREFIX);
+ LOG.debug("MONA******* sharelibpath"+ shareLibpath);
loadShareLibfromDFS(tempShareLibMap, shareLibpath);
if (shareLibpath != null) {
@@ -496,6 +497,7 @@ public class ShareLibService implements Service, Instrumentable {
private void loadShareLibfromDFS(Map<String, List<Path>> shareLibMap, Path shareLibpath) throws IOException {
if (shareLibpath == null) {
+ LOG.debug("MONA*******NULL? sharelibpath"+ shareLibpath);
LOG.info("No share lib directory found");
return;
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
index 05efefb..26beaf6 100644
--- a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
+++ b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
@@ -79,6 +79,7 @@ public abstract class WorkflowAppService implements Service {
String path = conf.get(SYSTEM_LIB_PATH, " ");
if (path.trim().length() > 0) {
systemLibPath = new Path(path.trim());
+ XLog.getLog(getClass()).debug("MONA******* systemlibpath="+systemLibPath);
}
maxWFLength = conf.getInt(CONFG_MAX_WF_LENGTH, 100000);
@@ -329,6 +330,7 @@ public abstract class WorkflowAppService implements Service {
* @return Oozie system libpath (sharelib) in HDFS if present, otherwise it returns <code>NULL</code>.
*/
public Path getSystemLibPath() {
+ XLog.getLog(getClass()).debug("MONA******* getting systemlibpath="+systemLibPath);
return systemLibPath;
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java b/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
index 5349b33..67d6237 100644
--- a/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
+++ b/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
@@ -53,7 +53,7 @@ public class SLACalcStatus extends SLAEvent {
public SLACalcStatus(SLARegistrationBean reg) {
this();
setSLARegistrationBean(reg);
- setLogPrefix();
+ LOG = LogUtils.setLogPrefix(LOG, this);
}
public SLACalcStatus(SLASummaryBean summary, SLARegistrationBean regBean) {
@@ -82,7 +82,7 @@ public class SLACalcStatus extends SLAEvent {
setEventStatus(summary.getEventStatus());
setLastModifiedTime(summary.getLastModifiedTime());
setEventProcessed(summary.getEventProcessed());
- setLogPrefix();
+ LOG = LogUtils.setLogPrefix(LOG, this);
}
/**
@@ -99,7 +99,7 @@ public class SLACalcStatus extends SLAEvent {
setActualEnd(a.getActualEnd());
setActualDuration(a.getActualDuration());
setEventProcessed(a.getEventProcessed());
- setLogPrefix();
+ LOG = LogUtils.setLogPrefix(LOG, this);
}
public SLACalcStatus() {
@@ -332,8 +332,4 @@ public class SLACalcStatus extends SLAEvent {
return Services.get().getConf().getLong(SLAService.CONF_SLA_CALC_LOCK_TIMEOUT, 5 * 1000);
}
- private void setLogPrefix() {
- LOG = XLog.resetPrefix(LOG);
- LogUtils.setLogPrefix(this.getId(), this.getAppName(), new XLog.Info());
- }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
index 5b30fc0..3801325 100644
--- a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
+++ b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
@@ -72,6 +72,7 @@ import org.apache.oozie.service.ServiceException;
import org.apache.oozie.service.Services;
import org.apache.oozie.sla.service.SLAService;
import org.apache.oozie.util.DateUtils;
+import org.apache.oozie.util.LogUtils;
import org.apache.oozie.util.XLog;
import com.google.common.annotations.VisibleForTesting;
@@ -83,7 +84,7 @@ import com.google.common.annotations.VisibleForTesting;
*/
public class SLACalculatorMemory implements SLACalculator {
- private static final XLog LOG = XLog.getLog(SLACalculatorMemory.class);
+ private static XLog LOG = XLog.getLog(SLACalculatorMemory.class);
// TODO optimization priority based insertion/processing/bumping up-down
protected Map<String, SLACalcStatus> slaMap;
protected Set<String> historySet;
@@ -598,7 +599,9 @@ public class SLACalculatorMemory implements SLACalculator {
updateJobSla(jobId);
}
catch (Exception e) {
+ setLogPrefix(jobId);
LOG.error("Exception in SLA processing for job [{0}]", jobId, e);
+ LogUtils.clearLogPrefix();
}
}
}
@@ -626,9 +629,11 @@ public class SLACalculatorMemory implements SLACalculator {
return true;
}
else {
+ setLogPrefix(reg.getId());
LOG.error(
"SLACalculator memory capacity reached. Cannot add or update new SLA Registration entry for job [{0}]",
reg.getId());
+ LogUtils.clearLogPrefix();
}
}
catch (JPAExecutorException jpa) {
@@ -675,9 +680,11 @@ public class SLACalculatorMemory implements SLACalculator {
return true;
}
else {
+ setLogPrefix(reg.getId());
LOG.error(
"SLACalculator memory capacity reached. Cannot add or update new SLA Registration entry for job [{0}]",
reg.getId());
+ LogUtils.clearLogPrefix();
}
}
catch (JPAExecutorException jpa) {
@@ -721,15 +728,19 @@ public class SLACalculatorMemory implements SLACalculator {
slaInfo.setEventProcessed(8);
historySet.remove(jobId);
hasSla = true;
- } else {
+ }
+ else if (Services.get().get(JobsConcurrencyService.class).isHighlyAvailableMode()) {
// jobid might not exist in slaMap in HA Setting
SLARegistrationBean slaRegBean = SLARegistrationQueryExecutor.getInstance().get(
SLARegQuery.GET_SLA_REG_ALL, jobId);
- SLASummaryBean slaSummaryBean = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY,
- jobId);
- slaCalc = new SLACalcStatus(slaSummaryBean, slaRegBean);
- if(slaCalc.getEventProcessed() < 7){
- slaMap.put(jobId, slaCalc);
+ if (slaRegBean != null) { // filter out jobs picked by SLA job event listener
+ // but not actually configured for SLA
+ SLASummaryBean slaSummaryBean = SLASummaryQueryExecutor.getInstance().get(
+ SLASummaryQuery.GET_SLA_SUMMARY, jobId);
+ slaCalc = new SLACalcStatus(slaSummaryBean, slaRegBean);
+ if (slaCalc.getEventProcessed() < 7) {
+ slaMap.put(jobId, slaCalc);
+ }
}
}
}
@@ -1080,4 +1091,8 @@ public class SLACalculatorMemory implements SLACalculator {
public boolean isJobIdInHistorySet(String jobId) {
return this.historySet.contains(jobId);
}
+
+ private void setLogPrefix(String jobId) {
+ LOG = LogUtils.setLogInfo(LOG, jobId, null, null);
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/util/LogUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/LogUtils.java b/core/src/main/java/org/apache/oozie/util/LogUtils.java
index 723ac36..814caff 100644
--- a/core/src/main/java/org/apache/oozie/util/LogUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/LogUtils.java
@@ -17,13 +17,19 @@
*/
package org.apache.oozie.util;
+import org.apache.oozie.AppType;
import org.apache.oozie.BundleJobBean;
import org.apache.oozie.CoordinatorActionBean;
import org.apache.oozie.CoordinatorJobBean;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.client.event.Event;
+import org.apache.oozie.client.event.JobEvent;
+import org.apache.oozie.client.event.SLAEvent;
import org.apache.oozie.service.DagXLogInfoService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.UUIDService;
import org.apache.oozie.service.XLogService;
/**
@@ -125,10 +131,52 @@ public class LogUtils {
XLog.Info.get().setParameters(logInfo);
}
- public static void setLogPrefix(String jobId, String appName, XLog.Info logInfo) {
+ public static XLog setLogInfo(XLog logObj, String jobId, String actionId, String appName) {
+ clearLogPrefix();
+ XLog.Info logInfo = XLog.Info.get();
logInfo.setParameter(DagXLogInfoService.JOB, jobId);
- logInfo.setParameter(DagXLogInfoService.APP, appName);
- XLog.Info.get().setParameters(logInfo);
+ if (actionId != null) {
+ logInfo.setParameter(DagXLogInfoService.ACTION, actionId);
+ }
+ if (appName != null) {
+ logInfo.setParameter(DagXLogInfoService.APP, appName);
+ }
+ return XLog.resetPrefix(logObj);
+ }
+
+ public static XLog setLogPrefix(XLog logObj, Event event) {
+ String jobId = null, actionId = null, appName = null;
+ if (event instanceof JobEvent) {
+ JobEvent je = (JobEvent) event;
+ if (je.getAppType() == AppType.WORKFLOW_JOB || je.getAppType() == AppType.COORDINATOR_JOB
+ || je.getAppType() == AppType.BUNDLE_JOB) {
+ jobId = je.getId();
+ }
+ else {
+ actionId = je.getId();
+ jobId = Services.get().get(UUIDService.class).getId(actionId);
+ }
+ appName = je.getAppName();
+ }
+ else if (event instanceof SLAEvent) {
+ SLAEvent se = (SLAEvent) event;
+ if (se.getAppType() == AppType.WORKFLOW_JOB || se.getAppType() == AppType.COORDINATOR_JOB
+ || se.getAppType() == AppType.BUNDLE_JOB) {
+ jobId = se.getId();
+ }
+ else {
+ actionId = se.getId();
+ jobId = Services.get().get(UUIDService.class).getId(actionId);
+ }
+ appName = se.getAppName();
+ }
+ return LogUtils.setLogInfo(logObj, jobId, actionId, appName);
+ }
+
+ public static void clearLogPrefix() {
+ XLog.Info.get().clearParameter(DagXLogInfoService.JOB);
+ XLog.Info.get().clearParameter(DagXLogInfoService.ACTION);
+ XLog.Info.get().clearParameter(DagXLogInfoService.APP);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java b/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java
index ffb25e7..8bedc95 100644
--- a/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java
@@ -17,9 +17,13 @@
*/
package org.apache.oozie.service;
+import java.util.List;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.CoordinatorAction;
@@ -107,7 +111,7 @@ public class TestEventHandlerService extends XDataTestCase {
/*
* Coordinator Action events
*/
- CoordinatorActionEvent event2 = new CoordinatorActionEvent("jobid", "parentid",
+ CoordinatorActionEvent event2 = new CoordinatorActionEvent("parentid@1", "parentid",
CoordinatorAction.Status.WAITING, getTestUser(), "myapp", null, null, null);
ehs.queueEvent(event2);
ehs.new EventWorker().run();
@@ -147,7 +151,7 @@ public class TestEventHandlerService extends XDataTestCase {
/*
* Workflow Action events
*/
- WorkflowActionEvent event3 = new WorkflowActionEvent("waction-1", "parentid",
+ WorkflowActionEvent event3 = new WorkflowActionEvent("parentid@wfaction", "parentid",
WorkflowAction.Status.RUNNING, getTestUser(), "myapp", null, null);
ehs.queueEvent(event3);
ehs.new EventWorker().run();
@@ -186,6 +190,58 @@ public class TestEventHandlerService extends XDataTestCase {
}
+ @Test
+ public void testEventLogging() throws Exception {
+ EventHandlerService ehs = _testEventHandlerService();
+ // job event
+ WorkflowJobEvent event = new WorkflowJobEvent("jobid", "parentid", WorkflowJob.Status.RUNNING, getTestUser(),
+ "myapp", null, null);
+
+ TestLogAppender appender = null;
+ Logger logger = null;
+ try {
+ appender = getTestLogAppender();
+ logger = Logger.getLogger(EventHandlerService.class);
+ logger.addAppender(appender);
+ logger.setLevel(Level.DEBUG);
+ ehs.queueEvent(event);
+
+ List<LoggingEvent> log = appender.getLog();
+ LoggingEvent logEntry = log.get(0);
+ assertEquals(Level.DEBUG, logEntry.getLevel());
+ assertTrue(logEntry.getMessage().toString().contains("APP[myapp] JOB[jobid] ACTION[-] Queueing event : ID: jobid"));
+ assertEquals("org.apache.oozie.service.EventHandlerService", logEntry.getLoggerName());
+
+ ehs.new EventWorker().run();
+
+ log = appender.getLog();
+ logEntry = log.get(1);
+ assertEquals(Level.DEBUG, logEntry.getLevel());
+ assertTrue(logEntry.getMessage().toString().contains("APP[myapp] JOB[jobid] ACTION[-] Processing event : ID: jobid"));
+
+ // action event
+ CoordinatorActionEvent event2 = new CoordinatorActionEvent("jobid2@1", "parentid",
+ CoordinatorAction.Status.WAITING, getTestUser(), "myapp", null, null, null);
+ ehs.queueEvent(event2);
+
+ log = appender.getLog();
+ logEntry = log.get(2);
+ assertTrue(logEntry.getMessage().toString().contains("APP[myapp] JOB[jobid2] ACTION[jobid2@1] Queueing event : ID: jobid2@1"));
+
+ WorkflowJobEvent event3 = new WorkflowJobEvent("jobid-other", "parentid", WorkflowJob.Status.RUNNING, getTestUser(),
+ "myapp-other", null, null);
+ ehs.queueEvent(event3);
+
+ log = appender.getLog();
+ logEntry = log.get(3);
+ assertTrue(logEntry.getMessage().toString()
+ .contains("APP[myapp-other] JOB[jobid-other] ACTION[-] Queueing event : ID: jobid-other"));
+ }
+ finally {
+ logger.removeAppender(appender);
+ }
+ }
+
private EventHandlerService _testEventHandlerService() throws Exception {
Services services = Services.get();
EventHandlerService ehs = services.get(EventHandlerService.class);
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestHASLAService.java b/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
index 419e98b..a472f1f 100644
--- a/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestHASLAService.java
@@ -68,6 +68,7 @@ public class TestHASLAService extends ZKXTestCase {
conf.setInt(SLAService.CONF_SLA_CHECK_INTERVAL, 100000);
conf.setInt(EventHandlerService.CONF_WORKER_THREADS, 0);
super.setUp(conf);
+ Services.get().setService(ZKJobsConcurrencyService.class);
}
@Override
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java b/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
index 438f2c2..210c99e 100644
--- a/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
+++ b/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java
@@ -20,11 +20,12 @@ package org.apache.oozie.sla;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
import org.apache.oozie.AppType;
import org.apache.oozie.CoordinatorActionBean;
import org.apache.oozie.WorkflowActionBean;
@@ -49,6 +50,7 @@ import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
import org.apache.oozie.service.EventHandlerService;
import org.apache.oozie.service.JPAService;
import org.apache.oozie.service.Services;
+import org.apache.oozie.sla.service.SLAService;
import org.apache.oozie.test.XDataTestCase;
import org.apache.oozie.workflow.WorkflowInstance;
import org.junit.After;
@@ -295,7 +297,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd");
SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
slaCalcMemory.init(new Configuration(false));
- SLARegistrationBean slaRegBean1 = _createSLARegistration("job-1", AppType.WORKFLOW_ACTION);
+ SLARegistrationBean slaRegBean1 = _createSLARegistration("job@1", AppType.WORKFLOW_ACTION);
String jobId1 = slaRegBean1.getId();
slaRegBean1.setExpectedEnd(sdf.parse("2013-03-07"));
slaRegBean1.setExpectedStart(sdf.parse("2012-03-07"));
@@ -324,7 +326,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
// As job succeeded, it should not be in memory
assertEquals(0, slaCalcMemory.size());
SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
- assertEquals("job-1", slaSummary.getId());
+ assertEquals("job@1", slaSummary.getId());
assertEquals(8, slaSummary.getEventProcessed());
assertEquals(AppType.WORKFLOW_ACTION, slaSummary.getAppType());
assertEquals("OK", slaSummary.getJobStatus());
@@ -341,7 +343,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd");
SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
slaCalcMemory.init(new Configuration(false));
- SLARegistrationBean slaRegBean1 = _createSLARegistration("job-1", AppType.COORDINATOR_ACTION);
+ SLARegistrationBean slaRegBean1 = _createSLARegistration("job@1", AppType.COORDINATOR_ACTION);
String jobId1 = slaRegBean1.getId();
slaRegBean1.setExpectedEnd(sdf.parse("2013-03-07"));
slaRegBean1.setExpectedStart(sdf.parse("2012-03-07"));
@@ -376,7 +378,7 @@ public class TestSLACalculatorMemory extends XDataTestCase {
assertEquals(0, slaCalcMemory.size());
SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
- assertEquals("job-1", slaSummary.getId());
+ assertEquals("job@1", slaSummary.getId());
assertEquals(8, slaSummary.getEventProcessed());
assertEquals(AppType.COORDINATOR_ACTION, slaSummary.getAppType());
assertEquals("FAILED", slaSummary.getJobStatus());
@@ -635,4 +637,28 @@ public class TestSLACalculatorMemory extends XDataTestCase {
assertFalse(slaCalcMemory.isJobIdInHistorySet(job1.getId()));
}
+ public void testErrorLoggingWithJobIdPrefix() throws Exception {
+ SLAService slaService = Services.get().get(SLAService.class);
+ SLACalculator slaCalculator = slaService.getSLACalculator();
+ slaCalculator.addRegistration("dummy-id", _createSLARegistration("dummy-id", AppType.WORKFLOW_JOB));
+
+ TestLogAppender appender = getTestLogAppender();
+ Logger logger = Logger.getLogger(SLACalculatorMemory.class);
+ logger.addAppender(appender);
+ logger.setLevel(Level.ERROR);
+ try {
+ slaService.runSLAWorker();
+ }
+ finally {
+ logger.removeAppender(appender);
+ }
+
+ List<LoggingEvent> log = appender.getLog();
+ LoggingEvent firstLogEntry = log.get(0);
+ assertEquals(Level.ERROR, firstLogEntry.getLevel());
+ assertTrue(firstLogEntry.getMessage().toString().contains("JOB[dummy-id]"));
+ assertEquals("org.apache.oozie.sla.SLACalculatorMemory", firstLogEntry.getLoggerName());
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/sla/TestSLAService.java b/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
index 205bcd1..e8f18cd 100644
--- a/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
+++ b/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
@@ -150,7 +150,7 @@ public class TestSLAService extends XDataTestCase {
// As expected duration is not set, duration shall be processed and job removed from map
assertEquals(2, slas.getSLACalculator().size());
// test same job multiple events (start-met, end-met) through job status event
- sla1 = _createSLARegistration("action-1", AppType.COORDINATOR_ACTION);
+ sla1 = _createSLARegistration("action@1", AppType.COORDINATOR_ACTION);
sla1.setExpectedStart(new Date(System.currentTimeMillis() + 1 * 3600 * 1000)); //1 hour ahead
sla1.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 3600 * 1000)); //2 hours ahead
slas.addRegistrationEvent(sla1);
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/test/XTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java
index e739ec3..7661ebe 100644
--- a/core/src/test/java/org/apache/oozie/test/XTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java
@@ -26,6 +26,7 @@ import java.io.OutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URL;
+import java.util.ArrayList;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
@@ -52,6 +53,8 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.util.Shell;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.spi.LoggingEvent;
import org.apache.oozie.BundleActionBean;
import org.apache.oozie.BundleJobBean;
import org.apache.oozie.CoordinatorActionBean;
@@ -1159,5 +1162,32 @@ public abstract class XTestCase extends TestCase {
setupServicesForHCataLogImpl(services);
return services;
}
+
+ protected class TestLogAppender extends AppenderSkeleton {
+ private final List<LoggingEvent> log = new ArrayList<LoggingEvent>();
+
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ @Override
+ protected void append(final LoggingEvent loggingEvent) {
+ log.add(loggingEvent);
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public List<LoggingEvent> getLog() {
+ return new ArrayList<LoggingEvent>(log);
+ }
+ }
+
+ public TestLogAppender getTestLogAppender() {
+ return new TestLogAppender();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 912e277..874e58a 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,9 +1,11 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1933 SLACalculatorMemory HA changes assume SLARegistrationBean exists for all jobs (mona)
+OOZIE-1935 Log level (ActionStartXCommand) and Doc fix (CoordinatorFunctionalSpec) (mona)
OOZIE-1586 upgrade oozie to hive 13.1 (including hcatalog) (raviprak via rkanter)
OOZIE-1932 Services should load CallableQueueService after MemoryLocksService (mona)
OOZIE-1740 Add a new function hadoop:conf() that can be invoked from the workflow.xml and will return a hadoop configuration option (sam liu via rkanter)
-OOZIE-1911 SLA calculation in HA mode does wrong bit comparison for 'start' and 'duration' (mna)
+OOZIE-1911 SLA calculation in HA mode does wrong bit comparison for 'start' and 'duration' (mona)
OOZIE-1926 make gz blob compression as default (ryota)
OOZIE-1916 Use Curator leader latch instead of checking the order of Oozie servers (rkanter)
OOZIE-1886 Queue operation talking longer time (shwethags via rohini)