You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/14 01:46:32 UTC
svn commit: r1446002 - in /oozie/branches/branch-3.3: ./
client/src/main/java/org/apache/oozie/cli/
core/src/main/java/org/apache/oozie/
core/src/main/java/org/apache/oozie/client/rest/
core/src/main/java/org/apache/oozie/command/coord/ core/src/main/j...
Author: rkanter
Date: Thu Feb 14 00:46:31 2013
New Revision: 1446002
URL: http://svn.apache.org/r1446002
Log:
OOZIE-1187 reduce memory usage of SLA query (invoked by CLI command) to avoid OOM (egashira via virag)
Modified:
oozie/branches/branch-3.3/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/SLAEventBean.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/store/SLAStore.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
oozie/branches/branch-3.3/docs/src/site/twiki/DG_CommandLineTool.twiki
oozie/branches/branch-3.3/release-log.txt
Modified: oozie/branches/branch-3.3/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/client/src/main/java/org/apache/oozie/cli/OozieCLI.java (original)
+++ oozie/branches/branch-3.3/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Thu Feb 14 00:46:31 2013
@@ -133,7 +133,7 @@ public class OozieCLI {
public static final String DEBUG_OPTION = "debug";
public static final String PIGFILE_OPTION = "file";
-
+
public static final String INFO_TIME_ZONES_OPTION = "timezones";
public static final String BULK_OPTION = "bulk";
@@ -256,9 +256,9 @@ public class OozieCLI {
Option len = new Option(LEN_OPTION, true, "number of actions (default TOTAL ACTIONS, requires -info)");
Option filter = new Option(FILTER_OPTION, true,
"status=<S1>[;status=<S2>]* (All Coordinator actions satisfying any one of the status filters will be retreived. Currently, only supported for Coordinator job)");
- Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" +
+ Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" +
TIME_ZONE_OPTION + "). Overrides -" + TIME_ZONE_OPTION + " option");
- Option timezone = new Option(TIME_ZONE_OPTION, true,
+ Option timezone = new Option(TIME_ZONE_OPTION, true,
"use time zone with the specified ID (default GMT).\nSee 'oozie info -timezones' for a list");
Option log = new Option(LOG_OPTION, true, "job log");
Option definition = new Option(DEFINITION_OPTION, true, "job definition");
@@ -328,9 +328,9 @@ public class OozieCLI {
Option len = new Option(LEN_OPTION, true, "number of jobs (default '100')");
Option filter = new Option(FILTER_OPTION, true, "user=<U>\\;name=<N>\\;group=<G>\\;status=<S>\\;frequency=<F>\\;unit=<M> " +
"(Valid unit values are 'months', 'days', 'hours' or 'minutes'.)");
- Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" +
+ Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" +
TIME_ZONE_OPTION + "). Overrides -" + TIME_ZONE_OPTION + " option");
- Option timezone = new Option(TIME_ZONE_OPTION, true,
+ Option timezone = new Option(TIME_ZONE_OPTION, true,
"use time zone with the specified ID (default GMT).\nSee 'oozie info -timezones' for a list");
Option verbose = new Option(VERBOSE_OPTION, false, "verbose mode");
Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
@@ -363,7 +363,7 @@ public class OozieCLI {
protected Options createSlaOptions() {
Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
Option start = new Option(OFFSET_OPTION, true, "start offset (default '0')");
- Option len = new Option(LEN_OPTION, true, "number of results (default '100')");
+ Option len = new Option(LEN_OPTION, true, "number of results (default '100', max '1000')");
start.setType(Integer.class);
len.setType(Integer.class);
Options slaOptions = new Options();
@@ -395,7 +395,7 @@ public class OozieCLI {
addAuthOptions(pigOptions);
return pigOptions;
}
-
+
/**
* Create option for command line option 'info'
* @return info options
@@ -536,7 +536,7 @@ public class OozieCLI {
}
return url;
}
-
+
private String getTimeZoneId(CommandLine commandLine)
{
if (commandLine.hasOption(LOCAL_TIME_OPTION)) {
@@ -548,7 +548,7 @@ public class OozieCLI {
String timeZoneId = System.getenv(ENV_OOZIE_TIME_ZONE);
if (timeZoneId != null) {
return timeZoneId;
- }
+ }
return "GMT";
}
@@ -856,7 +856,7 @@ public class OozieCLI {
if (filter != null) {
throw new OozieCLIException("Filter option is currently not supported for a Bundle job");
}
- printBundleJob(wc.getBundleJobInfo(commandLine.getOptionValue(INFO_OPTION)), timeZoneId,
+ printBundleJob(wc.getBundleJobInfo(commandLine.getOptionValue(INFO_OPTION)), timeZoneId,
options.contains(VERBOSE_OPTION));
}
else if (commandLine.getOptionValue(INFO_OPTION).endsWith("-C")) {
@@ -880,7 +880,7 @@ public class OozieCLI {
if (filter != null) {
throw new OozieCLIException("Filter option is not supported for a Workflow action");
}
- printWorkflowAction(wc.getWorkflowActionInfo(commandLine.getOptionValue(INFO_OPTION)), timeZoneId,
+ printWorkflowAction(wc.getWorkflowActionInfo(commandLine.getOptionValue(INFO_OPTION)), timeZoneId,
options.contains(VERBOSE_OPTION));
}
else {
@@ -894,7 +894,7 @@ public class OozieCLI {
String jobtype = commandLine.getOptionValue(JOBTYPE_OPTION);
jobtype = (jobtype != null) ? jobtype : "wf";
int len = Integer.parseInt((s != null) ? s : "0");
- printJob(wc.getJobInfo(commandLine.getOptionValue(INFO_OPTION), start, len), timeZoneId,
+ printJob(wc.getJobInfo(commandLine.getOptionValue(INFO_OPTION), start, len), timeZoneId,
options.contains(VERBOSE_OPTION));
}
}
@@ -1520,7 +1520,7 @@ public class OozieCLI {
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"sqoop-action-0.3.xsd")));
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
- "sqoop-action-0.4.xsd")));
+ "sqoop-action-0.4.xsd")));
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"ssh-action-0.1.xsd")));
SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
@@ -1572,7 +1572,7 @@ public class OozieCLI {
throw new OozieCLIException(ex.toString(), ex);
}
}
-
+
private void infoCommand(CommandLine commandLine) throws OozieCLIException {
for (Option option : commandLine.getOptions()) {
String opt = option.getOpt();
@@ -1581,7 +1581,7 @@ public class OozieCLI {
}
}
}
-
+
private void printAvailableTimeZones() {
System.out.println("The format is \"SHORT_NAME (ID)\"\nGive the ID to the -timezone argument");
System.out.println("GMT offsets can also be used (e.g. GMT-07:00, GMT-0700, GMT+05:30, GMT+0530)");
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/SLAEventBean.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/SLAEventBean.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/SLAEventBean.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/SLAEventBean.java Thu Feb 14 00:46:31 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -42,9 +42,9 @@ import org.json.simple.JSONObject;
@Entity
@NamedQueries({
+ @NamedQuery(name = "GET_SLA_EVENT_NEWER_SEQ_LIMITED", query = "select OBJECT(w) from SLAEventBean w where w.event_id > :id order by w.event_id"),
+ @NamedQuery(name = "GET_SLA_EVENTS", query = "select OBJECT(w) from SLAEventBean w") })
- @NamedQuery(name = "GET_SLA_EVENT_NEWER_SEQ_LIMITED", query = "select OBJECT(w) from SLAEventBean w where w.event_id > :id order by w.event_id"),
- @NamedQuery(name = "GET_SLA_EVENTS", query = "select OBJECT(w) from SLAEventBean w")})
public class SLAEventBean extends JsonSLAEvent implements Writable {
@Basic
@@ -180,8 +180,7 @@ public class SLAEventBean extends JsonSL
@Override
public String toString() {
- return MessageFormat.format("Event id[{0}] status[{1}]", getEvent_id(),
- getJobStatus());
+ return MessageFormat.format("Event id[{0}] status[{1}]", getEvent_id(), getJobStatus());
}
/**
@@ -233,7 +232,6 @@ public class SLAEventBean extends JsonSL
eReg.addContent(createATagElement("sequence-id", String.valueOf(getEvent_id())));
Element e = new Element("registration");
e.addContent(createATagElement("sla-id", getSlaId()));
- //e.addContent(createATagElement("sla-id", String.valueOf(getSlaId())));
e.addContent(createATagElement("app-type", getAppType().toString()));
e.addContent(createATagElement("app-name", getAppName()));
e.addContent(createATagElement("user", getUser()));
@@ -268,10 +266,11 @@ public class SLAEventBean extends JsonSL
eStat.addContent(createATagElement("sequence-id", String.valueOf(getEvent_id())));
Element e = new Element("status");
e.addContent(createATagElement("sla-id", getSlaId()));
- e.addContent(createATagElement("status-timestamp",
- getDateString(getStatusTimestamp())));
+ e.addContent(createATagElement("status-timestamp", getDateString(getStatusTimestamp())));
e.addContent(createATagElement("job-status", getJobStatus().toString()));
e.addContent(createATagElement("job-data", getJobData()));
+ e.addContent(createATagElement("user", getUser()));
+ e.addContent(createATagElement("group", getGroupName()));
eStat.addContent(e);
return eStat;
}
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java Thu Feb 14 00:46:31 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -298,7 +298,7 @@ public class JsonSLAEvent implements SLA
// TODO Auto-generated method stub
return null;
}
-
+
@SuppressWarnings("unchecked")
public JSONObject toJSONObject(String timeZoneId) {
return null;
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java Thu Feb 14 00:46:31 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,6 +27,7 @@ import org.apache.oozie.command.Precondi
import org.apache.oozie.command.XCommand;
import org.apache.oozie.executor.jpa.SLAEventsGetForSeqIdJPAExecutor;
import org.apache.oozie.service.JPAService;
+import org.apache.oozie.service.Service;
import org.apache.oozie.service.Services;
/**
@@ -39,10 +40,14 @@ public class SLAEventsXCommand extends X
private int maxNoEvents = 100; // Default
private long lastSeqId = -1;
+ public static final String SLA_DEFAULT_MAXEVENTS = Service.CONF_PREFIX + "sla.default.maxevents";
+
public SLAEventsXCommand(long seqId, int maxNoEvnts) {
super("SLAEventsXCommand", "SLAEventsXCommand", 1);
this.seqId = seqId;
- this.maxNoEvents = maxNoEvnts;
+ int sysMax = Services.get().getConf().getInt(SLA_DEFAULT_MAXEVENTS, 1000);
+ this.maxNoEvents = maxNoEvnts > sysMax ? sysMax : maxNoEvnts;
+
}
/* (non-Javadoc)
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java Thu Feb 14 00:46:31 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,10 +18,8 @@
package org.apache.oozie.executor.jpa;
import java.util.List;
-
import javax.persistence.EntityManager;
import javax.persistence.Query;
-
import org.apache.oozie.ErrorCode;
import org.apache.oozie.SLAEventBean;
@@ -64,5 +62,4 @@ public class SLAEventsGetForSeqIdJPAExec
}
return seBeans;
}
-
}
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/store/SLAStore.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/store/SLAStore.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/store/SLAStore.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/store/SLAStore.java Thu Feb 14 00:46:31 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,7 +27,11 @@ import javax.persistence.Query;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.SLAEventBean;
+import org.apache.oozie.command.CommandException;
+import org.apache.oozie.executor.jpa.JPAExecutorException;
+import org.apache.oozie.executor.jpa.SLAEventsGetForSeqIdJPAExecutor;
import org.apache.oozie.service.InstrumentationService;
+import org.apache.oozie.service.JPAService;
import org.apache.oozie.service.Services;
import org.apache.oozie.util.Instrumentation;
import org.apache.oozie.util.ParamChecker;
@@ -66,7 +70,8 @@ public class SLAStore extends Store {
}
/**
- * Get a list of SLA Events newer than a specific sequence with limit clause.
+ * Get a list of SLA Events newer than a specific sequence with limit
+ * clause.
*
* @param seqId sequence id
* @return List of SLA Events
@@ -80,24 +85,34 @@ public class SLAStore extends Store {
lastSeqId[0] = seqId;
List<SLAEventBean> seBeans = (List<SLAEventBean>) doOperation("getSLAEventListNewerSeqLimited",
- new Callable<List<SLAEventBean>>() {
+ new Callable<List<SLAEventBean>>() {
- public List<SLAEventBean> call() throws StoreException {
+ public List<SLAEventBean> call() throws StoreException, JPAExecutorException {
- List<SLAEventBean> seBeans;
- try {
- Query q = entityManager.createNamedQuery("GET_SLA_EVENT_NEWER_SEQ_LIMITED");
- q.setParameter("id", seqId);
- // q.setFirstResult(0);
- q.setMaxResults(limitLen);
- seBeans = q.getResultList();
- }
- catch (IllegalStateException e) {
- throw new StoreException(ErrorCode.E0601, e.getMessage(), e);
- }
- return seBeans;
- }
- });
+ List<SLAEventBean> seBeans;
+ try {
+
+ JPAService jpaService = Services.get().get(JPAService.class);
+ List<SLAEventBean> slaEventList = null;
+ long lastSeqId[] = new long[1];
+ if (jpaService != null) {
+ seBeans = jpaService.execute(new SLAEventsGetForSeqIdJPAExecutor(seqId, limitLen,
+ lastSeqId));
+ }
+ else {
+ throw new StoreException(ErrorCode.E0610);
+ }
+
+ }
+ catch (IllegalStateException e) {
+ throw new StoreException(ErrorCode.E0601, e.getMessage(), e);
+ }
+ catch (JPAExecutorException e) {
+ throw new JPAExecutorException(ErrorCode.E0610, e.getMessage(), e);
+ }
+ return seBeans;
+ }
+ });
List<SLAEventBean> eventList = new ArrayList<SLAEventBean>();
for (SLAEventBean j : seBeans) {
lastSeqId[0] = Math.max(lastSeqId[0], j.getEvent_id());
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java Thu Feb 14 00:46:31 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,6 +21,7 @@ import java.util.Date;
import java.util.List;
import org.apache.oozie.SLAEventBean;
+import org.apache.oozie.client.SLAEvent;
import org.apache.oozie.client.SLAEvent.Status;
import org.apache.oozie.service.JPAService;
import org.apache.oozie.service.Services;
@@ -44,14 +45,15 @@ public class TestSLAEventsGetForSeqIdJPA
}
public void testSLAEventsGetForSeqId() throws Exception {
- final String wfId = "0000000-" + new Date().getTime() + "-TestSLAEventsGetForSeqIdJPAExecutor-W";
- addRecordToSLAEventTable(wfId, Status.CREATED);
- addRecordToSLAEventTable(wfId, Status.STARTED);
- addRecordToSLAEventTable(wfId, Status.SUCCEEDED);
- _testGetSLAEventsForSeqId(wfId);
+ Date current = new Date();
+ final String wfId = "0000000-" + current.getTime() + "-TestSLAEventsGetForSeqIdJPAExecutor-W";
+ addRecordToSLAEventTable(wfId, Status.CREATED, current);
+ addRecordToSLAEventTable(wfId, Status.STARTED, current);
+ addRecordToSLAEventTable(wfId, Status.SUCCEEDED, current);
+ _testGetSLAEventsForSeqId(wfId, current);
}
- private void _testGetSLAEventsForSeqId(String jobId) throws Exception {
+ private void _testGetSLAEventsForSeqId(String jobId, Date current) throws Exception {
JPAService jpaService = Services.get().get(JPAService.class);
assertNotNull(jpaService);
@@ -64,6 +66,12 @@ public class TestSLAEventsGetForSeqIdJPA
List<SLAEventBean> list = jpaService.execute(slaEventsGetCmd);
assertNotNull(list);
assertEquals(2, list.size());
- }
+ SLAEventBean seBean = list.get(0);
+ assertEquals(seBean.getSlaId(), jobId);
+ assertEquals(seBean.getUser(), getTestUser());
+ assertEquals(seBean.getGroupName(), getTestGroup());
+ assertEquals(seBean.getJobStatus(), Status.STARTED);
+ assertEquals(seBean.getStatusTimestamp().getTime(), current.getTime());
+ }
}
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java Thu Feb 14 00:46:31 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -44,10 +44,11 @@ public class TestSLAEventsGetJPAExecutor
}
public void testSLAEventsGetForSeqId() throws Exception {
- final String wfId = "0000000-" + new Date().getTime() + "-TestSLAEventsGetJPAExecutor-W";
- addRecordToSLAEventTable(wfId, Status.CREATED);
- addRecordToSLAEventTable(wfId, Status.STARTED);
- addRecordToSLAEventTable(wfId, Status.SUCCEEDED);
+ Date current = new Date();
+ final String wfId = "0000000-" + current.getTime() + "-TestSLAEventsGetJPAExecutor-W";
+ addRecordToSLAEventTable(wfId, Status.CREATED, current);
+ addRecordToSLAEventTable(wfId, Status.STARTED, current);
+ addRecordToSLAEventTable(wfId, Status.SUCCEEDED, current);
_testGetSLAEventsForSeqId(wfId);
}
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XDataTestCase.java?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XDataTestCase.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XDataTestCase.java Thu Feb 14 00:46:31 2013
@@ -82,9 +82,9 @@ import org.jdom.JDOMException;
public abstract class XDataTestCase extends XFsTestCase {
- protected static String slaXml = " <sla:info xmlns:sla='uri:oozie:sla:0.1'>" + " <sla:app-name>test-app</sla:app-name>"
- + " <sla:nominal-time>2009-03-06T10:00Z</sla:nominal-time>" + " <sla:should-start>5</sla:should-start>"
- + " <sla:should-end>120</sla:should-end>"
+ protected static String slaXml = " <sla:info xmlns:sla='uri:oozie:sla:0.1'>"
+ + " <sla:app-name>test-app</sla:app-name>" + " <sla:nominal-time>2009-03-06T10:00Z</sla:nominal-time>"
+ + " <sla:should-start>5</sla:should-start>" + " <sla:should-end>120</sla:should-end>"
+ " <sla:notification-msg>Notifying User for nominal time : 2009-03-06T10:00Z </sla:notification-msg>"
+ " <sla:alert-contact>abc@example.com</sla:alert-contact>"
+ " <sla:dev-contact>abc@example.com</sla:dev-contact>"
@@ -103,7 +103,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws Exception
*/
- protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, boolean pending, boolean doneMatd) throws Exception {
+ protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, boolean pending,
+ boolean doneMatd) throws Exception {
CoordinatorJobBean coordJob = createCoordJob(status, pending, doneMatd);
try {
@@ -166,8 +167,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws Exception
*/
- protected CoordinatorJobBean addRecordToCoordJobTable(String testFileName, CoordinatorJob.Status status, Date start, Date end,
- boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
+ protected CoordinatorJobBean addRecordToCoordJobTable(String testFileName, CoordinatorJob.Status status,
+ Date start, Date end, boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
CoordinatorJobBean coordJob = createCoordJob(testFileName, status, start, end, pending, doneMatd, lastActionNum);
try {
@@ -221,7 +222,6 @@ public abstract class XDataTestCase exte
return coordJob;
}
-
/**
* Add coordinator job bean with bundle id info.
*
@@ -236,10 +236,12 @@ public abstract class XDataTestCase exte
* @return coordinator job bean
* @throws Exception
*/
- protected CoordinatorJobBean addRecordToCoordJobTableWithBundle(String bundleId, String coordId, CoordinatorJob.Status status, Date start, Date end, boolean pending, boolean doneMatd, int lastActionNumber) throws Exception {
+ protected CoordinatorJobBean addRecordToCoordJobTableWithBundle(String bundleId, String coordId,
+ CoordinatorJob.Status status, Date start, Date end, boolean pending, boolean doneMatd, int lastActionNumber)
+ throws Exception {
CoordinatorJobBean coordJob = createCoordJob(status, start, end, pending, doneMatd, 0);
coordJob.setBundleId(bundleId);
- //coord id and coord name are the same
+ // coord id and coord name are the same
coordJob.setId(coordId);
coordJob.setAppName(coordId);
coordJob.setLastActionNumber(lastActionNumber);
@@ -258,8 +260,6 @@ public abstract class XDataTestCase exte
return coordJob;
}
-
-
/**
* Create coord job bean
*
@@ -269,7 +269,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws IOException
*/
- protected CoordinatorJobBean createCoordJob(CoordinatorJob.Status status, boolean pending, boolean doneMatd) throws Exception {
+ protected CoordinatorJobBean createCoordJob(CoordinatorJob.Status status, boolean pending, boolean doneMatd)
+ throws Exception {
Path appPath = new Path(getFsTestCaseDir(), "coord");
String appXml = writeCoordXml(appPath);
@@ -377,8 +378,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws IOException
*/
- protected CoordinatorJobBean createCoordJob(String testFileName, CoordinatorJob.Status status, Date start, Date end, boolean pending,
- boolean doneMatd, int lastActionNum) throws Exception {
+ protected CoordinatorJobBean createCoordJob(String testFileName, CoordinatorJob.Status status, Date start,
+ Date end, boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
Path appPath = new Path(getFsTestCaseDir(), "coord");
String appXml = writeCoordXml(appPath, testFileName);
@@ -420,7 +421,6 @@ public abstract class XDataTestCase exte
return coordJob;
}
-
/**
* Write coordinator xml
*
@@ -463,7 +463,6 @@ public abstract class XDataTestCase exte
return appXml;
}
-
/**
* Write coordinator xml
*
@@ -521,7 +520,8 @@ public abstract class XDataTestCase exte
* @throws Exception thrown if unable to create coord action bean
*/
protected CoordinatorActionBean addRecordToCoordActionTable(String jobId, int actionNum,
- CoordinatorAction.Status status, String resourceXmlName, String wfId, String wfStatus, int pending) throws Exception {
+ CoordinatorAction.Status status, String resourceXmlName, String wfId, String wfStatus, int pending)
+ throws Exception {
CoordinatorActionBean action = createCoordAction(jobId, actionNum, status, resourceXmlName, pending);
action.setExternalId(wfId);
action.setExternalStatus(wfStatus);
@@ -543,6 +543,7 @@ public abstract class XDataTestCase exte
String resourceXmlName, int pending) throws Exception {
return createCoordAction(jobId, actionNum, status, resourceXmlName, pending, "Z");
}
+
/**
* Create coord action bean
*
@@ -596,9 +597,9 @@ public abstract class XDataTestCase exte
*/
protected WorkflowJobBean addRecordToWfJobTable(WorkflowJob.Status jobStatus, WorkflowInstance.Status instanceStatus)
throws Exception {
- WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>",
- new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).
- addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
+ WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", new StartNodeDef(
+ LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end",
+ LiteWorkflowStoreService.LiteControlNodeHandler.class));
Configuration conf = new Configuration();
Path appUri = new Path(getAppPath(), "workflow.xml");
conf.set(OozieClient.APP_PATH, appUri.toString());
@@ -665,14 +666,14 @@ public abstract class XDataTestCase exte
* @param status sla status
* @throws Exception thrown if unable to create sla bean
*/
- protected void addRecordToSLAEventTable(String slaId, SLAEvent.Status status) throws Exception {
+ protected void addRecordToSLAEventTable(String slaId, SLAEvent.Status status, Date today) throws Exception {
SLAEventBean sla = new SLAEventBean();
sla.setSlaId(slaId);
sla.setAppName("app-name");
- sla.setParentClientId("parent-child-id");
+ sla.setParentClientId("parent-client-id");
sla.setParentSlaId("parent-sla-id");
- sla.setExpectedStart(new Date());
- sla.setExpectedEnd(new Date());
+ sla.setExpectedStart(today);
+ sla.setExpectedEnd(today);
sla.setNotificationMsg("notification-msg");
sla.setAlertContact("alert-contact");
sla.setDevContact("dev-contact");
@@ -685,7 +686,7 @@ public abstract class XDataTestCase exte
sla.setUser(getTestUser());
sla.setGroupName(getTestGroup());
sla.setJobStatus(status);
- sla.setStatusTimestamp(new Date());
+ sla.setStatusTimestamp(today);
try {
JPAService jpaService = Services.get().get(JPAService.class);
@@ -918,7 +919,6 @@ public abstract class XDataTestCase exte
jobConf.set("nameNode", getNameNodeUri());
jobConf.set("wfAppPath", wfAppPath.toString());
-
String content = "<workflow-app xmlns='uri:oozie:workflow:0.1' xmlns:sla='uri:oozie:sla:0.1' name='no-op-wf'>";
content += "<start to='end' />";
content += "<end name='end' /></workflow-app>";
@@ -1055,8 +1055,6 @@ public abstract class XDataTestCase exte
.replaceAll("#app_path1", coordPath1.toString() + File.separator + "coordinator.xml");
bundleAppXml = bundleAppXml
.replaceAll("#app_path2", coordPath2.toString() + File.separator + "coordinator.xml");
- // bundleAppXml = bundleAppXml.replaceAll("#app_path1", coordPath1.toString());
- // bundleAppXml = bundleAppXml.replaceAll("#app_path2", coordPath2.toString());
writeToFile(bundleAppXml, bundleAppPath, "bundle.xml");
@@ -1083,7 +1081,8 @@ public abstract class XDataTestCase exte
bundle.setOrigJobXml(bundleAppXml);
if (pending) {
bundle.setPending();
- } else {
+ }
+ else {
bundle.resetPending();
}
bundle.setStatus(jobStatus);
@@ -1121,7 +1120,6 @@ public abstract class XDataTestCase exte
conf.set("jobTracker", getJobTrackerUri());
conf.set("nameNode", getNameNodeUri());
-
BundleJobBean bundle = new BundleJobBean();
bundle.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE));
bundle.setAppName("BUNDLE-TEST");
@@ -1154,10 +1152,11 @@ public abstract class XDataTestCase exte
/**
* Inserts a record to coord action table
+ *
* @param action the record to be inserted
* @throws Exception
*/
- protected void insertRecordCoordAction(CoordinatorActionBean action) throws Exception {
+ protected void insertRecordCoordAction(CoordinatorActionBean action) throws Exception {
try {
JPAService jpaService = Services.get().get(JPAService.class);
assertNotNull(jpaService);
@@ -1171,19 +1170,19 @@ public abstract class XDataTestCase exte
}
}
- // Exclude some of the services classes from loading so they dont interfere
- // while the test case is running
- protected void setClassesToBeExcluded(Configuration conf, String[] excludedServices) {
- String classes = conf.get(Services.CONF_SERVICE_CLASSES);
- StringBuilder builder = new StringBuilder(classes);
- for (String s : excludedServices) {
- int index = builder.indexOf(s);
- if (index != -1) {
- builder.replace(index, index + s.length() + 1, "");
- }
- }
- conf.set(Services.CONF_SERVICE_CLASSES, new String(builder));
- }
+ // Exclude some of the services classes from loading so they dont interfere
+ // while the test case is running
+ protected void setClassesToBeExcluded(Configuration conf, String[] excludedServices) {
+ String classes = conf.get(Services.CONF_SERVICE_CLASSES);
+ StringBuilder builder = new StringBuilder(classes);
+ for (String s : excludedServices) {
+ int index = builder.indexOf(s);
+ if (index != -1) {
+ builder.replace(index, index + s.length() + 1, "");
+ }
+ }
+ conf.set(Services.CONF_SERVICE_CLASSES, new String(builder));
+ }
/**
* Adds the db records for the Bulk Monitor tests
@@ -1247,6 +1246,7 @@ public abstract class XDataTestCase exte
/**
* Add a month to the current time
+ *
* @param incrementMonth
* @return
*/
Modified: oozie/branches/branch-3.3/docs/src/site/twiki/DG_CommandLineTool.twiki
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/docs/src/site/twiki/DG_CommandLineTool.twiki?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/docs/src/site/twiki/DG_CommandLineTool.twiki (original)
+++ oozie/branches/branch-3.3/docs/src/site/twiki/DG_CommandLineTool.twiki Thu Feb 14 00:46:31 2013
@@ -87,7 +87,7 @@ usage:
.
oozie sla <OPTIONS> : sla operations (Supported in Oozie-2.0 or later)
-auth <arg> select authentication type [SIMPLE|KERBEROS]
- -len <arg> number of results (default '100')
+ -len <arg> number of results (default '100', max limited by oozie server setting which defaults to '1000')
-offset <arg> start offset (default '0')
-oozie <arg> Oozie URL
.
@@ -699,35 +699,96 @@ It performs an XML Schema validation on
---++ SLA Operations
----+++ Getting the records of SLA events
+---+++ Getting a list of SLA events
* This feature is only supported in Oozie 2.0 or later.
Example:
<verbatim>
-$ oozie sla -oozie http://localhost:11000/oozie -len 1
+$ oozie sla -oozie http://localhost:11000/oozie -len 3
.
<sla-message>
- <event>
- <sequence-id>1091</sequence-id>
+ <event>
+ <sequence-id>1</sequence-id>
+ <registration>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <app-type>COORDINATOR_ACTION</app-type>
+ <app-name>aggregator-sla-app</app-name>
+ <user>joe</user>
+ <group />
+ <parent-sla-id>null</parent-sla-id>
+ <expected-start>2013-01-30T23:00Z</expected-start>
+ <expected-end>2013-01-30T23:30Z</expected-end>
+ <status-timestamp>2013-02-08T18:51Z</status-timestamp>
+ <notification-msg>Notifying User for 2013-01-30T23:00Z nominal time</notification-msg>
+ <alert-contact>www@yahoo.com</alert-contact>
+ <dev-contact>abc@yahoo.com</dev-contact>
+ <qa-contact>abc@yahoo.com</qa-contact>
+ <se-contact>abc@yahoo.com</se-contact>
+ <alert-percentage>80</alert-percentage>
+ <alert-frequency>LAST_HOUR</alert-frequency>
+ <upstream-apps />
+ <job-status>CREATED</job-status>
+ <job-data />
+ </registration>
+ </event>
+ <event>
+ <sequence-id>2</sequence-id>
<status>
- <sla-id>0000004-100521225917912-oozie-wrkf-C@1</sla-id>
- <status-timestamp>2010-05-22T00:41Z</status-timestamp>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <status-timestamp>2013-01-30T23:05Z</status-timestamp>
<job-status>STARTED</job-status>
<job-data />
</status>
</event>
- <last-sequence-id>1091</last-sequence-id>
+ <event>
+ <sequence-id>3</sequence-id>
+ <status>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <status-timestamp>2013-01-30T23:30Z</status-timestamp>
+ <job-status>SUCCEEDED</job-status>
+ <job-data />
+ </status>
+ </event>
+ <last-sequence-id>3</last-sequence-id>
</sla-message>
</verbatim>
-The =offset= and =len= option specified the offset and number of sla events to display, default values are =1= and =100=
-respectively.
+The =offset= and =len= option specified the offset and number of sla events to display, default values are =1= and =100= respectively.
+
+The =offset= corresponds to sequence ID of an event.
+
+The max value of =len= limited by oozie server setting which defaults to '1000'. To get more than =1000= events, it is necessary to iterate based on the number of records you want.
The return message is XML format that can be easily consumed by SLA users.
+
+---+++ Getting the SLA event with particular sequenceID
+
+* This feature is only supported in Oozie 2.0 or later.
+
+Example: Get the SLA event with sequenceID = 3 (Note that offset corresponds to sequence ID)
+
+<verbatim>
+$ oozie sla -oozie http://localhost:11000/oozie -offset 2 -len 1
+.
+<sla-message>
+ <event>
+ <sequence-id>3</sequence-id>
+ <status>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <status-timestamp>2013-01-30T23:05Z</status-timestamp>
+ <job-status>SUCCEEDED</job-status>
+ <job-data />
+ </status>
+ </event>
+ <last-sequence-id>3</last-sequence-id>
+</sla-message>
+
+</verbatim>
+
---++ Pig Operations
---+++ Submitting a pig job through HTTP
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1446002&r1=1446001&r2=1446002&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Thu Feb 14 00:46:31 2013
@@ -1,5 +1,18 @@
+-- Oozie 3.4.0 release (trunk - unreleased)
+
+OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki via rkanter)
+OOZIE-1015 HadoopAccessorService jobtracker validation should not have hardcoded conf key (mona)
+OOZIE-1078 Help -> Documentation and Help -> Online Help should link to oozie.apache.org/ (jun via mohammad)
+OOZIE-669 Deprecate oozie-start.sh, oozie-stop.sh & oozie-run.sh scripts (rkanter via tucu)
+OOZIE-809 MySQL TEXT columns should be MEDIUMTEXT (rkanter via tucu)
+OOZIE-979 bump up trunk version to 3.4.0-SNAPSHOT (tucu)
+OOZIE-1007 Add license headers to all files don't have them (egashira via tucu)
+OOZIE-976 add workflowgenerator into distro tarball (egashira via tucu)
+OOZIE-944 Implement Workflow Generator UI Tool (egashira via virag)
+
-- Oozie 3.3.2 (unreleased)
+OOZIE-1187 reduce memory usage of SLA query (invoked by CLI command) to avoid OOM (egashira via virag)
OOZIE-1221 mvn install failing in Oozie-Mini module (jaoki via virag)
OOZIE-1170 Update minitest module (MiniOozie) to use main pom as its parent (jaoki via rkanter)
OOZIE-1188 Typo in documentation for using login server example (rkanter)