You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2013/02/09 04:13:27 UTC
svn commit: r1444321 - in /oozie/branches/hcat-intre: ./
client/src/main/java/org/apache/oozie/cli/
core/src/main/java/org/apache/oozie/
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/main/java/org/apache/oozie/client/rest/ core/src/main/j...
Author: virag
Date: Sat Feb 9 03:13:27 2013
New Revision: 1444321
URL: http://svn.apache.org/r1444321
Log:
Sync hcat-intre with trunk
Modified:
oozie/branches/hcat-intre/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/SLAEventBean.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/store/SLAStore.java
oozie/branches/hcat-intre/core/src/main/resources/oozie-default.xml
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
oozie/branches/hcat-intre/docs/src/site/twiki/DG_CommandLineTool.twiki
oozie/branches/hcat-intre/hadooplibs/hadoop-2/pom.xml
oozie/branches/hcat-intre/release-log.txt
Modified: oozie/branches/hcat-intre/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/client/src/main/java/org/apache/oozie/cli/OozieCLI.java (original)
+++ oozie/branches/hcat-intre/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Sat Feb 9 03:13:27 2013
@@ -363,7 +363,7 @@ public class OozieCLI {
protected Options createSlaOptions() {
Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
Option start = new Option(OFFSET_OPTION, true, "start offset (default '0')");
- Option len = new Option(LEN_OPTION, true, "number of results (default '100')");
+ Option len = new Option(LEN_OPTION, true, "number of results (default '100', max '1000')");
start.setType(Integer.class);
len.setType(Integer.class);
Options slaOptions = new Options();
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/SLAEventBean.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/SLAEventBean.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/SLAEventBean.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/SLAEventBean.java Sat Feb 9 03:13:27 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -42,9 +42,9 @@ import org.json.simple.JSONObject;
@Entity
@NamedQueries({
+ @NamedQuery(name = "GET_SLA_EVENT_NEWER_SEQ_LIMITED", query = "select OBJECT(w) from SLAEventBean w where w.event_id > :id order by w.event_id"),
+ @NamedQuery(name = "GET_SLA_EVENTS", query = "select OBJECT(w) from SLAEventBean w") })
- @NamedQuery(name = "GET_SLA_EVENT_NEWER_SEQ_LIMITED", query = "select OBJECT(w) from SLAEventBean w where w.event_id > :id order by w.event_id"),
- @NamedQuery(name = "GET_SLA_EVENTS", query = "select OBJECT(w) from SLAEventBean w")})
public class SLAEventBean extends JsonSLAEvent implements Writable {
@Basic
@@ -180,8 +180,7 @@ public class SLAEventBean extends JsonSL
@Override
public String toString() {
- return MessageFormat.format("Event id[{0}] status[{1}]", getEvent_id(),
- getJobStatus());
+ return MessageFormat.format("Event id[{0}] status[{1}]", getEvent_id(), getJobStatus());
}
/**
@@ -233,7 +232,6 @@ public class SLAEventBean extends JsonSL
eReg.addContent(createATagElement("sequence-id", String.valueOf(getEvent_id())));
Element e = new Element("registration");
e.addContent(createATagElement("sla-id", getSlaId()));
- //e.addContent(createATagElement("sla-id", String.valueOf(getSlaId())));
e.addContent(createATagElement("app-type", getAppType().toString()));
e.addContent(createATagElement("app-name", getAppName()));
e.addContent(createATagElement("user", getUser()));
@@ -268,10 +266,11 @@ public class SLAEventBean extends JsonSL
eStat.addContent(createATagElement("sequence-id", String.valueOf(getEvent_id())));
Element e = new Element("status");
e.addContent(createATagElement("sla-id", getSlaId()));
- e.addContent(createATagElement("status-timestamp",
- getDateString(getStatusTimestamp())));
+ e.addContent(createATagElement("status-timestamp", getDateString(getStatusTimestamp())));
e.addContent(createATagElement("job-status", getJobStatus().toString()));
e.addContent(createATagElement("job-data", getJobData()));
+ e.addContent(createATagElement("user", getUser()));
+ e.addContent(createATagElement("group", getGroupName()));
eStat.addContent(e);
return eStat;
}
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java Sat Feb 9 03:13:27 2013
@@ -38,6 +38,9 @@ import java.util.Properties;
import java.util.StringTokenizer;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -204,6 +207,18 @@ public class LauncherMapper<K1, V1, K2,
actionConf.set(OOZIE_JOB_ID, jobId);
actionConf.set(OOZIE_ACTION_ID, actionId);
+ if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
+ List<String> purgedEntries = new ArrayList<String>();
+ Collection<String> entries = actionConf.getStringCollection("mapreduce.job.cache.files");
+ for (String entry : entries) {
+ if (entry.contains("#")) {
+ purgedEntries.add(entry);
+ }
+ }
+ actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()]));
+ launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
+ }
+
FileSystem fs =
Services.get().get(HadoopAccessorService.class).createFileSystem(launcherConf.get("user.name"),
actionDir.toUri(), launcherConf);
@@ -401,6 +416,9 @@ public class LauncherMapper<K1, V1, K2,
}
else {
String mainClass = getJobConf().get(CONF_OOZIE_ACTION_MAIN_CLASS);
+ if (getJobConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
+ System.err.println("WARNING, workaround for Hadoop 2.0.2-alpha distributed cached issue (MAPREDUCE-4820) enabled");
+ }
String msgPrefix = "Main class [" + mainClass + "], ";
int errorCode = 0;
Throwable errorCause = null;
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java Sat Feb 9 03:13:27 2013
@@ -174,10 +174,7 @@ public class PigActionExecutor extends J
protected void setActionCompletionData(Context context, FileSystem fs) throws HadoopAccessorException, IOException,
URISyntaxException {
String data = getExternalChildIDs(context, fs);
- if (data != null && !data.isEmpty()) {
- context.setExternalChildIDs(data);
- XLog.getLog(getClass()).info(XLog.STD, "Hadoop Jobs launched : [{0}]", data);
- }
+ context.setExternalChildIDs(data);
}
private String getExternalChildIDs(Context context, FileSystem actionFs) throws IOException,
@@ -186,6 +183,7 @@ public class PigActionExecutor extends J
String externalIDs = null;
if (actionFs.exists(actionOutput)) {
externalIDs = getDataFromPath(actionOutput, actionFs);
+ XLog.getLog(getClass()).info(XLog.STD, "Hadoop Jobs launched : [{0}]", externalIDs);
}
return externalIDs;
}
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/client/rest/JsonSLAEvent.java Sat Feb 9 03:13:27 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -298,7 +298,7 @@ public class JsonSLAEvent implements SLA
// TODO Auto-generated method stub
return null;
}
-
+
@SuppressWarnings("unchecked")
public JSONObject toJSONObject(String timeZoneId) {
return null;
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/SLAEventsXCommand.java Sat Feb 9 03:13:27 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,6 +27,7 @@ import org.apache.oozie.command.Precondi
import org.apache.oozie.command.XCommand;
import org.apache.oozie.executor.jpa.SLAEventsGetForSeqIdJPAExecutor;
import org.apache.oozie.service.JPAService;
+import org.apache.oozie.service.Service;
import org.apache.oozie.service.Services;
/**
@@ -39,10 +40,14 @@ public class SLAEventsXCommand extends X
private int maxNoEvents = 100; // Default
private long lastSeqId = -1;
+ public static final String SLA_DEFAULT_MAXEVENTS = Service.CONF_PREFIX + "sla.default.maxevents";
+
public SLAEventsXCommand(long seqId, int maxNoEvnts) {
super("SLAEventsXCommand", "SLAEventsXCommand", 1);
this.seqId = seqId;
- this.maxNoEvents = maxNoEvnts;
+ int sysMax = Services.get().getConf().getInt(SLA_DEFAULT_MAXEVENTS, 1000);
+ this.maxNoEvents = maxNoEvnts > sysMax ? sysMax : maxNoEvnts;
+
}
/* (non-Javadoc)
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/executor/jpa/SLAEventsGetForSeqIdJPAExecutor.java Sat Feb 9 03:13:27 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,10 +18,8 @@
package org.apache.oozie.executor.jpa;
import java.util.List;
-
import javax.persistence.EntityManager;
import javax.persistence.Query;
-
import org.apache.oozie.ErrorCode;
import org.apache.oozie.SLAEventBean;
@@ -64,5 +62,4 @@ public class SLAEventsGetForSeqIdJPAExec
}
return seBeans;
}
-
}
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java Sat Feb 9 03:13:27 2013
@@ -24,6 +24,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.security.token.Token;
@@ -71,6 +73,9 @@ public class HadoopAccessorService imple
private static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
/** The Kerberos principal for the resource manager.*/
private static final String RM_PRINCIPAL = "yarn.resourcemanager.principal";
+ private static final String HADOOP_JOB_TRACKER = "mapred.job.tracker";
+ private static final String HADOOP_JOB_TRACKER_2 = "mapreduce.jobtracker.address";
+ private static final String HADOOP_YARN_RM = "yarn.resourcemanager.address";
private static final Map<String, Text> mrTokenRenewers = new HashMap<String, Text>();
private Set<String> jobTrackerWhitelist = new HashSet<String>();
@@ -454,7 +459,17 @@ public class HadoopAccessorService imple
}
}
- public static Text getMRDelegationTokenRenewer(JobConf jobConf) {
+ public static Text getMRDelegationTokenRenewer(JobConf jobConf) throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) { // secure cluster
+ return getMRTokenRenewerInternal(jobConf);
+ }
+ else {
+ return MR_TOKEN_ALIAS; //Doesn't matter what we pass as renewer
+ }
+ }
+
+ // Package private for unit test purposes
+ static Text getMRTokenRenewerInternal(JobConf jobConf) throws IOException {
// Getting renewer correctly for JT principal also though JT in hadoop 1.x does not have
// support for renewing/cancelling tokens
String servicePrincipal = jobConf.get(RM_PRINCIPAL, jobConf.get(JT_PRINCIPAL));
@@ -462,9 +477,15 @@ public class HadoopAccessorService imple
if (servicePrincipal != null) { // secure cluster
renewer = mrTokenRenewers.get(servicePrincipal);
if (renewer == null) {
- // Remove host and domain
- renewer = new Text(servicePrincipal.split("[/@]")[0]);
- LOG.info("Delegation Token Renewer for " + servicePrincipal + " is " + renewer);
+ // Mimic org.apache.hadoop.mapred.Master.getMasterPrincipal()
+ String target = jobConf.get(HADOOP_YARN_RM, jobConf.get(HADOOP_JOB_TRACKER_2));
+ if (target == null) {
+ target = jobConf.get(HADOOP_JOB_TRACKER);
+ }
+ String addr = NetUtils.createSocketAddr(target).getHostName();
+ renewer = new Text(SecurityUtil.getServerPrincipal(servicePrincipal, addr));
+ LOG.info("Delegation Token Renewer details: Principal=" + servicePrincipal + ",Target=" + target
+ + ",Renewer=" + renewer);
mrTokenRenewers.put(servicePrincipal, renewer);
}
}
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/store/SLAStore.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/store/SLAStore.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/store/SLAStore.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/store/SLAStore.java Sat Feb 9 03:13:27 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,7 +27,11 @@ import javax.persistence.Query;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.SLAEventBean;
+import org.apache.oozie.command.CommandException;
+import org.apache.oozie.executor.jpa.JPAExecutorException;
+import org.apache.oozie.executor.jpa.SLAEventsGetForSeqIdJPAExecutor;
import org.apache.oozie.service.InstrumentationService;
+import org.apache.oozie.service.JPAService;
import org.apache.oozie.service.Services;
import org.apache.oozie.util.Instrumentation;
import org.apache.oozie.util.ParamChecker;
@@ -66,7 +70,8 @@ public class SLAStore extends Store {
}
/**
- * Get a list of SLA Events newer than a specific sequence with limit clause.
+ * Get a list of SLA Events newer than a specific sequence with limit
+ * clause.
*
* @param seqId sequence id
* @return List of SLA Events
@@ -80,24 +85,34 @@ public class SLAStore extends Store {
lastSeqId[0] = seqId;
List<SLAEventBean> seBeans = (List<SLAEventBean>) doOperation("getSLAEventListNewerSeqLimited",
- new Callable<List<SLAEventBean>>() {
+ new Callable<List<SLAEventBean>>() {
- public List<SLAEventBean> call() throws StoreException {
+ public List<SLAEventBean> call() throws StoreException, JPAExecutorException {
- List<SLAEventBean> seBeans;
- try {
- Query q = entityManager.createNamedQuery("GET_SLA_EVENT_NEWER_SEQ_LIMITED");
- q.setParameter("id", seqId);
- // q.setFirstResult(0);
- q.setMaxResults(limitLen);
- seBeans = q.getResultList();
- }
- catch (IllegalStateException e) {
- throw new StoreException(ErrorCode.E0601, e.getMessage(), e);
- }
- return seBeans;
- }
- });
+ List<SLAEventBean> seBeans;
+ try {
+
+ JPAService jpaService = Services.get().get(JPAService.class);
+ List<SLAEventBean> slaEventList = null;
+ long lastSeqId[] = new long[1];
+ if (jpaService != null) {
+ seBeans = jpaService.execute(new SLAEventsGetForSeqIdJPAExecutor(seqId, limitLen,
+ lastSeqId));
+ }
+ else {
+ throw new StoreException(ErrorCode.E0610);
+ }
+
+ }
+ catch (IllegalStateException e) {
+ throw new StoreException(ErrorCode.E0601, e.getMessage(), e);
+ }
+ catch (JPAExecutorException e) {
+ throw new JPAExecutorException(ErrorCode.E0610, e.getMessage(), e);
+ }
+ return seBeans;
+ }
+ });
List<SLAEventBean> eventList = new ArrayList<SLAEventBean>();
for (SLAEventBean j : seBeans) {
lastSeqId[0] = Math.max(lastSeqId[0], j.getEvent_id());
Modified: oozie/branches/hcat-intre/core/src/main/resources/oozie-default.xml
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/resources/oozie-default.xml?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/resources/oozie-default.xml (original)
+++ oozie/branches/hcat-intre/core/src/main/resources/oozie-default.xml Sat Feb 9 03:13:27 2013
@@ -1768,4 +1768,18 @@
</property>
+ <!-- Enable Distributed Cache workaround for Hadoop 2.0.2-alpha (MAPREDUCE-4820) -->
+ <property>
+ <name>oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache</name>
+ <value>false</value>
+ <description>
+ Due to a bug in Hadoop 2.0.2-alpha, MAPREDUCE-4820, launcher jobs fail to set
+ the distributed cache for the action job because the local JARs are implicitly
+ included triggering a duplicate check.
+ This flag removes the distributed cache files for the action as they'll be
+ included from the local JARs of the JobClient (MRApps) submitting the action
+ job from the launcher.
+ </description>
+ </property>
+
</configuration>
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java Sat Feb 9 03:13:27 2013
@@ -294,4 +294,34 @@ public class TestLauncher extends XFsTes
lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, prepareBlock);
assertTrue(jobConf.get("oozie.action.prepare.xml").equals(prepareBlock));
}
+
+ // Test to ensure that the property value "oozie.action.prepare.xml" in the configuration of the job is properly set
+ // when there is prepare block in workflow XML
+ public void testSetupLauncherInfoHadoop2_0_2_alphaWorkaround() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+
+ // Setting up the job configuration
+ JobConf jobConf = Services.get().get(HadoopAccessorService.class).
+ createJobConf(new URI(getNameNodeUri()).getAuthority());
+ jobConf.set("user.name", getTestUser());
+ jobConf.set("fs.default.name", getNameNodeUri());
+
+ LauncherMapper lm = new LauncherMapper();
+ Configuration actionConf = new XConfiguration();
+ actionConf.set("mapreduce.job.cache.files", "a.jar,aa.jar#aa.jar");
+ lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, "");
+ assertFalse(jobConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false));
+ assertEquals("a.jar,aa.jar#aa.jar", actionConf.get("mapreduce.job.cache.files"));
+
+ Services.get().getConf().setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
+ lm = new LauncherMapper();
+ actionConf = new XConfiguration();
+ actionConf.set("mapreduce.job.cache.files", "a.jar,aa.jar#aa.jar");
+ lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, "");
+ assertTrue(jobConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false));
+ assertEquals("aa.jar#aa.jar", actionConf.get("mapreduce.job.cache.files"));
+ }
+
}
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetForSeqIdJPAExecutor.java Sat Feb 9 03:13:27 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,6 +21,7 @@ import java.util.Date;
import java.util.List;
import org.apache.oozie.SLAEventBean;
+import org.apache.oozie.client.SLAEvent;
import org.apache.oozie.client.SLAEvent.Status;
import org.apache.oozie.service.JPAService;
import org.apache.oozie.service.Services;
@@ -44,14 +45,15 @@ public class TestSLAEventsGetForSeqIdJPA
}
public void testSLAEventsGetForSeqId() throws Exception {
- final String wfId = "0000000-" + new Date().getTime() + "-TestSLAEventsGetForSeqIdJPAExecutor-W";
- addRecordToSLAEventTable(wfId, Status.CREATED);
- addRecordToSLAEventTable(wfId, Status.STARTED);
- addRecordToSLAEventTable(wfId, Status.SUCCEEDED);
- _testGetSLAEventsForSeqId(wfId);
+ Date current = new Date();
+ final String wfId = "0000000-" + current.getTime() + "-TestSLAEventsGetForSeqIdJPAExecutor-W";
+ addRecordToSLAEventTable(wfId, Status.CREATED, current);
+ addRecordToSLAEventTable(wfId, Status.STARTED, current);
+ addRecordToSLAEventTable(wfId, Status.SUCCEEDED, current);
+ _testGetSLAEventsForSeqId(wfId, current);
}
- private void _testGetSLAEventsForSeqId(String jobId) throws Exception {
+ private void _testGetSLAEventsForSeqId(String jobId, Date current) throws Exception {
JPAService jpaService = Services.get().get(JPAService.class);
assertNotNull(jpaService);
@@ -64,6 +66,12 @@ public class TestSLAEventsGetForSeqIdJPA
List<SLAEventBean> list = jpaService.execute(slaEventsGetCmd);
assertNotNull(list);
assertEquals(2, list.size());
- }
+ SLAEventBean seBean = list.get(0);
+ assertEquals(seBean.getSlaId(), jobId);
+ assertEquals(seBean.getUser(), getTestUser());
+ assertEquals(seBean.getGroupName(), getTestGroup());
+ assertEquals(seBean.getJobStatus(), Status.STARTED);
+ assertEquals(seBean.getStatusTimestamp().getTime(), current.getTime());
+ }
}
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/executor/jpa/TestSLAEventsGetJPAExecutor.java Sat Feb 9 03:13:27 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -44,10 +44,11 @@ public class TestSLAEventsGetJPAExecutor
}
public void testSLAEventsGetForSeqId() throws Exception {
- final String wfId = "0000000-" + new Date().getTime() + "-TestSLAEventsGetJPAExecutor-W";
- addRecordToSLAEventTable(wfId, Status.CREATED);
- addRecordToSLAEventTable(wfId, Status.STARTED);
- addRecordToSLAEventTable(wfId, Status.SUCCEEDED);
+ Date current = new Date();
+ final String wfId = "0000000-" + current.getTime() + "-TestSLAEventsGetJPAExecutor-W";
+ addRecordToSLAEventTable(wfId, Status.CREATED, current);
+ addRecordToSLAEventTable(wfId, Status.STARTED, current);
+ addRecordToSLAEventTable(wfId, Status.SUCCEEDED, current);
_testGetSLAEventsForSeqId(wfId);
}
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java Sat Feb 9 03:13:27 2013
@@ -116,12 +116,21 @@ public class TestHadoopAccessorService e
public void testGetMRDelegationTokenRenewer() throws Exception {
JobConf jobConf = new JobConf();
- assertEquals(new Text("oozie mr token"), HadoopAccessorService.getMRDelegationTokenRenewer(jobConf));
- jobConf.set("mapreduce.jobtracker.kerberos.principal", "mapred/host.domain.com@KDC.DOMAIN.COM");
- assertEquals(new Text("mapred"), HadoopAccessorService.getMRDelegationTokenRenewer(jobConf));
+ assertEquals(new Text("oozie mr token"), HadoopAccessorService.getMRTokenRenewerInternal(jobConf));
+ jobConf.set("mapred.job.tracker", "localhost:50300");
+ jobConf.set("mapreduce.jobtracker.kerberos.principal", "mapred/_HOST@KDC.DOMAIN.COM");
+ assertEquals(new Text("mapred/localhost@KDC.DOMAIN.COM"),
+ HadoopAccessorService.getMRTokenRenewerInternal(jobConf));
jobConf = new JobConf();
- jobConf.set("yarn.resourcemanager.principal", "rm/host.domain.com@KDC.DOMAIN.COM");
- assertEquals(new Text("rm"), HadoopAccessorService.getMRDelegationTokenRenewer(jobConf));
+ jobConf.set("mapreduce.jobtracker.address", "127.0.0.1:50300");
+ jobConf.set("mapreduce.jobtracker.kerberos.principal", "mapred/_HOST@KDC.DOMAIN.COM");
+ assertEquals(new Text("mapred/localhost@KDC.DOMAIN.COM"),
+ HadoopAccessorService.getMRTokenRenewerInternal(jobConf));
+ jobConf = new JobConf();
+ jobConf.set("yarn.resourcemanager.address", "localhost:8032");
+ jobConf.set("yarn.resourcemanager.principal", "rm/server.com@KDC.DOMAIN.COM");
+ assertEquals(new Text("rm/server.com@KDC.DOMAIN.COM"),
+ HadoopAccessorService.getMRTokenRenewerInternal(jobConf));
}
}
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XDataTestCase.java?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XDataTestCase.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XDataTestCase.java Sat Feb 9 03:13:27 2013
@@ -82,9 +82,9 @@ import org.jdom.JDOMException;
public abstract class XDataTestCase extends XHCatTestCase {
- protected static String slaXml = " <sla:info xmlns:sla='uri:oozie:sla:0.1'>" + " <sla:app-name>test-app</sla:app-name>"
- + " <sla:nominal-time>2009-03-06T10:00Z</sla:nominal-time>" + " <sla:should-start>5</sla:should-start>"
- + " <sla:should-end>120</sla:should-end>"
+ protected static String slaXml = " <sla:info xmlns:sla='uri:oozie:sla:0.1'>"
+ + " <sla:app-name>test-app</sla:app-name>" + " <sla:nominal-time>2009-03-06T10:00Z</sla:nominal-time>"
+ + " <sla:should-start>5</sla:should-start>" + " <sla:should-end>120</sla:should-end>"
+ " <sla:notification-msg>Notifying User for nominal time : 2009-03-06T10:00Z </sla:notification-msg>"
+ " <sla:alert-contact>abc@example.com</sla:alert-contact>"
+ " <sla:dev-contact>abc@example.com</sla:dev-contact>"
@@ -103,7 +103,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws Exception
*/
- protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, boolean pending, boolean doneMatd) throws Exception {
+ protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, boolean pending,
+ boolean doneMatd) throws Exception {
CoordinatorJobBean coordJob = createCoordJob(status, pending, doneMatd);
try {
@@ -166,8 +167,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws Exception
*/
- protected CoordinatorJobBean addRecordToCoordJobTable(String testFileName, CoordinatorJob.Status status, Date start, Date end,
- boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
+ protected CoordinatorJobBean addRecordToCoordJobTable(String testFileName, CoordinatorJob.Status status,
+ Date start, Date end, boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
CoordinatorJobBean coordJob = createCoordJob(testFileName, status, start, end, pending, doneMatd, lastActionNum);
try {
@@ -221,7 +222,6 @@ public abstract class XDataTestCase exte
return coordJob;
}
-
/**
* Add coordinator job bean with bundle id info.
*
@@ -236,10 +236,12 @@ public abstract class XDataTestCase exte
* @return coordinator job bean
* @throws Exception
*/
- protected CoordinatorJobBean addRecordToCoordJobTableWithBundle(String bundleId, String coordId, CoordinatorJob.Status status, Date start, Date end, boolean pending, boolean doneMatd, int lastActionNumber) throws Exception {
+ protected CoordinatorJobBean addRecordToCoordJobTableWithBundle(String bundleId, String coordId,
+ CoordinatorJob.Status status, Date start, Date end, boolean pending, boolean doneMatd, int lastActionNumber)
+ throws Exception {
CoordinatorJobBean coordJob = createCoordJob(status, start, end, pending, doneMatd, 0);
coordJob.setBundleId(bundleId);
- //coord id and coord name are the same
+ // coord id and coord name are the same
coordJob.setId(coordId);
coordJob.setAppName(coordId);
coordJob.setLastActionNumber(lastActionNumber);
@@ -258,8 +260,6 @@ public abstract class XDataTestCase exte
return coordJob;
}
-
-
/**
* Create coord job bean
*
@@ -269,7 +269,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws IOException
*/
- protected CoordinatorJobBean createCoordJob(CoordinatorJob.Status status, boolean pending, boolean doneMatd) throws Exception {
+ protected CoordinatorJobBean createCoordJob(CoordinatorJob.Status status, boolean pending, boolean doneMatd)
+ throws Exception {
Path appPath = new Path(getFsTestCaseDir(), "coord");
String appXml = writeCoordXml(appPath);
@@ -377,8 +378,8 @@ public abstract class XDataTestCase exte
* @return coord job bean
* @throws IOException
*/
- protected CoordinatorJobBean createCoordJob(String testFileName, CoordinatorJob.Status status, Date start, Date end, boolean pending,
- boolean doneMatd, int lastActionNum) throws Exception {
+ protected CoordinatorJobBean createCoordJob(String testFileName, CoordinatorJob.Status status, Date start,
+ Date end, boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
Path appPath = new Path(getFsTestCaseDir(), "coord");
String appXml = writeCoordXml(appPath, testFileName);
@@ -420,7 +421,6 @@ public abstract class XDataTestCase exte
return coordJob;
}
-
/**
* Write coordinator xml
*
@@ -463,7 +463,6 @@ public abstract class XDataTestCase exte
return appXml;
}
-
/**
* Write coordinator xml
*
@@ -521,7 +520,8 @@ public abstract class XDataTestCase exte
* @throws Exception thrown if unable to create coord action bean
*/
protected CoordinatorActionBean addRecordToCoordActionTable(String jobId, int actionNum,
- CoordinatorAction.Status status, String resourceXmlName, String wfId, String wfStatus, int pending) throws Exception {
+ CoordinatorAction.Status status, String resourceXmlName, String wfId, String wfStatus, int pending)
+ throws Exception {
CoordinatorActionBean action = createCoordAction(jobId, actionNum, status, resourceXmlName, pending);
action.setExternalId(wfId);
action.setExternalStatus(wfStatus);
@@ -543,6 +543,7 @@ public abstract class XDataTestCase exte
String resourceXmlName, int pending) throws Exception {
return createCoordAction(jobId, actionNum, status, resourceXmlName, pending, "Z");
}
+
/**
* Create coord action bean
*
@@ -597,9 +598,9 @@ public abstract class XDataTestCase exte
*/
protected WorkflowJobBean addRecordToWfJobTable(WorkflowJob.Status jobStatus, WorkflowInstance.Status instanceStatus)
throws Exception {
- WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>",
- new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).
- addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
+ WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", new StartNodeDef(
+ LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end",
+ LiteWorkflowStoreService.LiteControlNodeHandler.class));
Configuration conf = new Configuration();
Path appUri = new Path(getAppPath(), "workflow.xml");
conf.set(OozieClient.APP_PATH, appUri.toString());
@@ -666,14 +667,14 @@ public abstract class XDataTestCase exte
* @param status sla status
* @throws Exception thrown if unable to create sla bean
*/
- protected void addRecordToSLAEventTable(String slaId, SLAEvent.Status status) throws Exception {
+ protected void addRecordToSLAEventTable(String slaId, SLAEvent.Status status, Date today) throws Exception {
SLAEventBean sla = new SLAEventBean();
sla.setSlaId(slaId);
sla.setAppName("app-name");
- sla.setParentClientId("parent-child-id");
+ sla.setParentClientId("parent-client-id");
sla.setParentSlaId("parent-sla-id");
- sla.setExpectedStart(new Date());
- sla.setExpectedEnd(new Date());
+ sla.setExpectedStart(today);
+ sla.setExpectedEnd(today);
sla.setNotificationMsg("notification-msg");
sla.setAlertContact("alert-contact");
sla.setDevContact("dev-contact");
@@ -686,7 +687,7 @@ public abstract class XDataTestCase exte
sla.setUser(getTestUser());
sla.setGroupName(getTestGroup());
sla.setJobStatus(status);
- sla.setStatusTimestamp(new Date());
+ sla.setStatusTimestamp(today);
try {
JPAService jpaService = Services.get().get(JPAService.class);
@@ -919,7 +920,6 @@ public abstract class XDataTestCase exte
jobConf.set("nameNode", getNameNodeUri());
jobConf.set("wfAppPath", wfAppPath.toString());
-
String content = "<workflow-app xmlns='uri:oozie:workflow:0.1' xmlns:sla='uri:oozie:sla:0.1' name='no-op-wf'>";
content += "<start to='end' />";
content += "<end name='end' /></workflow-app>";
@@ -1056,8 +1056,6 @@ public abstract class XDataTestCase exte
.replaceAll("#app_path1", coordPath1.toString() + File.separator + "coordinator.xml");
bundleAppXml = bundleAppXml
.replaceAll("#app_path2", coordPath2.toString() + File.separator + "coordinator.xml");
- // bundleAppXml = bundleAppXml.replaceAll("#app_path1", coordPath1.toString());
- // bundleAppXml = bundleAppXml.replaceAll("#app_path2", coordPath2.toString());
writeToFile(bundleAppXml, bundleAppPath, "bundle.xml");
@@ -1084,7 +1082,8 @@ public abstract class XDataTestCase exte
bundle.setOrigJobXml(bundleAppXml);
if (pending) {
bundle.setPending();
- } else {
+ }
+ else {
bundle.resetPending();
}
bundle.setStatus(jobStatus);
@@ -1122,7 +1121,6 @@ public abstract class XDataTestCase exte
conf.set("jobTracker", getJobTrackerUri());
conf.set("nameNode", getNameNodeUri());
-
BundleJobBean bundle = new BundleJobBean();
bundle.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE));
bundle.setAppName("BUNDLE-TEST");
@@ -1155,10 +1153,11 @@ public abstract class XDataTestCase exte
/**
* Inserts a record to coord action table
+ *
* @param action the record to be inserted
* @throws Exception
*/
- protected void insertRecordCoordAction(CoordinatorActionBean action) throws Exception {
+ protected void insertRecordCoordAction(CoordinatorActionBean action) throws Exception {
try {
JPAService jpaService = Services.get().get(JPAService.class);
assertNotNull(jpaService);
@@ -1172,19 +1171,19 @@ public abstract class XDataTestCase exte
}
}
- // Exclude some of the services classes from loading so they dont interfere
- // while the test case is running
- protected void setClassesToBeExcluded(Configuration conf, String[] excludedServices) {
- String classes = conf.get(Services.CONF_SERVICE_CLASSES);
- StringBuilder builder = new StringBuilder(classes);
- for (String s : excludedServices) {
- int index = builder.indexOf(s);
- if (index != -1) {
- builder.replace(index, index + s.length() + 1, "");
- }
- }
- conf.set(Services.CONF_SERVICE_CLASSES, new String(builder));
- }
+ // Exclude some of the services classes from loading so they dont interfere
+ // while the test case is running
+ protected void setClassesToBeExcluded(Configuration conf, String[] excludedServices) {
+ String classes = conf.get(Services.CONF_SERVICE_CLASSES);
+ StringBuilder builder = new StringBuilder(classes);
+ for (String s : excludedServices) {
+ int index = builder.indexOf(s);
+ if (index != -1) {
+ builder.replace(index, index + s.length() + 1, "");
+ }
+ }
+ conf.set(Services.CONF_SERVICE_CLASSES, new String(builder));
+ }
/**
* Add a particular service class to be run in addition to default ones
@@ -1260,6 +1259,7 @@ public abstract class XDataTestCase exte
/**
* Add a month to the current time
+ *
* @param incrementMonth
* @return
*/
Modified: oozie/branches/hcat-intre/docs/src/site/twiki/DG_CommandLineTool.twiki
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/docs/src/site/twiki/DG_CommandLineTool.twiki?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/docs/src/site/twiki/DG_CommandLineTool.twiki (original)
+++ oozie/branches/hcat-intre/docs/src/site/twiki/DG_CommandLineTool.twiki Sat Feb 9 03:13:27 2013
@@ -87,7 +87,7 @@ usage:
.
oozie sla <OPTIONS> : sla operations (Supported in Oozie-2.0 or later)
-auth <arg> select authentication type [SIMPLE|KERBEROS]
- -len <arg> number of results (default '100')
+ -len <arg> number of results (default '100', max limited by oozie server setting which defaults to '1000')
-offset <arg> start offset (default '0')
-oozie <arg> Oozie URL
.
@@ -699,35 +699,96 @@ It performs an XML Schema validation on
---++ SLA Operations
----+++ Getting the records of SLA events
+---+++ Getting a list of SLA events
* This feature is only supported in Oozie 2.0 or later.
Example:
<verbatim>
-$ oozie sla -oozie http://localhost:11000/oozie -len 1
+$ oozie sla -oozie http://localhost:11000/oozie -len 3
.
<sla-message>
- <event>
- <sequence-id>1091</sequence-id>
+ <event>
+ <sequence-id>1</sequence-id>
+ <registration>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <app-type>COORDINATOR_ACTION</app-type>
+ <app-name>aggregator-sla-app</app-name>
+ <user>joe</user>
+ <group />
+ <parent-sla-id>null</parent-sla-id>
+ <expected-start>2013-01-30T23:00Z</expected-start>
+ <expected-end>2013-01-30T23:30Z</expected-end>
+ <status-timestamp>2013-02-08T18:51Z</status-timestamp>
+ <notification-msg>Notifying User for 2013-01-30T23:00Z nominal time</notification-msg>
+ <alert-contact>www@yahoo.com</alert-contact>
+ <dev-contact>abc@yahoo.com</dev-contact>
+ <qa-contact>abc@yahoo.com</qa-contact>
+ <se-contact>abc@yahoo.com</se-contact>
+ <alert-percentage>80</alert-percentage>
+ <alert-frequency>LAST_HOUR</alert-frequency>
+ <upstream-apps />
+ <job-status>CREATED</job-status>
+ <job-data />
+ </registration>
+ </event>
+ <event>
+ <sequence-id>2</sequence-id>
<status>
- <sla-id>0000004-100521225917912-oozie-wrkf-C@1</sla-id>
- <status-timestamp>2010-05-22T00:41Z</status-timestamp>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <status-timestamp>2013-01-30T23:05Z</status-timestamp>
<job-status>STARTED</job-status>
<job-data />
</status>
</event>
- <last-sequence-id>1091</last-sequence-id>
+ <event>
+ <sequence-id>3</sequence-id>
+ <status>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <status-timestamp>2013-01-30T23:30Z</status-timestamp>
+ <job-status>SUCCEEDED</job-status>
+ <job-data />
+ </status>
+ </event>
+ <last-sequence-id>3</last-sequence-id>
</sla-message>
</verbatim>
-The =offset= and =len= option specified the offset and number of sla events to display, default values are =1= and =100=
-respectively.
+The =offset= and =len= option specified the offset and number of sla events to display, default values are =1= and =100= respectively.
+
+The =offset= corresponds to sequence ID of an event.
+
+The max value of =len= limited by oozie server setting which defaults to '1000'. To get more than =1000= events, it is necessary to iterate based on the number of records you want.
The return message is XML format that can be easily consumed by SLA users.
+
+---+++ Getting the SLA event with particular sequenceID
+
+* This feature is only supported in Oozie 2.0 or later.
+
+Example: Get the SLA event with sequenceID = 3 (Note that offset corresponds to sequence ID)
+
+<verbatim>
+$ oozie sla -oozie http://localhost:11000/oozie -offset 2 -len 1
+.
+<sla-message>
+ <event>
+ <sequence-id>3</sequence-id>
+ <status>
+ <sla-id>0000000-130130150445097-oozie-joe-C@1</sla-id>
+ <status-timestamp>2013-01-30T23:05Z</status-timestamp>
+ <job-status>SUCCEEDED</job-status>
+ <job-data />
+ </status>
+ </event>
+ <last-sequence-id>3</last-sequence-id>
+</sla-message>
+
+</verbatim>
+
---++ Pig Operations
---+++ Submitting a pig job through HTTP
Modified: oozie/branches/hcat-intre/hadooplibs/hadoop-2/pom.xml
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/hadooplibs/hadoop-2/pom.xml?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/hadooplibs/hadoop-2/pom.xml (original)
+++ oozie/branches/hcat-intre/hadooplibs/hadoop-2/pom.xml Sat Feb 9 03:13:27 2013
@@ -27,7 +27,7 @@
</parent>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-hadoop</artifactId>
- <version>2.0.0-alpha.oozie-3.4.0-SNAPSHOT</version>
+ <version>2.0.2-alpha.oozie-3.4.0-SNAPSHOT</version>
<description>Apache Oozie Hadoop ${project.version}</description>
<name>Apache Oozie Hadoop ${project.version}</name>
<packaging>jar</packaging>
@@ -36,7 +36,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>2.0.0-alpha</version>
+ <version>2.0.2-alpha</version>
<scope>compile</scope>
</dependency>
</dependencies>
Modified: oozie/branches/hcat-intre/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1444321&r1=1444320&r2=1444321&view=diff
==============================================================================
--- oozie/branches/hcat-intre/release-log.txt (original)
+++ oozie/branches/hcat-intre/release-log.txt Sat Feb 9 03:13:27 2013
@@ -1,6 +1,7 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1187 reduce memory usage of SLA query (invoked by CLI command) to avoid OOM (egashira via virag)
OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki via rkanter)
OOZIE-809 MySQL TEXT columns should be MEDIUMTEXT (rkanter via tucu)
OOZIE-669 Deprecate oozie-start.sh, oozie-stop.sh & oozie-run.sh scripts (rkanter via tucu)
@@ -125,6 +126,7 @@ OOZIE-654 Provide a way to use 'uber' ja
-- Oozie 3.3.1 (unreleased)
OOZIE-1151 HbaseCredentials doesn't use properties from the credentials module (virag)
+OOZIE-1159 Set the RM token renewer as the full service principal instead of short name (rohini via virag)
OOZIE-1148 Set the renewer correctly for JT/RM delegation tokens (rohini via virag)
OOZIE-1147 HCatCredentialHelper uses the wrong API for getDelegationToken (rohini via virag)
OOZIE-1149 Update 3.3 branch POM's to 3.3.1-SNAPSHOT (virag)
@@ -147,6 +149,7 @@ OOZIE-1100 HFTP coordinator input check
OOZIE-1058 ACL modify-job should not be hardcoded to group name(mona via mohammad)
OOZIE-1052 HadoopAccessorService.createFileSystem throws exception in map-reduce action, failing workflow (egashira via mohammad)
+OOZIE-1089 DistributedCache workaround for Hadoop 2.0.2-alpha (tucu)
OOZIE-993 Hadoop 23 doesn't accept user defined jobtracker (virag)
OOZIE-1005 Tests from OOZIE-994 use wrong condition in waitFor (rkanter via virag)
OOZIE-994 ActionCheckXCommand does not handle failures properly (rkanter via virag)