You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2016/07/26 01:25:08 UTC
[2/3] oozie git commit: OOZIE-2590 OYA: Create basic Oozie Launcher
Application Master (rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 879bfeb..794ad81 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -21,6 +21,7 @@ package org.apache.oozie.action.hadoop;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
@@ -35,18 +36,22 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.action.ActionExecutor;
@@ -55,6 +60,7 @@ import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.LiteWorkflowStoreService;
import org.apache.oozie.service.Services;
@@ -62,7 +68,6 @@ import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.service.UUIDService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
-import org.apache.oozie.util.FSUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
@@ -336,7 +341,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
return new Context(wf, action);
}
- protected RunningJob submitAction(Context context, JavaActionExecutor javaActionExecutor) throws Exception {
+ // TODO: OYA: void
+ protected RunningJob submitAction(Context context, JavaActionExecutor javaActionExecutor) throws ActionExecutorException {
WorkflowAction action = context.getAction();
javaActionExecutor.prepareActionDir(getFileSystem(), context);
@@ -348,21 +354,37 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertNotNull(jobId);
assertNotNull(jobTracker);
assertNotNull(consoleUrl);
-
- JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
- jobConf.set("yarn.resourcemanager.address", jobTracker);
-
- JobClient jobClient =
- Services.get().get(HadoopAccessorService.class).createJobClient(getTestUser(), jobConf);
- final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
- assertNotNull(runningJob);
- return runningJob;
+ return null;
}
- protected RunningJob submitAction(Context context) throws Exception {
+ // TODO: OYA: void
+ protected RunningJob submitAction(Context context) throws ActionExecutorException {
return submitAction(context, new JavaActionExecutor());
}
+ private void waitUntilYarnAppState(String externalId, final YarnApplicationState state)
+ throws HadoopAccessorException, IOException, YarnException {
+ final ApplicationId appId = ConverterUtils.toApplicationId(externalId);
+
+ JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(getJobTrackerUri());
+ // This is needed here because we need a mutable final YarnClient
+ final MutableObject<YarnClient> yarnClientMO = new MutableObject<YarnClient>(null);
+ try {
+ yarnClientMO.setValue(Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), jobConf));
+ waitFor(60 * 1000, new Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return yarnClientMO.getValue().getApplicationReport(appId).getYarnApplicationState().equals(state);
+ }
+ });
+ } finally {
+ if (yarnClientMO.getValue() != null) {
+ yarnClientMO.getValue().close();
+ }
+ }
+ assertTrue(yarnClientMO.getValue().getApplicationReport(appId).getYarnApplicationState().equals(state));
+ }
+
public void testSimpestSleSubmitOK() throws Exception {
String actionXml = "<java>" +
"<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
@@ -370,14 +392,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
"<main-class>" + LauncherMainTester.class.getName() + "</main-class>" +
"</java>";
Context context = createContext(actionXml, null);
- final RunningJob runningJob = submitAction(context);
- waitFor(60 * 1000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
+ submitAction(context);
+ waitUntilYarnAppState(context.getAction().getExternalId(), YarnApplicationState.FINISHED);
ActionExecutor ae = new JavaActionExecutor();
ae.check(context, context.getAction());
assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
@@ -1817,118 +1833,12 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
JavaActionExecutor jae = new JavaActionExecutor("java");
jae.setupLauncherConf(conf, xml, appPath, createContext("<java/>", null));
assertEquals(5, conf.size());
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
assertEquals("v1", conf.get("oozie.launcher.p1"));
assertEquals("v1", conf.get("p1"));
assertEquals("v2b", conf.get("oozie.launcher.p2"));
assertEquals("v2b", conf.get("p2"));
}
- public void testInjectLauncherUseUberMode() throws Exception {
- // default -- should set to true
- JavaActionExecutor jae = new JavaActionExecutor();
- Configuration conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- jae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
-
- // action conf set to true -- should keep at true
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", true);
- jae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
-
- // action conf set to false -- should keep at false
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", false);
- jae.injectLauncherUseUberMode(conf);
- assertEquals("false", conf.get("mapreduce.job.ubertask.enable"));
-
- // disable at oozie-site level for just the "test" action
- ConfigurationService.setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", false);
- JavaActionExecutor tjae = new JavaActionExecutor("test");
-
- // default -- should not set
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- tjae.injectLauncherUseUberMode(conf);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- // default -- should be true
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- jae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
-
- // action conf set to true -- should keep at true
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", true);
- tjae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
- // action conf set to true -- should keep at true
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", true);
- jae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
-
- // action conf set to false -- should keep at false
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", false);
- tjae.injectLauncherUseUberMode(conf);
- assertEquals("false", conf.get("mapreduce.job.ubertask.enable"));
- // action conf set to false -- should keep at false
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", false);
- jae.injectLauncherUseUberMode(conf);
- assertEquals("false", conf.get("mapreduce.job.ubertask.enable"));
-
- // disable at oozie-site level for all actions except for the "test" action
- ConfigurationService.setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", true);
- ConfigurationService.setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", false);
-
- // default -- should be true
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- tjae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
- // default -- should not set
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- jae.injectLauncherUseUberMode(conf);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
-
- // action conf set to true -- should keep at true
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", true);
- tjae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
- // action conf set to true -- should keep at true
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", true);
- jae.injectLauncherUseUberMode(conf);
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
-
- // action conf set to false -- should keep at false
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", false);
- tjae.injectLauncherUseUberMode(conf);
- assertEquals("false", conf.get("mapreduce.job.ubertask.enable"));
- // action conf set to false -- should keep at false
- conf = new Configuration(false);
- assertNull(conf.get("mapreduce.job.ubertask.enable"));
- conf.setBoolean("mapreduce.job.ubertask.enable", false);
- jae.injectLauncherUseUberMode(conf);
- assertEquals("false", conf.get("mapreduce.job.ubertask.enable"));
- }
-
public void testUpdateConfForJavaTmpDir() throws Exception {
//Test UpdateCOnfForJavaTmpDir for launcherConf
@@ -1993,355 +1903,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals("-Xmx2560m -XX:NewRatio=8", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS));
assertEquals("-Xmx1024m -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
}
- public void testUpdateConfForUberMode() throws Exception {
- Element actionXml1 = XmlUtils
- .parseXml("<java>"
- + "<job-tracker>"
- + getJobTrackerUri()
- + "</job-tracker>"
- + "<name-node>"
- + getNameNodeUri()
- + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.mapreduce.map.memory.mb</name><value>2048</value></property>"
- + "<property><name>oozie.launcher.mapred.child.java.opts</name>"
- + "<value>-Xmx2048m -Djava.net.preferIPv4Stack=true</value></property>"
- + "<property><name>oozie.launcher.mapred.child.env</name><value>A=foo</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>");
- JavaActionExecutor ae = new JavaActionExecutor();
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
- WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
- WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
- action.setType(ae.getType());
-
- Context context = new Context(wf, action);
- JobConf launcherConf = new JobConf();
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf);
- // memoryMB (2048 + 512)
- assertEquals("2560", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB));
- // heap size in child.opts (2048 + 512)
- int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true",
- launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true",
- launcherConf.get("mapreduce.map.java.opts"));
-
- assertEquals("-Xmx1024m -Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx2560m -Djava.io.tmpdir=./tmp",
- launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
-
- assertEquals(2560, heapSize);
-
- // env
- assertEquals("A=foo", launcherConf.get(JavaActionExecutor.YARN_AM_ENV));
-
- Element actionXml2 = XmlUtils
- .parseXml("<java>"
- + "<job-tracker>"
- + getJobTrackerUri()
- + "</job-tracker>"
- + "<name-node>"
- + getNameNodeUri()
- + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.resource.mb</name><value>3072</value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.memory.mb</name><value>2048</value></property>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
- + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true </value></property>"
- + "<property><name>oozie.launcher.mapred.child.java.opts</name><value>-Xmx1536m</value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.java.opts</name>"
- + "<value>-Xmx2560m -XX:NewRatio=8</value></property>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name><value>A=foo</value></property>"
- + "<property><name>oozie.launcher.mapred.child.env</name><value>B=bar</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>");
-
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf);
-
- // memoryMB (3072 + 512)
- assertEquals("3584", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB));
-
- // heap size (2560 + 512)
- heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts"));
- assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx2560m -XX:NewRatio=8 " +
- "-Xmx3072m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
- assertEquals(3072, heapSize);
-
- // env (equqls to mapreduce.map.env + am.env)
- assertTrue(launcherConf.get(JavaActionExecutor.YARN_AM_ENV).trim().equals("A=foo,B=bar"));
-
- // Test limit is applied in case of 32 bit
- Element actionXml3 = XmlUtils
- .parseXml("<java>"
- + "<job-tracker>"
- + getJobTrackerUri()
- + "</job-tracker>"
- + "<name-node>"
- + getNameNodeUri()
- + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.resource.mb</name><value>3072</value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.memory.mb</name><value>4000</value></property>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
- + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true</value></property>"
- + "<property><name>oozie.launcher.mapred.child.java.opts</name><value>-Xmx1536m</value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.java.opts</name>"
- + "<value>-Xmx4000m -XX:NewRatio=8</value></property>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name><value>A=foo</value></property>"
- + "<property><name>oozie.launcher.mapred.child.env</name><value>B=bar</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>");
-
- launcherConf = ae.createBaseHadoopConf(context, actionXml3);
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml3, launcherConf);
-
- // memoryMB (limit to 4096)
- assertEquals("4096", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB));
-
- // heap size (limit to 3584)
- heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts"));
- assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx4000m -XX:NewRatio=8 " +
- "-Xmx3584m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
- assertEquals(3584, heapSize);
-
- // env (equqls to mapreduce.map.env + am.env)
- assertEquals("A=foo,B=bar", launcherConf.get(JavaActionExecutor.YARN_AM_ENV));
- }
-
- public void testUpdateConfForUberModeWithEnvDup() throws Exception {
- Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
- + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name>"
- + "<value>JAVA_HOME=/home/blah/java/jdk64/current,A=foo,B=bar</value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.env</name>"
- + "<value>JAVA_HOME=/home/blah/java/jdk64/latest,C=blah</value></property>" + "</configuration>"
- + "<main-class>MAIN-CLASS</main-class>" + "</java>");
- JavaActionExecutor ae = new JavaActionExecutor();
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
- WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
- WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
- action.setType(ae.getType());
-
- Context context = new Context(wf, action);
- JobConf launcherConf = new JobConf();
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf);
-
- // uber mode should be disabled since JAVA_HOME points to different paths in am.evn and map.env
- assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
-
- // testing complicated env setting case
- Element actionXml2 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
- + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property>"
- + "<name>oozie.launcher.yarn.app.mapreduce.am.env</name>"
- + "<value>LD_LIBRARY_PATH=$HADOOP_HOME_1/lib/native/`$JAVA_HOME/bin/java -d32 -version;"
- + "if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value></property>"
- + "<property>" + "<name>oozie.launcher.mapreduce.map.env</name>"
- + "<value>LD_LIBRARY_PATH=$HADOOP_HOME_2/lib/native/`$JAVA_HOME/bin/java -d32 -version;"
- + "if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>");
-
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf);
-
- // uber mode should be disabled since LD_LIBRARY_PATH is different in am.evn and map.env
- assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
-
- Element actionXml3 = XmlUtils
- .parseXml("<java>"
- + "<job-tracker>"
- + getJobTrackerUri()
- + "</job-tracker>"
- + "<name-node>"
- + getNameNodeUri()
- + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name>"
- + "<value>JAVA_HOME=/home/blah/java/jdk64/current,PATH=A,PATH=B</value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.env</name>"
- + "<value>JAVA_HOME=/home/blah/java/jdk64/current,PATH=A</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>");
-
- launcherConf = ae.createBaseHadoopConf(context, actionXml3);
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml3, launcherConf);
-
- // uber mode should be enabled since JAVA_HOME is the same, and PATH doesn't conflict
- assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
-
- // JAVA_HOME, PATH=A duplication is removed
- String a = launcherConf.get(JavaActionExecutor.YARN_AM_ENV);
- assertEquals("JAVA_HOME=/home/blah/java/jdk64/current,PATH=A,PATH=B",
- launcherConf.get(JavaActionExecutor.YARN_AM_ENV));
- }
-
- public void testUpdateConfForUberModeForJavaOpts() throws Exception {
- Element actionXml1 = XmlUtils
- .parseXml("<java>"
- + "<job-tracker>"
- + getJobTrackerUri()
- + "</job-tracker>"
- + "<name-node>"
- + getNameNodeUri()
- + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
- + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true </value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.java.opts</name><value>-Xmx1536m</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>"
- + "<java-opt>-Xmx2048m</java-opt>"
- + "<java-opt>-Dkey1=val1</java-opt>"
- + "<java-opt>-Dkey2=val2</java-opt>"
- + "</java>");
- JavaActionExecutor ae = new JavaActionExecutor();
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
- WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
- WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
- action.setType(ae.getType());
-
- Context context = new Context(wf, action);
- JobConf launcherConf = new JobConf();
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf);
-
- // heap size (2048 + 512)
- int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2",
- launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2",
- launcherConf.get("mapreduce.map.java.opts"));
- assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Xmx2560m " +
- "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
- assertEquals(2560, heapSize);
-
- Element actionXml2 = XmlUtils
- .parseXml("<java>"
- + "<job-tracker>"
- + getJobTrackerUri()
- + "</job-tracker>"
- + "<name-node>"
- + getNameNodeUri()
- + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
- + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true </value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.java.opts</name><value>-Xmx1536m</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>"
- + "<java-opts>-Xmx2048m -Dkey1=val1</java-opts>"
- + "</java>");
-
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf);
-
- // heap size (2048 + 512)
- heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1",
- launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1",
- launcherConf.get("mapreduce.map.java.opts"));
- assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Xmx2560m " +
- "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
- assertEquals(2560, heapSize);
-
- Element actionXml3 = XmlUtils
- .parseXml("<java>"
- + "<job-tracker>"
- + getJobTrackerUri()
- + "</job-tracker>"
- + "<name-node>"
- + getNameNodeUri()
- + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
- + "<value>-Xmx2048m -Djava.net.preferIPv4Stack=true </value></property>"
- + "<property><name>oozie.launcher.mapreduce.map.java.opts</name><value>-Xmx3072m</value></property>"
- + "</configuration>" + "<main-class>MAIN-CLASS</main-class>"
- + "<java-opts>-Xmx1024m -Dkey1=val1</java-opts>"
- + "</java>");
-
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml3, launcherConf);
-
- // heap size (2048 + 512)
- heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1",
- launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1",
- launcherConf.get("mapreduce.map.java.opts"));
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Xmx2560m " +
- "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
- assertEquals(2560, heapSize);
- }
-
- public void testDisableUberForProperties() throws Exception {
- Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
- + "<name-node>" + getNameNodeUri() + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.mapreduce.job.classloader</name>"
- + "<value>true</value></property>"
- + "</configuration>"
- + "<main-class>MAIN-CLASS</main-class>" + "</java>");
- JavaActionExecutor ae = new JavaActionExecutor();
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
- WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
- WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
- action.setType(ae.getType());
-
- Context context = new Context(wf, action);
- JobConf launcherConf = new JobConf();
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf);
-
- // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true
- assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
-
- Element actionXml2 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
- + "<name-node>" + getNameNodeUri() + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.mapreduce.user.classpath.first</name>"
- + "<value>true</value></property>"
- + "</configuration>"
- + "<main-class>MAIN-CLASS</main-class>" + "</java>");
- ae = new JavaActionExecutor();
- protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
- wf = createBaseWorkflow(protoConf, "action");
- action = (WorkflowActionBean) wf.getActions().get(0);
- action.setType(ae.getType());
-
- context = new Context(wf, action);
- launcherConf = new JobConf();
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf);
-
- // uber mode should be disabled since oozie.launcher.mapreduce.user.classpath.first=true
- assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
- }
-
- public void testDisableUberForUserProperties() throws Exception {
- Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
- + "<name-node>" + getNameNodeUri() + "</name-node>"
- + "<configuration>"
- + "<property><name>oozie.launcher.mapreduce.job.ubertask.enable</name>"
- + "<value>false</value></property>"
- + "</configuration>"
- + "<main-class>MAIN-CLASS</main-class>" + "</java>");
- JavaActionExecutor ae = new JavaActionExecutor();
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
- WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
- WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
- action.setType(ae.getType());
-
- Context context = new Context(wf, action);
- JobConf launcherConf = new JobConf();
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf);
- // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true
- assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
- }
public void testUpdateConfForTimeLineServiceEnabled() throws Exception {
Element actionXml = XmlUtils
@@ -2766,12 +2327,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals("AA", conf.get("a"));
assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo"));
assertEquals("action.barbar", conf.get("action.foofoo"));
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
- if (conf.size() == 7) {
- assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address"));
- } else {
- assertEquals(6, conf.size());
- }
+ assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address"));
+ assertEquals(6, conf.size());
conf = new Configuration(false);
Assert.assertEquals(0, conf.size());
@@ -2780,12 +2337,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address"));
assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo"));
assertEquals("action.barbar", conf.get("action.foofoo"));
- assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
- if (conf.size() == 5) {
- assertEquals(getJobTrackerUri(), conf.get("mapreduce.jobtracker.address"));
- } else {
- assertEquals(4, conf.size());
- }
+ assertEquals(getJobTrackerUri(), conf.get("mapreduce.jobtracker.address"));
+ assertEquals(4, conf.size());
}
public void testSetRootLoggerLevel() throws Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
new file mode 100644
index 0000000..ed29299
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.util.IOUtils;
+import org.apache.oozie.util.XConfiguration;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.URI;
+import java.util.Map;
+
+public class TestLauncherAM extends XFsTestCase {
+
+
+ // TODO: OYA: write tests later
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java
new file mode 100644
index 0000000..d0b4d5b
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.oozie.QueryServlet;
+import org.apache.oozie.command.wf.HangServlet;
+import org.apache.oozie.test.EmbeddedServletContainer;
+import org.apache.oozie.test.XTestCase;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import java.net.Proxy;
+import java.util.HashMap;
+import java.util.Map;
+
+// A lot of this adapted from org.apache.hadoop.mapreduce.v2.app.TestJobEndNotifier and org.apache.hadoop.mapred.TestJobEndNotifier
+public class TestLauncherAMCallbackNotifier extends XTestCase {
+
+ public void testConfiguration() throws Exception {
+ Configuration conf = new Configuration(false);
+
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "0");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "10");
+ LauncherAMCallbackNotifier cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(0, cn.numTries);
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(1, cn.numTries);
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "20");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(11, cn.numTries); //11 because number of _retries_ is 10
+
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "1000");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(1000, cn.waitInterval);
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "10000");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(5000, cn.waitInterval);
+ //Test negative numbers are set to default
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "-10");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(5000, cn.waitInterval);
+
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_TIMEOUT, "1000");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(1000, cn.timeout);
+
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "somehost");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(Proxy.Type.DIRECT, cn.proxyToUse.type());
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "somehost:someport");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals(Proxy.Type.DIRECT, cn.proxyToUse.type());
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "somehost:1000");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals("HTTP @ somehost:1000", cn.proxyToUse.toString());
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "socks@somehost:1000");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals("SOCKS @ somehost:1000", cn.proxyToUse.toString());
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "SOCKS@somehost:1000");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals("SOCKS @ somehost:1000", cn.proxyToUse.toString());
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "sfafn@somehost:1000");
+ cn = new LauncherAMCallbackNotifier(conf);
+ assertEquals("HTTP @ somehost:1000", cn.proxyToUse.toString());
+ }
+
+ public void testNotifyRetries() throws InterruptedException {
+ Configuration conf = new Configuration(false);
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "0");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, "http://nonexistent");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "5000");
+
+ LauncherAMCallbackNotifier cnSpy = Mockito.spy(new LauncherAMCallbackNotifier(conf));
+ long start = System.currentTimeMillis();
+ cnSpy.notifyURL(FinalApplicationStatus.SUCCEEDED);
+ long end = System.currentTimeMillis();
+ Mockito.verify(cnSpy, Mockito.times(1)).notifyURLOnce();
+ Assert.assertTrue("Should have taken more than 5 seconds but it only took " + (end - start), end - start >= 5000);
+
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "3");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "3");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "3000");
+
+ cnSpy = Mockito.spy(new LauncherAMCallbackNotifier(conf));
+ start = System.currentTimeMillis();
+ cnSpy.notifyURL(FinalApplicationStatus.SUCCEEDED);
+ end = System.currentTimeMillis();
+ Mockito.verify(cnSpy, Mockito.times(3)).notifyURLOnce();
+ Assert.assertTrue("Should have taken more than 9 seconds but it only took " + (end - start), end - start >= 9000);
+ }
+
+ public void testNotifyTimeout() throws Exception {
+ EmbeddedServletContainer container = null;
+ try {
+ container = new EmbeddedServletContainer("blah");
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(HangServlet.SLEEP_TIME_MS, "1000000");
+ container.addServletEndpoint("/hang/*", HangServlet.class, params);
+ container.start();
+
+ Configuration conf = new Configuration(false);
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "0");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, container.getServletURL("/hang/*"));
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "5000");
+
+ LauncherAMCallbackNotifier cnSpy = Mockito.spy(new LauncherAMCallbackNotifier(conf));
+ long start = System.currentTimeMillis();
+ cnSpy.notifyURL(FinalApplicationStatus.SUCCEEDED);
+ long end = System.currentTimeMillis();
+ Mockito.verify(cnSpy, Mockito.times(1)).notifyURLOnce();
+ Assert.assertTrue("Should have taken more than 5 seconds but it only took " + (end - start), end - start >= 5000);
+ } finally {
+ if (container != null) {
+ container.stop();
+ }
+ }
+ }
+
+ public void testNotify() throws Exception {
+ EmbeddedServletContainer container = null;
+ try {
+ container = new EmbeddedServletContainer("blah");
+ container.addServletEndpoint("/count/*", QueryServlet.class);
+ container.start();
+
+ Configuration conf = new Configuration(false);
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "0");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1");
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, container.getServletURL("/count/?status=$jobStatus"));
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "5000");
+
+ LauncherAMCallbackNotifier cn = new LauncherAMCallbackNotifier(conf);
+ QueryServlet.lastQueryString = null;
+ assertNull(QueryServlet.lastQueryString);
+ cn.notifyURL(FinalApplicationStatus.SUCCEEDED);
+ waitFor(5000, new Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return "status=SUCCEEDED".equals(QueryServlet.lastQueryString);
+ }
+ });
+ assertEquals("status=SUCCEEDED", QueryServlet.lastQueryString);
+ } finally {
+ if (container != null) {
+ container.stop();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
index df9e939..e940d39 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
@@ -24,6 +24,9 @@ import org.apache.oozie.service.Services;
import org.apache.oozie.test.XFsTestCase;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.ParserConfigurationException;
public class TestPrepareActionsDriver extends XFsTestCase {
@@ -40,7 +43,7 @@ public class TestPrepareActionsDriver extends XFsTestCase {
}
// Test to check if prepare action is performed as expected when the prepare XML block is a valid one
- public void testDoOperationsWithValidXML() throws LauncherException, IOException {
+ public void testDoOperationsWithValidXML() throws LauncherException, IOException, ParserConfigurationException, SAXException {
Path actionDir = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Path newDir = new Path(actionDir, "newDir");
@@ -57,7 +60,7 @@ public class TestPrepareActionsDriver extends XFsTestCase {
assertTrue(fs.exists(actionDir));
}
- // Test to check if LauncherException is thrown when the prepare XML block is invalid
+ // Test to check if Exception is thrown when the prepare XML block is invalid
public void testDoOperationsWithInvalidXML() throws LauncherException, IOException {
Path actionDir = getFsTestCaseDir();
FileSystem fs = getFileSystem();
@@ -75,11 +78,9 @@ public class TestPrepareActionsDriver extends XFsTestCase {
LauncherMapperHelper.setupLauncherURIHandlerConf(conf);
PrepareActionsDriver.doOperations(prepareXML, conf);
fail("Expected to catch an exception but did not encounter any");
- } catch (LauncherException le) {
- assertEquals(le.getCause().getClass(), org.xml.sax.SAXParseException.class);
- assertEquals(le.getMessage(), "Content is not allowed in prolog.");
- } catch(Exception ex){
- fail("Expected a LauncherException but received an Exception");
+ } catch (Exception ex) {
+ assertEquals(ex.getClass(), org.xml.sax.SAXParseException.class);
+ assertEquals(ex.getMessage(), "Content is not allowed in prolog.");
}
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
index 6a962a1..9468fad 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
@@ -367,25 +367,4 @@ public class TestShellActionExecutor extends ActionExecutorTestCase {
assertNotNull(runningJob);
return runningJob;
}
-
- public void testShellMainPathInUber() throws Exception {
- Services.get().getConf().setBoolean("oozie.action.shell.launcher.mapreduce.job.ubertask.enable", true);
-
- Element actionXml = XmlUtils.parseXml("<shell>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
- + "<name-node>" + getNameNodeUri() + "</name-node>" + "<exec>script.sh</exec>"
- + "<argument>a=A</argument>" + "<argument>b=B</argument>" + "</shell>");
- ShellActionExecutor ae = new ShellActionExecutor();
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
- WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
- WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
- action.setType(ae.getType());
-
- Context context = new Context(wf, action);
- JobConf launcherConf = new JobConf();
- launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, launcherConf);
- // env
- assertEquals("PATH=.:$PATH", launcherConf.get(JavaActionExecutor.YARN_AM_ENV));
- }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java b/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java
index 3344cf9..d90aeb6 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java
@@ -18,6 +18,8 @@
package org.apache.oozie.command.wf;
+import org.apache.oozie.util.XLog;
+
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
@@ -25,14 +27,27 @@ import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
/**
- * Servlet that 'hangs' for 200 ms. Used by TestNotificationXCommand
+ * Servlet that 'hangs' for some amount of time (200ms) by default.
+ * The time can be configured by setting {@link HangServlet#SLEEP_TIME_MS} as an init parameter for the servlet.
*/
public class HangServlet extends HttpServlet {
+ public static final String SLEEP_TIME_MS = "sleep_time_ms";
+
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
- Thread.sleep(200);
+ long time = 200;
+ String sleeptime = getInitParameter(SLEEP_TIME_MS);
+ if (sleeptime != null) {
+ try {
+ time = Long.parseLong(sleeptime);
+ } catch (NumberFormatException nfe) {
+ XLog.getLog(HangServlet.class).error("Invalid sleep time, using default (200)", nfe);
+ }
+ }
+ XLog.getLog(HangServlet.class).info("Sleeping for " + time + " ms");
+ Thread.sleep(time);
}
catch (Exception ex) {
//NOP
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
index 2153bf1..9b48df5 100644
--- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
@@ -209,9 +209,6 @@ public class TestConfigurationService extends XTestCase {
assertEquals(2048, ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA));
assertEquals("http://localhost:8080/oozie?job=", ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL));
- assertEquals(true, ConfigurationService.getBoolean(JavaActionExecutor.CONF_HADOOP_YARN_UBER_MODE));
- assertEquals(false, ConfigurationService.getBoolean(
- "oozie.action.shell.launcher." + JavaActionExecutor.HADOOP_YARN_UBER_MODE));
assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED));
assertEquals(0, ConfigurationService.getStrings("no.defined").length);
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
index 96faa48..2798719 100644
--- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
@@ -18,7 +18,17 @@
package org.apache.oozie.service;
-import org.apache.oozie.test.XTestCase;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.security.authorize.*;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.oozie.test.XFsTestCase;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.fs.FileSystem;
@@ -34,7 +44,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.util.XConfiguration;
-public class TestHadoopAccessorService extends XTestCase {
+public class TestHadoopAccessorService extends XFsTestCase {
protected void setUp() throws Exception {
super.setUp();
@@ -136,45 +146,89 @@ public class TestHadoopAccessorService extends XTestCase {
*/
assertEquals("100", conf.get("action.testprop"));
assertEquals("1", conf.get("default.testprop"));
+ }
+
+ public void testCreateJobClient() throws Exception {
+ HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
+ JobConf conf = has.createJobConf(getJobTrackerUri());
+
+ JobClient jc = has.createJobClient(getTestUser(), conf);
+ assertNotNull(jc);
+ jc.getAllJobs();
+
+ try {
+ has.createJobClient("invalid-user", conf);
+ fail("Should have thrown exception because not allowed to impersonate 'invalid-user'");
+ }
+ catch (HadoopAccessorException ex) {
+ assertEquals(ErrorCode.E0902, ex.getErrorCode());
+ }
+ JobConf conf2 = new JobConf(false);
+ conf2.set("mapred.job.tracker", getJobTrackerUri());
+ try {
+ has.createJobClient(getTestUser(), conf2);
+ fail("Should have thrown exception because Configuration not created by HadoopAccessorService");
+ }
+ catch (HadoopAccessorException ex) {
+ assertEquals(ErrorCode.E0903, ex.getErrorCode());
+ }
}
- public void testAccessor() throws Exception {
- Services services = Services.get();
- HadoopAccessorService has = services.get(HadoopAccessorService.class);
+ public void testCreateYarnClient() throws Exception {
+ HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
JobConf conf = has.createJobConf(getJobTrackerUri());
- conf.set("mapred.job.tracker", getJobTrackerUri());
- conf.set("fs.default.name", getNameNodeUri());
- URI uri = new URI(getNameNodeUri());
+ YarnClient yc = has.createYarnClient(getTestUser(), conf);
+ assertNotNull(yc);
+ yc.getApplications();
- //valid user
- String user = getTestUser();
- String group = getTestGroup();
+ try {
+ yc = has.createYarnClient("invalid-user", conf);
+ assertNotNull(yc);
+ yc.getApplications();
+ fail("Should have thrown exception because not allowed to impersonate 'invalid-user'");
+ }
+ catch (AuthorizationException ex) {
+ }
- JobClient jc = has.createJobClient(user, conf);
- assertNotNull(jc);
- FileSystem fs = has.createFileSystem(user, new URI(getNameNodeUri()), conf);
- assertNotNull(fs);
- fs = has.createFileSystem(user, uri, conf);
- assertNotNull(fs);
+ JobConf conf2 = new JobConf(false);
+ conf2.set("yarn.resourcemanager.address", getJobTrackerUri());
+ try {
+ has.createYarnClient(getTestUser(), conf2);
+ fail("Should have thrown exception because Configuration not created by HadoopAccessorService");
+ }
+ catch (HadoopAccessorException ex) {
+ assertEquals(ErrorCode.E0903, ex.getErrorCode());
+ }
+ }
- //invalid user
+ public void testCreateFileSystem() throws Exception {
+ HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
+ JobConf conf = has.createJobConf(getJobTrackerUri());
- user = "invalid";
+ FileSystem fs = has.createFileSystem(getTestUser(), new URI(getNameNodeUri()), conf);
+ assertNotNull(fs);
+ fs.exists(new Path(getNameNodeUri(), "/foo"));
try {
- has.createJobClient(user, conf);
- fail();
+ fs = has.createFileSystem("invalid-user", new URI(getNameNodeUri()), conf);
+ assertNotNull(fs);
+ fs.exists(new Path(getNameNodeUri(), "/foo"));
+ fail("Should have thrown exception because not allowed to impersonate 'invalid-user'");
}
- catch (Throwable ex) {
+ catch (RemoteException ex) {
+ assertEquals(AuthorizationException.class.getName(), ex.getClassName());
}
+ JobConf conf2 = new JobConf(false);
+ conf2.set("fs.default.name", getNameNodeUri());
try {
- has.createFileSystem(user, uri, conf);
- fail();
+ has.createFileSystem(getTestUser(), new URI(getNameNodeUri()), conf2);
+ fail("Should have thrown exception because Configuration not created by HadoopAccessorService");
}
- catch (Throwable ex) {
+ catch (HadoopAccessorException ex) {
+ assertEquals(ErrorCode.E0903, ex.getErrorCode());
}
}
@@ -290,4 +344,21 @@ public class TestHadoopAccessorService extends XTestCase {
}
has.destroy();
}
+
+ public void testCreateLocalResourceForConfigurationFile() throws Exception {
+ HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
+ String filename = "foo.xml";
+ Configuration conf = has.createJobConf(getNameNodeUri());
+ conf.set("foo", "bar");
+ LocalResource lRes = has.createLocalResourceForConfigurationFile(filename, getTestUser(), conf, getFileSystem().getUri(),
+ getFsTestCaseDir());
+ assertNotNull(lRes);
+ assertEquals(LocalResourceType.FILE, lRes.getType());
+ assertEquals(LocalResourceVisibility.APPLICATION, lRes.getVisibility());
+ Path resPath = ConverterUtils.getPathFromYarnURL(lRes.getResource());
+ assertEquals(new Path(getFsTestCaseDir(), "foo.xml"), resPath);
+ Configuration conf2 = new Configuration(false);
+ conf2.addResource(getFileSystem().open(resPath));
+ assertEquals("bar", conf2.get("foo"));
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/test/XTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java
index e360369..81a33fd 100644
--- a/core/src/test/java/org/apache/oozie/test/XTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java
@@ -82,6 +82,7 @@ import org.apache.oozie.sla.SLASummaryBean;
import org.apache.oozie.store.StoreException;
import org.apache.oozie.test.MiniHCatServer.RUNMODE;
import org.apache.oozie.test.hive.MiniHS2;
+import org.apache.oozie.util.ClasspathUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.util.XConfiguration;
@@ -877,6 +878,7 @@ public abstract class XTestCase extends TestCase {
private static MiniDFSCluster dfsCluster = null;
private static MiniDFSCluster dfsCluster2 = null;
+ // TODO: OYA: replace with MiniYarnCluster or MiniMRYarnCluster
private static MiniMRCluster mrCluster = null;
private static MiniHCatServer hcatServer = null;
private static MiniHS2 hiveserver2 = null;
@@ -886,6 +888,8 @@ public abstract class XTestCase extends TestCase {
if (System.getProperty("hadoop.log.dir") == null) {
System.setProperty("hadoop.log.dir", testCaseDir);
}
+ // Tell the ClasspathUtils that we're using a mini cluster
+ ClasspathUtils.setUsingMiniYarnCluster(true);
int taskTrackers = 2;
int dataNodes = 2;
String oozieUser = getOozieUser();
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java b/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java
new file mode 100644
index 0000000..3a7215b
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.util;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.test.XTestCase;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestClasspathUtils extends XFsTestCase {
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ // This is normally true, and adds the entirety of the current classpath in ClasspathUtils, which we don't want to test or
+ // worry about here. Temporarily set this back to false so it behaves normally.
+ ClasspathUtils.setUsingMiniYarnCluster(false);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ // Make sure to turn this back on for subsequent tests
+ ClasspathUtils.setUsingMiniYarnCluster(true);
+ super.tearDown();
+ }
+
+ public void testSetupClasspath() throws Exception {
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<String, String>();
+
+ Path p1 = new Path(getFsTestCaseDir(), "foo.xml");
+ getFileSystem().createNewFile(p1);
+ DistributedCache.addFileToClassPath(p1, conf);
+
+ Path p2 = new Path(getFsTestCaseDir(), "foo.txt");
+ getFileSystem().createNewFile(p2);
+ DistributedCache.addFileToClassPath(p2, conf);
+
+ Path p3 = new Path(getFsTestCaseDir(), "foo.zip");
+ getFileSystem().createNewFile(p3);
+ DistributedCache.addArchiveToClassPath(p3, conf);
+
+ ClasspathUtils.setupClasspath(env, conf);
+
+ assertEquals(2, env.size());
+ assertTrue(env.containsKey("CLASSPATH"));
+ String[] paths = env.get("CLASSPATH").split(":");
+ assertEquals(12, paths.length);
+ Arrays.sort(paths);
+ assertEquals("$HADOOP_COMMON_HOME/share/hadoop/common/*", paths[0]);
+ assertEquals("$HADOOP_COMMON_HOME/share/hadoop/common/lib/*", paths[1]);
+ assertEquals("$HADOOP_CONF_DIR", paths[2]);
+ assertEquals("$HADOOP_HDFS_HOME/share/hadoop/hdfs/*", paths[3]);
+ assertEquals("$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*", paths[4]);
+ assertEquals("$HADOOP_YARN_HOME/share/hadoop/yarn/*", paths[5]);
+ assertEquals("$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*", paths[6]);
+ assertEquals("$PWD", paths[7]);
+ assertEquals("$PWD/*", paths[8]);
+ assertEquals("job.jar/classes/", paths[9]);
+ assertEquals("job.jar/job.jar", paths[10]);
+ assertEquals("job.jar/lib/*", paths[11]);
+
+ assertTrue(env.containsKey("$PWD"));
+ paths = env.get("$PWD").split(":");
+ assertEquals(3, paths.length);
+ Arrays.sort(paths);
+ assertEquals("$PWD/foo.txt", paths[0]);
+ assertEquals("$PWD/foo.xml", paths[1]);
+ assertEquals("$PWD/foo.zip", paths[2]);
+ }
+
+ public void testAddMapReduceToClasspath() throws Exception {
+ Configuration conf = new Configuration(false);
+ Map<String, String> env = new HashMap<String, String>();
+
+ ClasspathUtils.addMapReduceToClasspath(env, conf);
+
+ assertEquals(1, env.size());
+ assertTrue(env.containsKey("CLASSPATH"));
+ String[] paths = env.get("CLASSPATH").split(":");
+ assertEquals(2, paths.length);
+ Arrays.sort(paths);
+ assertEquals("$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*", paths[0]);
+ assertEquals("$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*", paths[1]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 3a23cbf..c75911e 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2590 OYA: Create basic Oozie Launcher Application Master (rkanter)
OOZIE-2316 Drop support for Hadoop 1 and 0.23 (asasvari via rkanter)
OOZIE-2503 show ChildJobURLs to spark action (satishsaley via puru)
OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru)
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/distcp/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/distcp/pom.xml b/sharelib/distcp/pom.xml
index c8cc47c..cb01faa 100644
--- a/sharelib/distcp/pom.xml
+++ b/sharelib/distcp/pom.xml
@@ -91,18 +91,6 @@
<outputFile>${project.build.directory}/classpath</outputFile>
</configuration>
</execution>
- <execution>
- <id>create-mrapp-generated-classpath</id>
- <phase>generate-test-resources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <!-- needed to run the unit test for DS to generate the required classpath
- that is required in the env of the launch container in the mini mr/yarn cluster -->
- <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hcatalog/pom.xml b/sharelib/hcatalog/pom.xml
index 2b0c504..f4273a5 100644
--- a/sharelib/hcatalog/pom.xml
+++ b/sharelib/hcatalog/pom.xml
@@ -297,18 +297,6 @@
<outputFile>${project.build.directory}/classpath</outputFile>
</configuration>
</execution>
- <execution>
- <id>create-mrapp-generated-classpath</id>
- <phase>generate-test-resources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <!-- needed to run the unit test for DS to generate the required classpath
- that is required in the env of the launch container in the mini mr/yarn cluster -->
- <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/hive/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml
index d10d7b8..ba49403 100644
--- a/sharelib/hive/pom.xml
+++ b/sharelib/hive/pom.xml
@@ -171,18 +171,6 @@
<outputFile>${project.build.directory}/classpath</outputFile>
</configuration>
</execution>
- <execution>
- <id>create-mrapp-generated-classpath</id>
- <phase>generate-test-resources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <!-- needed to run the unit test for DS to generate the required classpath
- that is required in the env of the launch container in the mini mr/yarn cluster -->
- <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/hive2/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml
index ce967c5..329832d 100644
--- a/sharelib/hive2/pom.xml
+++ b/sharelib/hive2/pom.xml
@@ -152,18 +152,6 @@
<outputFile>${project.build.directory}/classpath</outputFile>
</configuration>
</execution>
- <execution>
- <id>create-mrapp-generated-classpath</id>
- <phase>generate-test-resources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <!-- needed to run the unit test for DS to generate the required classpath
- that is required in the env of the launch container in the mini mr/yarn cluster -->
- <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/oozie/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml
index dd95b45..b2da4e2 100644
--- a/sharelib/oozie/pom.xml
+++ b/sharelib/oozie/pom.xml
@@ -85,18 +85,6 @@
<outputFile>${project.build.directory}/classpath</outputFile>
</configuration>
</execution>
- <execution>
- <id>create-mrapp-generated-classpath</id>
- <phase>generate-test-resources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <!-- needed to run the unit test for DS to generate the required classpath
- that is required in the env of the launch container in the mini mr/yarn cluster -->
- <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>