You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/06/10 22:15:13 UTC
svn commit: r1491589 [2/2] - in /oozie/trunk: ./ core/
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/main/java/org/apache/oozie/util/ core/src/main/resources/
core/src/test/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apach...
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java Mon Jun 10 20:15:11 2013
@@ -295,8 +295,14 @@ public abstract class XTestCase extends
System.exit(-1);
}
}
+ // Copy the specified oozie-site file from oozieSiteSourceStream to the test case dir as oozie-site.xml
+ // We also need to inject oozie.action.ship.launcher.jar as false (if not already set) or else a lot of tests will fail in
+ // weird ways because the ActionExecutors can't find their corresponding Main classes
+ Configuration oozieSiteConf = new Configuration(false);
+ oozieSiteConf.addResource(oozieSiteSourceStream);
+ oozieSiteConf.setBooleanIfUnset("oozie.action.ship.launcher.jar", false);
File target = new File(testCaseConfDir, "oozie-site.xml");
- IOUtils.copyStream(oozieSiteSourceStream, new FileOutputStream(target));
+ oozieSiteConf.writeXml(new FileOutputStream(target));
File hadoopConfDir = new File(testCaseConfDir, "hadoop-conf");
hadoopConfDir.mkdir();
Modified: oozie/trunk/docs/src/site/twiki/AG_Install.twiki
URL: http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/AG_Install.twiki?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/AG_Install.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/AG_Install.twiki Mon Jun 10 20:15:11 2013
@@ -613,10 +613,22 @@ command line tool.
---++ Oozie Share Lib
-The Oozie share lib TAR.GZ file bundled with the distribution contains the necessary files to run Oozie map-reduce
-streaming and pig actions.
+The Oozie sharelib TAR.GZ file bundled with the distribution contains the necessary files to run Oozie map-reduce streaming, pig,
+hive, sqooop, and distcp actions. There is also a sharelib for HCatalog. The sharelib is required for these actions to work; any
+other actions (mapreduce, shell, ssh, and java) do not require the sharelib to be installed.
+
+As of Oozie 4.0, the following property is included. If true, Oozie will create and ship a "launcher jar" that contains classes
+necessary for the launcher job. If false, Oozie will not do this, and it is assumed that the necessary classes are in their
+respective sharelib jars or the "oozie" sharelib instead. When false, the sharelib is required for ALL actions; when true, the
+sharelib is only required for actions that need additional jars (the original list from above). The main advantage of setting this
+to false is that launching jobs should be slightly faster.
-The bundled Streaming and Pig JARs are the ones used by Oozie testcases.
+<verbatim>
+<property>
+ <name>oozie.action.ship.launcher.jar</name>
+ <value>true</value>
+</property>
+</verbatim>
---++ Oozie Coordinators/Bundles Processing Timezone
Modified: oozie/trunk/examples/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/examples/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/examples/pom.xml (original)
+++ oozie/trunk/examples/pom.xml Mon Jun 10 20:15:11 2013
@@ -82,6 +82,12 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
Modified: oozie/trunk/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/pom.xml (original)
+++ oozie/trunk/pom.xml Mon Jun 10 20:15:11 2013
@@ -229,6 +229,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-hcatalog</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-docs</artifactId>
<version>${project.version}</version>
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Mon Jun 10 20:15:11 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1315 Refactor classes from launcher jar into Oozie sharelib (rkanter)
OOZIE-1377 OpenJPA runtime enhancement should be disabled and update OpenJPA to 2.2.2 (tucu)
OOZIE-1339 Implement SLA Bootstrap Service and fix bugs in SLACalculator (virag)
OOZIE-1400 REST API to fetch SLA (rohini)
Modified: oozie/trunk/sharelib/distcp/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/distcp/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/distcp/pom.xml (original)
+++ oozie/trunk/sharelib/distcp/pom.xml Mon Jun 10 20:15:11 2013
@@ -74,6 +74,11 @@
<artifactId>oozie-hcatalog</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
Modified: oozie/trunk/sharelib/hcatalog/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/hcatalog/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/hcatalog/pom.xml (original)
+++ oozie/trunk/sharelib/hcatalog/pom.xml Mon Jun 10 20:15:11 2013
@@ -205,6 +205,26 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
Modified: oozie/trunk/sharelib/hive/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/hive/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/hive/pom.xml (original)
+++ oozie/trunk/sharelib/hive/pom.xml Mon Jun 10 20:15:11 2013
@@ -147,6 +147,11 @@
<artifactId>derby</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
Modified: oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java (original)
+++ oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java Mon Jun 10 20:15:11 2013
@@ -65,15 +65,44 @@ public class TestHiveActionExecutor exte
HiveActionExecutor.class.getName());
}
- public void testSetupMethods() throws Exception {
- HiveActionExecutor ae = new HiveActionExecutor();
- assertEquals("hive", ae.getType());
+ public void testSetupMethodsWithLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "true");
+ _testSetupMethods(true);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
+ }
+
+ public void testSetupMethodsWithoutLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "false");
+ _testSetupMethods(false);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
}
- public void testLauncherJar() throws Exception {
+ public void _testSetupMethods(boolean launcherJarShouldExist) throws Exception {
HiveActionExecutor ae = new HiveActionExecutor();
Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
- assertTrue(new File(jar.toString()).exists());
+ File fJar = new File(jar.toString());
+ fJar.delete();
+ assertFalse(fJar.exists());
+ ae.createLauncherJar();
+ assertEquals(launcherJarShouldExist, fJar.exists());
+
+ assertEquals("hive", ae.getType());
}
private String getHiveScript(String inputPath, String outputPath) {
@@ -144,7 +173,7 @@ public class TestHiveActionExecutor exte
});
assertTrue(launcherJob.isSuccessful());
- assertFalse(LauncherMapper.hasIdSwap(launcherJob));
+ assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
HiveActionExecutor ae = new HiveActionExecutor();
ae.check(context, context.getAction());
Modified: oozie/trunk/sharelib/oozie/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/oozie/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/oozie/pom.xml (original)
+++ oozie/trunk/sharelib/oozie/pom.xml Mon Jun 10 20:15:11 2013
@@ -43,6 +43,21 @@
<artifactId>json-simple</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
Copied: oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (from r1490336, oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java)
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?p2=oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java&p1=oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java&r1=1490336&r2=1491589&rev=1491589&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (original)
+++ oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java Mon Jun 10 20:15:11 2013
@@ -33,29 +33,18 @@ import java.lang.reflect.Method;
import java.net.URI;
import java.security.Permission;
import java.text.MessageFormat;
-import java.util.Map.Entry;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.oozie.service.HadoopAccessorException;
-import org.apache.oozie.service.HadoopAccessorService;
-import org.apache.oozie.service.Services;
-import org.apache.oozie.service.URIHandlerService;
-import org.apache.oozie.util.XLog;
public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, Runnable {
@@ -63,21 +52,21 @@ public class LauncherMapper<K1, V1, K2,
public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = "oozie.action.max.output.data";
- private static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = "oozie.action.main.arg.count";
- private static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = "oozie.action.main.arg.";
- private static final String CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE = "oozie.external.stats.max.size";
-
- private static final String COUNTER_GROUP = "oozie.launcher";
- private static final String COUNTER_DO_ID_SWAP = "oozie.do.id.swap";
- private static final String COUNTER_OUTPUT_DATA = "oozie.output.data";
- private static final String COUNTER_STATS_DATA = "oozie.stats.data";
- private static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error";
+ static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = "oozie.action.main.arg.count";
+ static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = "oozie.action.main.arg.";
+ static final String CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE = "oozie.external.stats.max.size";
+
+ static final String COUNTER_GROUP = "oozie.launcher";
+ static final String COUNTER_DO_ID_SWAP = "oozie.do.id.swap";
+ static final String COUNTER_OUTPUT_DATA = "oozie.output.data";
+ static final String COUNTER_STATS_DATA = "oozie.stats.data";
+ static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error";
- private static final String OOZIE_JOB_ID = "oozie.job.id";
- private static final String OOZIE_ACTION_ID = "oozie.action.id";
+ static final String OOZIE_JOB_ID = "oozie.job.id";
+ static final String OOZIE_ACTION_ID = "oozie.action.id";
- private static final String OOZIE_ACTION_DIR_PATH = "oozie.action.dir.path";
- private static final String OOZIE_ACTION_RECOVERY_ID = "oozie.action.recovery.id";
+ static final String OOZIE_ACTION_DIR_PATH = "oozie.action.dir.path";
+ static final String OOZIE_ACTION_RECOVERY_ID = "oozie.action.recovery.id";
public static final String ACTION_PREFIX = "oozie.action.";
public static final String EXTERNAL_CHILD_IDS = ACTION_PREFIX + "externalChildIDs.properties";
@@ -85,11 +74,11 @@ public class LauncherMapper<K1, V1, K2,
static final String ACTION_CONF_XML = "action.xml";
public static final String ACTION_PREPARE_XML = "oozie.action.prepare.xml";
- private static final String ACTION_OUTPUT_PROPS = "output.properties";
- private static final String ACTION_STATS_PROPS = "stats.properties";
- private static final String ACTION_EXTERNAL_CHILD_IDS_PROPS = "externalChildIds.properties";
- private static final String ACTION_NEW_ID_PROPS = "newId.properties";
- private static final String ACTION_ERROR_PROPS = "error.properties";
+ static final String ACTION_OUTPUT_PROPS = "output.properties";
+ static final String ACTION_STATS_PROPS = "stats.properties";
+ static final String ACTION_EXTERNAL_CHILD_IDS_PROPS = "externalChildIds.properties";
+ static final String ACTION_NEW_ID_PROPS = "newId.properties";
+ static final String ACTION_ERROR_PROPS = "error.properties";
private void setRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws LauncherException {
try {
@@ -124,267 +113,6 @@ public class LauncherMapper<K1, V1, K2,
}
}
- /**
- * @param launcherConf
- * @param actionDir
- * @param recoveryId
- * @return
- * @throws HadoopAccessorException
- * @throws IOException
- */
- public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId)
- throws HadoopAccessorException, IOException {
- String jobId = null;
- Path recoveryFile = new Path(actionDir, recoveryId);
- FileSystem fs = Services.get().get(HadoopAccessorService.class)
- .createFileSystem(launcherConf.get("user.name"),recoveryFile.toUri(), launcherConf);
-
- if (fs.exists(recoveryFile)) {
- InputStream is = fs.open(recoveryFile);
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
- jobId = reader.readLine();
- reader.close();
- }
- return jobId;
-
- }
-
- public static void setupMainClass(Configuration launcherConf, String javaMainClass) {
- // Only set the javaMainClass if its not null or empty string (should be the case except for java action), this way the user
- // can override the action's main class via <configuration> property
- if (javaMainClass != null && !javaMainClass.equals("")) {
- launcherConf.set(CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
- }
- }
-
- public static void setupLauncherURIHandlerConf(Configuration launcherConf) {
- for(Entry<String, String> entry : Services.get().get(URIHandlerService.class).getLauncherConfig()) {
- launcherConf.set(entry.getKey(), entry.getValue());
- }
- }
-
- public static void setupMainArguments(Configuration launcherConf, String[] args) {
- launcherConf.setInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
- for (int i = 0; i < args.length; i++) {
- launcherConf.set(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]);
- }
- }
-
- public static void setupMaxOutputData(Configuration launcherConf, int maxOutputData) {
- launcherConf.setInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData);
- }
-
- /**
- * Set the maximum value of stats data
- *
- * @param launcherConf the oozie launcher configuration
- * @param maxStatsData the maximum allowed size of stats data
- */
- public static void setupMaxExternalStatsSize(Configuration launcherConf, int maxStatsData){
- launcherConf.setInt(CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData);
- }
-
- /**
- * @param launcherConf
- * @param jobId
- * @param actionId
- * @param actionDir
- * @param recoveryId
- * @param actionConf
- * @throws IOException
- * @throws HadoopAccessorException
- */
- public static void setupLauncherInfo(JobConf launcherConf, String jobId, String actionId, Path actionDir,
- String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException {
-
- launcherConf.setMapperClass(LauncherMapper.class);
- launcherConf.setSpeculativeExecution(false);
- launcherConf.setNumMapTasks(1);
- launcherConf.setNumReduceTasks(0);
-
- launcherConf.set(OOZIE_JOB_ID, jobId);
- launcherConf.set(OOZIE_ACTION_ID, actionId);
- launcherConf.set(OOZIE_ACTION_DIR_PATH, actionDir.toString());
- launcherConf.set(OOZIE_ACTION_RECOVERY_ID, recoveryId);
- launcherConf.set(ACTION_PREPARE_XML, prepareXML);
-
- actionConf.set(OOZIE_JOB_ID, jobId);
- actionConf.set(OOZIE_ACTION_ID, actionId);
-
- if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
- List<String> purgedEntries = new ArrayList<String>();
- Collection<String> entries = actionConf.getStringCollection("mapreduce.job.cache.files");
- for (String entry : entries) {
- if (entry.contains("#")) {
- purgedEntries.add(entry);
- }
- }
- actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()]));
- launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
- }
-
- FileSystem fs =
- Services.get().get(HadoopAccessorService.class).createFileSystem(launcherConf.get("user.name"),
- actionDir.toUri(), launcherConf);
- fs.mkdirs(actionDir);
-
- OutputStream os = fs.create(new Path(actionDir, ACTION_CONF_XML));
- actionConf.writeXml(os);
- os.close();
-
- Path inputDir = new Path(actionDir, "input");
- fs.mkdirs(inputDir);
- Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "dummy.txt")));
- writer.write("dummy");
- writer.close();
-
- launcherConf.set("mapred.input.dir", inputDir.toString());
- launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString());
- }
-
- public static boolean isMainDone(RunningJob runningJob) throws IOException {
- return runningJob.isComplete();
- }
-
- public static boolean isMainSuccessful(RunningJob runningJob) throws IOException {
- boolean succeeded = runningJob.isSuccessful();
- if (succeeded) {
- Counters counters = runningJob.getCounters();
- if (counters != null) {
- Counters.Group group = counters.getGroup(COUNTER_GROUP);
- if (group != null) {
- succeeded = group.getCounter(COUNTER_LAUNCHER_ERROR) == 0;
- }
- }
- }
- return succeeded;
- }
-
- public static boolean hasOutputData(RunningJob runningJob) throws IOException {
- boolean output = false;
- Counters counters = runningJob.getCounters();
- if (counters != null) {
- Counters.Group group = counters.getGroup(COUNTER_GROUP);
- if (group != null) {
- output = group.getCounter(COUNTER_OUTPUT_DATA) == 1;
- }
- }
- return output;
- }
-
- /**
- * Check whether runningJob has stats data or not
- *
- * @param runningJob the runningJob
- * @return returns whether the running Job has stats data or not
- * @throws IOException
- */
- public static boolean hasStatsData(RunningJob runningJob) throws IOException{
- boolean output = false;
- Counters counters = runningJob.getCounters();
- if (counters != null) {
- Counters.Group group = counters.getGroup(COUNTER_GROUP);
- if (group != null) {
- output = group.getCounter(COUNTER_STATS_DATA) == 1;
- }
- }
- return output;
- }
-
- /**
- * @param runningJob
- * @return
- * @throws IOException
- */
- public static boolean hasIdSwap(RunningJob runningJob) throws IOException {
- boolean swap = false;
- Counters counters = runningJob.getCounters();
- if (counters != null) {
- Counters.Group group = counters.getGroup(COUNTER_GROUP);
- if (group != null) {
- swap = group.getCounter(COUNTER_DO_ID_SWAP) == 1;
- }
- }
- return swap;
- }
-
- /**
- * @param runningJob
- * @param user
- * @param group
- * @param actionDir
- * @return
- * @throws IOException
- * @throws HadoopAccessorException
- */
- public static boolean hasIdSwap(RunningJob runningJob, String user, String group, Path actionDir)
- throws IOException, HadoopAccessorException {
- boolean swap = false;
-
- XLog log = XLog.getLog("org.apache.oozie.action.hadoop.LauncherMapper");
-
- Counters counters = runningJob.getCounters();
- if (counters != null) {
- Counters.Group counterGroup = counters.getGroup(COUNTER_GROUP);
- if (counterGroup != null) {
- swap = counterGroup.getCounter(COUNTER_DO_ID_SWAP) == 1;
- }
- }
- // additional check for swapped hadoop ID
- // Can't rely on hadoop counters existing
- // we'll check for the newID file in hdfs if the hadoop counters is null
- else {
-
- Path p = getIdSwapPath(actionDir);
- // log.debug("Checking for newId file in: [{0}]", p);
-
- HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
- Configuration conf = has.createJobConf(p.toUri().getAuthority());
- FileSystem fs = has.createFileSystem(user, p.toUri(), conf);
- if (fs.exists(p)) {
- log.debug("Hadoop Counters is null, but found newID file.");
-
- swap = true;
- }
- else {
- log.debug("Hadoop Counters is null, and newID file doesn't exist at: [{0}]", p);
- }
- }
- return swap;
- }
-
- public static Path getOutputDataPath(Path actionDir) {
- return new Path(actionDir, ACTION_OUTPUT_PROPS);
- }
-
- /**
- * Get the location of stats file
- *
- * @param actionDir the action directory
- * @return the hdfs location of the file
- */
- public static Path getActionStatsDataPath(Path actionDir){
- return new Path(actionDir, ACTION_STATS_PROPS);
- }
-
- /**
- * Get the location of external Child IDs file
- *
- * @param actionDir the action directory
- * @return the hdfs location of the file
- */
- public static Path getExternalChildIDsDataPath(Path actionDir){
- return new Path(actionDir, ACTION_EXTERNAL_CHILD_IDS_PROPS);
- }
-
- public static Path getErrorPath(Path actionDir) {
- return new Path(actionDir, ACTION_ERROR_PROPS);
- }
-
- public static Path getIdSwapPath(Path actionDir) {
- return new Path(actionDir, ACTION_NEW_ID_PROPS);
- }
-
private JobConf jobConf;
private Path actionDir;
private ScheduledThreadPoolExecutor timer;
@@ -651,7 +379,7 @@ public class LauncherMapper<K1, V1, K2,
}
}
- private void setupMainConfiguration() throws IOException, HadoopAccessorException {
+ private void setupMainConfiguration() throws IOException {
Path pathNew = new Path(new Path(actionDir, ACTION_CONF_XML),
new Path(new File(ACTION_CONF_XML).getAbsolutePath()));
FileSystem fs = FileSystem.get(pathNew.toUri(), getJobConf());
Copied: oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java (from r1490336, oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java)
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java?p2=oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java&p1=oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java&r1=1490336&r2=1491589&rev=1491589&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java (original)
+++ oozie/trunk/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java Mon Jun 10 20:15:11 2013
@@ -36,6 +36,8 @@ import java.io.File;
public class MapReduceMain extends LauncherMain {
+ public static final String OOZIE_MAPREDUCE_UBER_JAR = "oozie.mapreduce.uber.jar";
+
public static void main(String[] args) throws Exception {
run(MapReduceMain.class, args);
}
@@ -80,7 +82,7 @@ public class MapReduceMain extends Launc
addActionConf(jobConf, actionConf);
// Set for uber jar
- String uberJar = actionConf.get(MapReduceActionExecutor.OOZIE_MAPREDUCE_UBER_JAR);
+ String uberJar = actionConf.get(OOZIE_MAPREDUCE_UBER_JAR);
if (uberJar != null && uberJar.trim().length() > 0) {
jobConf.setJar(uberJar);
}
Modified: oozie/trunk/sharelib/pig/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/pig/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/pig/pom.xml (original)
+++ oozie/trunk/sharelib/pig/pom.xml Mon Jun 10 20:15:11 2013
@@ -97,7 +97,11 @@
<artifactId>oozie-hcatalog</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
Modified: oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java (original)
+++ oozie/trunk/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java Mon Jun 10 20:15:11 2013
@@ -82,14 +82,42 @@ public class TestPigActionExecutor exten
setSystemProperty("oozie.service.ActionService.executor.classes", PigActionExecutor.class.getName());
}
- public void testLauncherJar() throws Exception {
- PigActionExecutor ae = new PigActionExecutor();
- Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
- assertTrue(new File(jar.toString()).exists());
+ public void testSetupMethodsWithLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "true");
+ _testSetupMethods(true);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
+ }
+
+ public void testSetupMethodsWithoutLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "false");
+ _testSetupMethods(false);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
}
- public void testSetupMethods() throws Exception {
+ public void _testSetupMethods(boolean launcherJarShouldExist) throws Exception {
PigActionExecutor ae = new PigActionExecutor();
+ Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
+ File fJar = new File(jar.toString());
+ fJar.delete();
+ assertFalse(fJar.exists());
+ ae.createLauncherJar();
+ assertEquals(launcherJarShouldExist, fJar.exists());
assertEquals("pig", ae.getType());
@@ -198,9 +226,9 @@ public class TestPigActionExecutor exten
assertTrue(launcherJob.isSuccessful());
sleep(2000);
- assertFalse(LauncherMapper.hasIdSwap(launcherJob));
+ assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
if (checkForSuccess) {
- assertTrue(LauncherMapper.hasStatsData(launcherJob));
+ assertTrue(LauncherMapperHelper.hasStatsData(launcherJob));
}
PigActionExecutor ae = new PigActionExecutor();
@@ -234,7 +262,7 @@ public class TestPigActionExecutor exten
final RunningJob launcherJob = submitAction(context);
evaluateLauncherJob(launcherJob);
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapper.hasStatsData(launcherJob));
+ assertTrue(LauncherMapperHelper.hasStatsData(launcherJob));
PigActionExecutor ae = new PigActionExecutor();
WorkflowAction wfAction = context.getAction();
@@ -321,7 +349,7 @@ public class TestPigActionExecutor exten
final RunningJob launcherJob = submitAction(context);
evaluateLauncherJob(launcherJob);
assertTrue(launcherJob.isSuccessful());
- assertFalse(LauncherMapper.hasStatsData(launcherJob));
+ assertFalse(LauncherMapperHelper.hasStatsData(launcherJob));
PigActionExecutor ae = new PigActionExecutor();
WorkflowAction wfAction = context.getAction();
Modified: oozie/trunk/sharelib/sqoop/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/sqoop/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/sqoop/pom.xml (original)
+++ oozie/trunk/sharelib/sqoop/pom.xml Mon Jun 10 20:15:11 2013
@@ -212,6 +212,11 @@
<artifactId>oozie-sharelib-hive</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
Modified: oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java (original)
+++ oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java Mon Jun 10 20:15:11 2013
@@ -102,15 +102,44 @@ public class TestSqoopActionExecutor ext
setSystemProperty("oozie.service.ActionService.executor.classes", SqoopActionExecutor.class.getName());
}
- public void testSetupMethods() throws Exception {
- SqoopActionExecutor ae = new SqoopActionExecutor();
- assertEquals("sqoop", ae.getType());
+ public void testSetupMethodsWithLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "true");
+ _testSetupMethods(true);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
+ }
+
+ public void testSetupMethodsWithoutLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "false");
+ _testSetupMethods(false);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
}
- public void testLauncherJar() throws Exception {
+ public void _testSetupMethods(boolean launcherJarShouldExist) throws Exception {
SqoopActionExecutor ae = new SqoopActionExecutor();
Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
- assertTrue(new File(jar.toString()).exists());
+ File fJar = new File(jar.toString());
+ fJar.delete();
+ assertFalse(fJar.exists());
+ ae.createLauncherJar();
+ assertEquals(launcherJarShouldExist, fJar.exists());
+
+ assertEquals("sqoop", ae.getType());
}
private String getDbFile() {
@@ -170,7 +199,7 @@ public class TestSqoopActionExecutor ext
});
assertTrue(launcherJob.isSuccessful());
- assertFalse(LauncherMapper.hasIdSwap(launcherJob));
+ assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
SqoopActionExecutor ae = new SqoopActionExecutor();
ae.check(context, context.getAction());
@@ -217,7 +246,7 @@ public class TestSqoopActionExecutor ext
});
assertTrue(launcherJob.isSuccessful());
- assertFalse(LauncherMapper.hasIdSwap(launcherJob));
+ assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
SqoopActionExecutor ae = new SqoopActionExecutor();
ae.check(context, context.getAction());
Modified: oozie/trunk/sharelib/streaming/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/streaming/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/sharelib/streaming/pom.xml (original)
+++ oozie/trunk/sharelib/streaming/pom.xml Mon Jun 10 20:15:11 2013
@@ -80,6 +80,11 @@
<artifactId>oozie-hcatalog</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
Copied: oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java (from r1490336, oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java)
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?p2=oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java&p1=oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java&r1=1490336&r2=1491589&rev=1491589&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java (original)
+++ oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java Mon Jun 10 20:15:11 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.streaming.StreamJob;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.client.WorkflowAction;
@@ -65,12 +66,6 @@ public class TestMapReduceActionExecutor
setSystemProperty("oozie.credentials.credentialclasses", "cred=org.apache.oozie.action.hadoop.CredentialForTest");
}
- public void testLauncherJar() throws Exception {
- MapReduceActionExecutor ae = new MapReduceActionExecutor();
- Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
- assertTrue(new File(jar.toString()).exists());
- }
-
public Element createUberJarActionXML(String uberJarPath, String additional) throws Exception{
return XmlUtils.parseXml("<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
+ "<name-node>" + getNameNodeUri() + "</name-node>" + additional + "<configuration>"
@@ -78,27 +73,63 @@ public class TestMapReduceActionExecutor
+ "</configuration>" + "</map-reduce>");
}
- public void testSetupMethods() throws Exception {
+ public void testSetupMethodsWithLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "true");
+ _testSetupMethods(true);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
+ }
+
+ public void testSetupMethodsWithoutLauncherJar() throws Exception {
+ String defaultVal = Services.get().getConf().get("oozie.action.ship.launcher.jar");
+ try {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", "false");
+ _testSetupMethods(false);
+ }
+ finally {
+ // back to default
+ if (defaultVal != null) {
+ Services.get().getConf().set("oozie.action.ship.launcher.jar", defaultVal);
+ }
+ }
+ }
+
+ public void _testSetupMethods(boolean launcherJarShouldExist) throws Exception {
MapReduceActionExecutor ae = new MapReduceActionExecutor();
+ Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
+ File fJar = new File(jar.toString());
+ fJar.delete();
+ assertFalse(fJar.exists());
+ ae.createLauncherJar();
+ assertEquals(launcherJarShouldExist, fJar.exists());
assertEquals("map-reduce", ae.getType());
assertEquals("map-reduce-launcher.jar", ae.getLauncherJarName());
- List<Class> classes = new ArrayList<Class>();
- classes.add(LauncherMapper.class);
- classes.add(LauncherSecurityManager.class);
- classes.add(LauncherException.class);
- classes.add(LauncherMainException.class);
- classes.add(PrepareActionsDriver.class);
- classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
- classes.add(ActionStats.class);
- classes.add(ActionType.class);
- classes.add(LauncherMain.class);
- classes.add(MapReduceMain.class);
- classes.add(PipesMain.class);
- // TODO - Remove comment when Main class refactoring is done
- //assertEquals(classes, ae.getLauncherClasses());
+ if (launcherJarShouldExist) {
+ List<Class> classes = new ArrayList<Class>();
+ classes.add(LauncherMapper.class);
+ classes.add(LauncherSecurityManager.class);
+ classes.add(LauncherException.class);
+ classes.add(LauncherMainException.class);
+ classes.add(PrepareActionsDriver.class);
+ classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
+ classes.add(ActionStats.class);
+ classes.add(ActionType.class);
+ classes.add(LauncherMain.class);
+ classes.add(MapReduceMain.class);
+ classes.add(PipesMain.class);
+ classes.add(StreamingMain.class);
+ assertEquals(classes, ae.getLauncherClasses());
+ }
Element actionXml = XmlUtils.parseXml("<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
+ "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>"
@@ -300,7 +331,7 @@ public class TestMapReduceActionExecutor
return runningJob;
}
- protected String _testSubmit(String name, String actionXml) throws Exception {
+ private String _testSubmit(String name, String actionXml) throws Exception {
Context context = createContext(name, actionXml);
final RunningJob launcherJob = submitAction(context);
@@ -312,7 +343,7 @@ public class TestMapReduceActionExecutor
});
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapper.hasIdSwap(launcherJob));
+ assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -361,7 +392,7 @@ public class TestMapReduceActionExecutor
});
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapper.hasIdSwap(launcherJob));
+ assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -568,6 +599,37 @@ public class TestMapReduceActionExecutor
}
}
+ protected XConfiguration getStreamingConfig(String inputDir, String outputDir) {
+ XConfiguration conf = new XConfiguration();
+ conf.set("mapred.input.dir", inputDir);
+ conf.set("mapred.output.dir", outputDir);
+ return conf;
+ }
+
+ public void testStreaming() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path streamingJar = new Path(getFsTestCaseDir(), "jar/hadoop-streaming.jar");
+
+ InputStream is = new FileInputStream(ClassUtils.findContainingJar(StreamJob.class));
+ OutputStream os = fs.create(new Path(getAppPath(), streamingJar));
+ IOUtils.copyStream(is, os);
+
+ Path inputDir = new Path(getFsTestCaseDir(), "input");
+ Path outputDir = new Path(getFsTestCaseDir(), "output");
+
+ Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
+ w.write("dummy\n");
+ w.write("dummy\n");
+ w.close();
+
+ String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ + getNameNodeUri() + "</name-node>" + " <streaming>" + " <mapper>cat</mapper>"
+ + " <reducer>wc</reducer>" + " </streaming>"
+ + getStreamingConfig(inputDir.toString(), outputDir.toString()).toXmlString(false) + "<file>"
+ + streamingJar + "</file>" + "</map-reduce>";
+ _testSubmit("streaming", actionXml);
+ }
+
protected XConfiguration getPipesConfig(String inputDir, String outputDir) {
XConfiguration conf = new XConfiguration();
conf.setBoolean("hadoop.pipes.java.recordreader", true);
@@ -653,7 +715,7 @@ public class TestMapReduceActionExecutor
});
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapper.hasIdSwap(launcherJob));
+ assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -728,7 +790,7 @@ public class TestMapReduceActionExecutor
});
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapper.hasIdSwap(launcherJob));
+ assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
MapReduceActionExecutor ae = new MapReduceActionExecutor();
ae.check(context, context.getAction());
@@ -809,7 +871,7 @@ public class TestMapReduceActionExecutor
});
assertTrue(launcherJob.isSuccessful());
- assertTrue(LauncherMapper.hasIdSwap(launcherJob));
+ assertTrue(LauncherMapperHelper.hasIdSwap(launcherJob));
// Assert launcher job name has been set
System.out.println("Launcher job name: " + launcherJob.getJobName());
assertTrue(launcherJob.getJobName().equals(launcherJobName));
Copied: oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutorUberJar.java (from r1490336, oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutorUberJar.java)
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutorUberJar.java?p2=oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutorUberJar.java&p1=oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutorUberJar.java&r1=1490336&r2=1491589&rev=1491589&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutorUberJar.java (original)
+++ oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutorUberJar.java Mon Jun 10 20:15:11 2013
@@ -46,7 +46,12 @@ public class TestMapReduceActionExecutor
}
@Override
- public void testLauncherJar() throws Exception {
+ public void testSetupMethodsWithLauncherJar() throws Exception {
+ // skip test
+ }
+
+ @Override
+ public void testSetupMethodsWithoutLauncherJar() throws Exception {
// skip test
}
@@ -71,7 +76,7 @@ public class TestMapReduceActionExecutor
}
@Override
- public void testSetupMethods() throws Exception {
+ public void testStreaming() throws Exception {
// skip test
}
Modified: oozie/trunk/webapp/pom.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/webapp/pom.xml?rev=1491589&r1=1491588&r2=1491589&view=diff
==============================================================================
--- oozie/trunk/webapp/pom.xml (original)
+++ oozie/trunk/webapp/pom.xml Mon Jun 10 20:15:11 2013
@@ -113,6 +113,24 @@
<artifactId>oozie-sharelib-streaming</artifactId>
<scope>compile</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-oozie</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-hcatalog</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-distcp</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>