You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2012/07/11 19:03:37 UTC
svn commit: r1360297 - in /incubator/oozie/trunk: ./
client/src/main/java/org/apache/oozie/cli/
core/src/test/java/org/apache/oozie/client/ docs/src/site/twiki/
Author: virag
Date: Wed Jul 11 17:03:37 2012
New Revision: 1360297
URL: http://svn.apache.org/viewvc?rev=1360297&view=rev
Log:
OOZIE-890 Support for map-reduce in OozieCLI (britt via virag)
Modified:
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
incubator/oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki
incubator/oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1360297&r1=1360296&r2=1360297&view=diff
==============================================================================
--- incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java (original)
+++ incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Wed Jul 11 17:03:37 2012
@@ -85,6 +85,7 @@ public class OozieCLI {
public static final String VALIDATE_CMD = "validate";
public static final String SLA_CMD = "sla";
public static final String PIG_CMD = "pig";
+ public static final String MR_CMD = "mapreduce";
public static final String INFO_CMD = "info";
public static final String OOZIE_OPTION = "oozie";
@@ -144,6 +145,11 @@ public class OozieCLI {
private static final String INSTANCE_SEPARATOR = "#";
+ private static final String MAPRED_MAPPER = "mapred.mapper.class";
+ private static final String MAPRED_REDUCER = "mapred.reducer.class";
+ private static final String MAPRED_INPUT = "mapred.input.dir";
+ private static final String MAPRED_OUTPUT = "mapred.output.dir";
+
static {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < LINE_WIDTH; i++) {
@@ -388,7 +394,27 @@ public class OozieCLI {
infoOptions.addOption(timezones);
return infoOptions;
}
-
+
+ /**
+ * Create option for command line option 'mapreduce'
+ * @return mapreduce options
+ */
+ @SuppressWarnings("static-access")
+ protected Options createMROptions() {
+ Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
+ Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'");
+ Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
+ "set/override value for given property").create("D");
+ Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
+ Options mrOptions = new Options();
+ mrOptions.addOption(oozie);
+ mrOptions.addOption(doAs);
+ mrOptions.addOption(config);
+ mrOptions.addOption(property);
+ addAuthOptions(mrOptions);
+ return mrOptions;
+ }
+
/**
* Run a CLI programmatically.
* <p/>
@@ -416,6 +442,7 @@ public class OozieCLI {
parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig",
createPigOptions(), true);
parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false);
+ parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false);
try {
final CLIParser.Command command = parser.parse(args);
@@ -482,6 +509,9 @@ public class OozieCLI {
else if (command.getName().equals(INFO_CMD)) {
infoCommand(command.getCommandLine());
}
+ else if (command.getName().equals(MR_CMD)){
+ mrCommand(command.getCommandLine());
+ }
}
protected String getOozieUrl(CommandLine commandLine) {
String url = commandLine.getOptionValue(OOZIE_OPTION);
@@ -1486,4 +1516,37 @@ public class OozieCLI {
}
}
}
+
+
+ private void mrCommand(CommandLine commandLine) throws IOException, OozieCLIException {
+ try {
+ XOozieClient wc = createXOozieClient(commandLine);
+ Properties conf = getConfiguration(wc, commandLine);
+
+ String mapper = conf.getProperty(MAPRED_MAPPER);
+ if (mapper == null) {
+ throw new OozieCLIException("mapper is not specified in conf");
+ }
+
+ String reducer = conf.getProperty(MAPRED_REDUCER);
+ if (reducer == null) {
+ throw new OozieCLIException("reducer is not specified in conf");
+ }
+
+ String inputDir = conf.getProperty(MAPRED_INPUT);
+ if (inputDir == null) {
+ throw new OozieCLIException("mapred.input.dir is not specified in conf");
+ }
+
+ String outputDir = conf.getProperty(MAPRED_OUTPUT);
+ if (outputDir == null) {
+ throw new OozieCLIException("mapred.output.dir is not specified in conf");
+ }
+
+ System.out.println(JOB_ID_PREFIX + wc.submitMapReduce(conf));
+ }
+ catch (OozieClientException ex) {
+ throw new OozieCLIException(ex.toString(), ex);
+ }
+ }
}
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java?rev=1360297&r1=1360296&r2=1360297&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java Wed Jul 11 17:03:37 2012
@@ -17,6 +17,7 @@
*/
package org.apache.oozie.client;
+import java.io.DataOutputStream;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.io.StringReader;
@@ -101,6 +102,59 @@ public class TestOozieCLI extends DagSer
return path;
}
+ private String createPigPropertiesFile(String appPath) throws Exception {
+ String path = getTestCaseDir() + "/" + getName() + ".properties";
+ Properties props = new Properties();
+ props.setProperty(OozieClient.USER_NAME, getTestUser());
+ props.setProperty(XOozieClient.NN, "localhost:9000");
+ props.setProperty(XOozieClient.JT, "localhost:9001");
+ props.setProperty("oozie.libpath", appPath);
+ props.setProperty("mapred.output.dir", appPath);
+ props.setProperty("a", "A");
+
+ OutputStream os = new FileOutputStream(path);
+ props.store(os, "");
+ os.close();
+ return path;
+ }
+
+ private String createMRProperties(String appPath) throws Exception {
+ String path = getTestCaseDir() + "/" + getName() + ".properties";
+ Properties props = new Properties();
+ props.setProperty(OozieClient.USER_NAME, getTestUser());
+ props.setProperty(OozieClient.GROUP_NAME, getTestGroup());
+ props.setProperty(OozieClient.APP_PATH, appPath);
+ props.setProperty(OozieClient.RERUN_SKIP_NODES, "node");
+ props.setProperty(XOozieClient.NN, "localhost:9000");
+ props.setProperty(XOozieClient.JT, "localhost:9001");
+ props.setProperty("mapred.mapper.class", "mapper.class");
+ props.setProperty("mapred.reducer.class", "reducer.class");
+ props.setProperty("mapred.input.dir", "input");
+ props.setProperty("mapred.output.dir", "output");
+ props.setProperty("oozie.libpath", appPath);
+ props.setProperty("a", "A");
+
+ OutputStream os = new FileOutputStream(path);
+ props.store(os, "");
+ os.close();
+ return path;
+ }
+
+ private String createPigScript(String appPath) throws Exception {
+ String path = getTestCaseDir() + "/" + getName() + ".properties";
+
+ DataOutputStream dos = new DataOutputStream(new FileOutputStream(path));
+
+ String pigScript = "A = load \'/user/data\' using PigStorage(:);\n" +
+ "B = foreach A generate $0" +
+ "dumb B;";
+
+ dos.writeBytes(pigScript);
+ dos.close();
+
+ return path;
+ }
+
public void testSubmit() throws Exception {
runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() {
public Void call() throws Exception {
@@ -143,6 +197,43 @@ public class TestOozieCLI extends DagSer
});
}
+ public void testSubmitPig() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() {
+ public Void call() throws Exception {
+ String oozieUrl = getContextURL();
+ int wfCount = MockDagEngineService.INIT_WF_COUNT;
+
+ Path appPath = new Path(getFsTestCaseDir(), "app");
+ getFileSystem().mkdirs(appPath);
+
+ String[] args = new String[]{"pig", "-oozie", oozieUrl, "-file", createPigScript(appPath.toString()), "-config",
+ createPigPropertiesFile(appPath.toString())};
+ assertEquals(0, new OozieCLI().run(args));
+ assertEquals("submitPig", MockDagEngineService.did);
+ assertTrue(MockDagEngineService.started.get(wfCount));
+ return null;
+ }
+ });
+ }
+
+ public void testSubmitMapReduce() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() {
+ public Void call() throws Exception {
+ String oozieUrl = getContextURL();
+ int wfCount = MockDagEngineService.INIT_WF_COUNT;
+
+ Path appPath = new Path(getFsTestCaseDir(), "app");
+ getFileSystem().mkdirs(appPath);
+
+ String[] args = new String[]{"mapreduce", "-oozie", oozieUrl, "-config", createMRProperties(appPath.toString())};
+ assertEquals(0, new OozieCLI().run(args));
+ assertEquals("submitMR", MockDagEngineService.did);
+ assertTrue(MockDagEngineService.started.get(wfCount));
+ return null;
+ }
+ });
+ }
+
public void testSubmitDoAs() throws Exception {
runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() {
public Void call() throws Exception {
Modified: incubator/oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki?rev=1360297&r1=1360296&r2=1360297&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki Wed Jul 11 17:03:37 2012
@@ -724,7 +724,7 @@ oozie.libpath=hdfs://localhost:9000/user
</verbatim>
The parameters for the job must be provided in a Java Properties file (.properties). jobtracker, namenode, libpath must be
-specified in this file. pigScriptFile is a local file. All jar files (including pig jar file) and all other files needed by the pig job (e.g., parameter file in above example) need to be uploaded onto HDFS under libpath beforehand. The workflow.xml will be created in Oozie server internally. Users can get the workflow.xml from console or command line(-definition).
+specified in this file. pigScriptFile is a local file. All jar files (including pig jar file) and all other files needed by the pig job (e.g., parameter file in above example) need to be uploaded onto HDFS under libpath beforehand. The workflow.xml will be created in Oozie server internally. Users can get the workflow.xml from console or command line(-definition). Everything after -X will be pass through parameters to pig.
The job will be created and run right away.
@@ -758,6 +758,21 @@ The <code>-timezones</code> option will
These IDs (the text in the parentheses) are what should be used for the <code>-timezone TIME_ZONE_ID</code> option in the =job=
and =jobs= sub-commands
+---++ Map-reduce Operations
+
+---+++ Submitting a map-reduce job
+
+Example:
+
+<verbatim>
+$ oozie mapreduce -oozie http://localhost:8080/oozie -config job.properties
+</verbatim>
+
+The parameters must be in the Java Properties file (.properties). This file must be specified for a map-reduce job.
+The properties file must specify the =mapred.mapper.class=, =mapred.reducer.class=, =mapred.input.dir=, =mapred.output.dir=, =oozie.libpath= properties.
+
+The map-reduce job will be created and submitted. All jar files and all other files needed by the mapreduce job need to be uploaded onto HDFS under libpath beforehand. The workflow.xml will be created in Oozie server internally. Users can get the workflow.xml from console or command line(-definition).
+
[[index][::Go back to Oozie Documentation Index::]]
</noautolink>
Modified: incubator/oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki?rev=1360297&r1=1360296&r2=1360297&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki Wed Jul 11 17:03:37 2012
@@ -347,11 +347,13 @@ These endpoints is for submitting, manag
A HTTP POST request with an XML configuration as payload creates a job.
-The type of job is determined by the presence of one of the following 2 properties (only one must be present):
+The type of job is determined by the presence of one of the following 3 properties:
* =oozie.wf.application.path= : path to a workflow aplication directory, creates a workflow job
* =oozie.coord.application.path= : path to a coordinator application file, creates a coordinator job
* =oozie.bundle.application.path= : path to a bundle application file, creates a bundle job
+
+Or, if none of those are present, the jobtype parameter determines the type of job to run. It can either be mapreduce or pig.
*Request:*
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1360297&r1=1360296&r2=1360297&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Wed Jul 11 17:03:37 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-890 Support for map-reduce in OozieCLI (britt via virag)
OOZIE-887 Support for choosing timezone in Oozie UI (rkanter via tucu)
OOZIE-902 XLogService doesn't properly disable WS log streaming if log4j.appender.oozie.File is missing (rkanter via tucu)
OOZIE-885 A race condition can cause the workflow/coordinator to run even after the bundle job is killed (virag)