You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/05/10 19:30:42 UTC
svn commit: r1481115 - in /oozie/trunk: ./
client/src/main/java/org/apache/oozie/cli/
client/src/main/java/org/apache/oozie/client/ client/src/main/resources/
core/src/main/conf/ core/src/main/java/org/apache/oozie/
core/src/main/java/org/apache/oozie/...
Author: rkanter
Date: Fri May 10 17:30:41 2013
New Revision: 1481115
URL: http://svn.apache.org/r1481115
Log:
OOZIE-1353 hive CLI fails with -X argument (rkanter)
Added:
oozie/trunk/client/src/main/resources/hive-action-0.5.xsd
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
oozie/trunk/core/src/main/conf/oozie-site.xml
oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
oozie/trunk/docs/src/site/twiki/DG_HiveActionExtension.twiki
oozie/trunk/release-log.txt
oozie/trunk/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
Modified: oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL: http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java (original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Fri May 10 17:30:41 2013
@@ -390,12 +390,15 @@ public class OozieCLI {
Option file = new Option(SCRIPTFILE_OPTION, true, jobType + " script");
Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
"set/override value for given property").create("D");
+ Option params = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
+ "set parameters for script").create("P");
Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
Options Options = new Options();
Options.addOption(oozie);
Options.addOption(doAs);
Options.addOption(config);
Options.addOption(property);
+ Options.addOption(params);
Options.addOption(file);
addAuthOptions(Options);
return Options;
@@ -456,10 +459,10 @@ public class OozieCLI {
parser.addCommand(ADMIN_CMD, "", "admin operations", createAdminOptions(), false);
parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true);
parser.addCommand(SLA_CMD, "", "sla operations (Supported in Oozie-2.0 or later)", createSlaOptions(), false);
- parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig",
- createScriptLanguageOptions(PIG_CMD), true);
- parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive",
- createScriptLanguageOptions(HIVE_CMD), true);
+ parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig, any '-D' "
+ + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(PIG_CMD), true);
+ parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive, any '-D' "
+ + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(HIVE_CMD), true);
parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false);
parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false);
@@ -1549,6 +1552,8 @@ public class OozieCLI {
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"hive-action-0.4.xsd")));
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
+ "hive-action-0.5.xsd")));
+ sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"sqoop-action-0.2.xsd")));
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"sqoop-action-0.3.xsd")));
@@ -1574,34 +1579,36 @@ public class OozieCLI {
}
private void scriptLanguageCommand(CommandLine commandLine, String jobType) throws IOException, OozieCLIException {
- List<String> Args = commandLine.getArgList();
- if (Args.size() > 0) {
- // checking is a pigArgs starts with -X (because CLIParser cannot check this)
- if (!Args.get(0).equals("-X")) {
- throw new OozieCLIException("Unrecognized option: " + Args.get(0) + " Expecting -X");
+ List<String> args = commandLine.getArgList();
+ if (args.size() > 0) {
+ // checking if args starts with -X (because CLIParser cannot check this)
+ if (!args.get(0).equals("-X")) {
+ throw new OozieCLIException("Unrecognized option: " + args.get(0) + " Expecting -X");
}
- Args.remove(0);
+ args.remove(0);
}
- List<String> options = new ArrayList<String>();
- for (Option option : commandLine.getOptions()) {
- options.add(option.getOpt());
- }
-
- if (!options.contains(SCRIPTFILE_OPTION)) {
+ if (!commandLine.hasOption(SCRIPTFILE_OPTION)) {
throw new OozieCLIException("Need to specify -file <scriptfile>");
}
- if (!options.contains(CONFIG_OPTION)) {
+ if (!commandLine.hasOption(CONFIG_OPTION)) {
throw new OozieCLIException("Need to specify -config <configfile>");
}
-
try {
XOozieClient wc = createXOozieClient(commandLine);
Properties conf = getConfiguration(wc, commandLine);
String script = commandLine.getOptionValue(SCRIPTFILE_OPTION);
- System.out.println(JOB_ID_PREFIX + wc.submitScriptLanguage(conf, script, Args.toArray(new String[Args.size()]), jobType));
+ List<String> paramsList = new ArrayList<String>();
+ if (commandLine.hasOption("P")) {
+ Properties params = commandLine.getOptionProperties("P");
+ for (String key : params.stringPropertyNames()) {
+ paramsList.add(key + "=" + params.getProperty(key));
+ }
+ }
+ System.out.println(JOB_ID_PREFIX + wc.submitScriptLanguage(conf, script, args.toArray(new String[args.size()]),
+ paramsList.toArray(new String[paramsList.size()]), jobType));
}
catch (OozieClientException ex) {
throw new OozieCLIException(ex.toString(), ex);
Modified: oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
URL: http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java (original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java Fri May 10 17:30:41 2013
@@ -49,10 +49,14 @@ public class XOozieClient extends OozieC
public static final String PIG_OPTIONS = "oozie.pig.options";
+ public static final String PIG_SCRIPT_PARAMS = "oozie.pig.script.params";
+
public static final String HIVE_SCRIPT = "oozie.hive.script";
public static final String HIVE_OPTIONS = "oozie.hive.options";
+ public static final String HIVE_SCRIPT_PARAMS = "oozie.hive.script.params";
+
public static final String FILES = "oozie.files";
public static final String ARCHIVES = "oozie.archives";
@@ -144,15 +148,45 @@ public class XOozieClient extends OozieC
}
/**
+ * Submit a Pig job via HTTP.
+ *
+ * @param conf job configuration.
+ * @param pigScriptFile pig script file.
+ * @param pigArgs pig arguments string.
+ * @return the job Id.
+ * @throws OozieClientException thrown if the job could not be submitted.
+ */
+ @Deprecated
+ public String submitPig(Properties conf, String pigScriptFile, String[] pigArgs) throws IOException, OozieClientException {
+ return submitScriptLanguage(conf, pigScriptFile, pigArgs, OozieCLI.PIG_CMD);
+ }
+
+ /**
+ * Submit a Pig or Hive job via HTTP.
+ *
+ * @param conf job configuration.
+ * @param scriptFile script file.
+ * @param args arguments string.
+ * @return the job Id.
+ * @throws OozieClientException thrown if the job could not be submitted.
+ */
+ public String submitScriptLanguage(Properties conf, String scriptFile, String[] args, String jobType)
+ throws IOException, OozieClientException {
+ return submitScriptLanguage(conf, scriptFile, args, null, jobType);
+ }
+
+ /**
* Submit a Pig or Hive job via HTTP.
*
* @param conf job configuration.
* @param scriptFile script file.
* @param args arguments string.
+ * @param params parameters string.
* @return the job Id.
* @throws OozieClientException thrown if the job could not be submitted.
*/
- public String submitScriptLanguage(Properties conf, String scriptFile, String[] args, String jobType) throws IOException, OozieClientException {
+ public String submitScriptLanguage(Properties conf, String scriptFile, String[] args, String[] params, String jobType)
+ throws IOException, OozieClientException {
if (conf == null) {
throw new IllegalArgumentException("conf cannot be null");
}
@@ -164,14 +198,17 @@ public class XOozieClient extends OozieC
String script = "";
String options = "";
+ String scriptParams = "";
if (jobType.equals(OozieCLI.HIVE_CMD)) {
script = XOozieClient.HIVE_SCRIPT;
options = XOozieClient.HIVE_OPTIONS;
+ scriptParams = XOozieClient.HIVE_SCRIPT_PARAMS;
}
else if (jobType.equals(OozieCLI.PIG_CMD)) {
script = XOozieClient.PIG_SCRIPT;
options = XOozieClient.PIG_OPTIONS;
+ scriptParams = XOozieClient.PIG_SCRIPT_PARAMS;
}
else {
throw new IllegalArgumentException("jobType must be either pig or hive");
@@ -179,6 +216,7 @@ public class XOozieClient extends OozieC
conf.setProperty(script, readScript(scriptFile));
setStrings(conf, options, args);
+ setStrings(conf, scriptParams, params);
return (new HttpJobSubmit(conf, jobType)).call();
}
Added: oozie/trunk/client/src/main/resources/hive-action-0.5.xsd
URL: http://svn.apache.org/viewvc/oozie/trunk/client/src/main/resources/hive-action-0.5.xsd?rev=1481115&view=auto
==============================================================================
--- oozie/trunk/client/src/main/resources/hive-action-0.5.xsd (added)
+++ oozie/trunk/client/src/main/resources/hive-action-0.5.xsd Fri May 10 17:30:41 2013
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:hive="uri:oozie:hive-action:0.5" elementFormDefault="qualified"
+ targetNamespace="uri:oozie:hive-action:0.5">
+
+ <xs:element name="hive" type="hive:ACTION"/>
+
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="prepare" type="hive:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="hive:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="script" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="param" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="argument" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="PREPARE">
+ <xs:sequence>
+ <xs:element name="delete" type="hive:DELETE" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="mkdir" type="hive:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="DELETE">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="MKDIR">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+
+</xs:schema>
Modified: oozie/trunk/core/src/main/conf/oozie-site.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/conf/oozie-site.xml?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/main/conf/oozie-site.xml (original)
+++ oozie/trunk/core/src/main/conf/oozie-site.xml Fri May 10 17:30:41 2013
@@ -36,7 +36,7 @@
<property>
<name>oozie.service.SchemaService.wf.ext.schemas</name>
- <value>shell-action-0.1.xsd,shell-action-0.2.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd</value>
+ <value>shell-action-0.1.xsd,shell-action-0.2.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd</value>
</property>
<property>
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/DagEngine.java Fri May 10 17:30:41 2013
@@ -119,12 +119,12 @@ public class DagEngine extends BaseEngin
}
/**
- * Submit a pig/mapreduce job through HTTP.
+ * Submit a pig/hive/mapreduce job through HTTP.
* <p/>
* It validates configuration properties.
*
* @param conf job configuration.
- * @param jobType job type - can be "pig" or "mapreduce".
+ * @param jobType job type - can be "pig", "hive, or "mapreduce".
* @return the job Id.
* @throws DagEngineException thrown if the job could not be created.
*/
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java Fri May 10 17:30:41 2013
@@ -46,6 +46,7 @@ public class HiveActionExecutor extends
private static final String HIVE_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.HiveMain";
static final String HIVE_SCRIPT = "oozie.hive.script";
static final String HIVE_PARAMS = "oozie.hive.params";
+ static final String HIVE_ARGS = "oozie.hive.args";
public HiveActionExecutor() {
super("hive");
@@ -90,14 +91,23 @@ public class HiveActionExecutor extends
for (int i = 0; i < params.size(); i++) {
strParams[i] = params.get(i).getTextTrim();
}
+ String[] strArgs = null;
+ List<Element> eArgs = actionXml.getChildren("argument", ns);
+ if (eArgs != null && eArgs.size() > 0) {
+ strArgs = new String[eArgs.size()];
+ for (int i = 0; i < eArgs.size(); i++) {
+ strArgs[i] = eArgs.get(i).getTextTrim();
+ }
+ }
- setHiveScript(conf, scriptName, strParams);
+ setHiveScript(conf, scriptName, strParams, strArgs);
return conf;
}
- public static void setHiveScript(Configuration conf, String script, String[] params) {
+ public static void setHiveScript(Configuration conf, String script, String[] params, String[] args) {
conf.set(HIVE_SCRIPT, script);
MapReduceMain.setStrings(conf, HIVE_PARAMS, params);
+ MapReduceMain.setStrings(conf, HIVE_ARGS, args);
}
@Override
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHiveXCommand.java Fri May 10 17:30:41 2013
@@ -21,13 +21,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.oozie.client.XOozieClient;
import org.jdom.Namespace;
-/**
- * Created with IntelliJ IDEA.
- * User: bzhang
- * Date: 12/26/12
- * Time: 2:49 PM
- * To change this template use File | Settings | File Templates.
- */
public class SubmitHiveXCommand extends SubmitScriptLanguageXCommand {
public SubmitHiveXCommand(Configuration conf, String authToken) {
super("submitHive", "submitHive", conf, authToken);
@@ -41,7 +34,11 @@ public class SubmitHiveXCommand extends
return XOozieClient.HIVE_OPTIONS;
}
+ protected String getScriptParamters() {
+ return XOozieClient.HIVE_SCRIPT_PARAMS;
+ }
+
protected Namespace getSectionNamespace(){
- return Namespace.getNamespace("uri:oozie:hive-action:0.2");
+ return Namespace.getNamespace("uri:oozie:hive-action:0.5");
}
}
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitPigXCommand.java Fri May 10 17:30:41 2013
@@ -32,4 +32,8 @@ public class SubmitPigXCommand extends S
protected String getOptions(){
return XOozieClient.PIG_OPTIONS;
}
+
+ protected String getScriptParamters() {
+ return XOozieClient.PIG_SCRIPT_PARAMS;
+ }
}
\ No newline at end of file
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java Fri May 10 17:30:41 2013
@@ -28,29 +28,21 @@ import org.jdom.Namespace;
import java.util.ArrayList;
import java.util.List;
-/**
- * Created with IntelliJ IDEA.
- * User: bzhang
- * Date: 12/26/12
- * Time: 2:27 PM
- * To change this template use File | Settings | File Templates.
- */
-public class SubmitScriptLanguageXCommand extends SubmitHttpXCommand {
+public abstract class SubmitScriptLanguageXCommand extends SubmitHttpXCommand {
public SubmitScriptLanguageXCommand(String name, String type, Configuration conf, String authToken) {
super(name, type, conf, authToken);
}
- protected String getLanguageName() throws UnsupportedOperationException{
- throw new UnsupportedOperationException("subclass needs to implement this method");
- }
+ protected abstract String getLanguageName();
- protected String getOptions() throws UnsupportedOperationException{
- throw new UnsupportedOperationException("subclass needs to implement this method");
- }
+ protected abstract String getOptions();
+
+ protected abstract String getScriptParamters();
- protected Namespace getSectionNamespace(){
+ protected Namespace getSectionNamespace() {
return Namespace.getNamespace("uri:oozie:workflow:0.2");
}
+
private Element generateSection(Configuration conf, Namespace ns) {
String name = getLanguageName();
Element ele = new Element(name, ns);
@@ -63,8 +55,8 @@ public class SubmitScriptLanguageXComman
List<String> Dargs = new ArrayList<String>();
List<String> otherArgs = new ArrayList<String>();
- String[] Args = MapReduceMain.getStrings(conf, getOptions());
- for (String arg : Args) {
+ String[] args = MapReduceMain.getStrings(conf, getOptions());
+ for (String arg : args) {
if (arg.startsWith("-D")) {
Dargs.add(arg);
}
@@ -72,6 +64,7 @@ public class SubmitScriptLanguageXComman
otherArgs.add(arg);
}
}
+ String [] params = MapReduceMain.getStrings(conf, getScriptParamters());
// configuration section
if (Dargs.size() > 0) {
@@ -83,6 +76,13 @@ public class SubmitScriptLanguageXComman
script.addContent("dummy." + name);
ele.addContent(script);
+ // parameter section
+ for (String param : params) {
+ Element parameter = new Element("param", ns);
+ parameter.addContent(param);
+ ele.addContent(parameter);
+ }
+
// argument section
for (String arg : otherArgs) {
Element argument = new Element("argument", ns);
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java Fri May 10 17:30:41 2013
@@ -59,15 +59,16 @@ public class TestWorkflowXClient extends
conf.setProperty(OozieClient.LIBPATH, libPath.toString());
conf.setProperty(XOozieClient.JT, "localhost:9001");
conf.setProperty(XOozieClient.NN, "hdfs://localhost:9000");
+ String[] params = new String[]{"INPUT=input.txt"};
String pigScriptFile = getTestCaseDir() + "/test";
BufferedWriter writer = new BufferedWriter(new FileWriter(pigScriptFile));
- writer.write("a = load 'input.txt';\n dump a;");
+ writer.write("a = load '${INPUT}';\n dump a;");
writer.close();
assertEquals(MockDagEngineService.JOB_ID + wfCount + MockDagEngineService.JOB_ID_END,
- wc.submitScriptLanguage(conf, pigScriptFile, null, "pig"));
+ wc.submitScriptLanguage(conf, pigScriptFile, null, params, "pig"));
assertTrue(MockDagEngineService.started.get(wfCount));
return null;
@@ -88,16 +89,16 @@ public class TestWorkflowXClient extends
conf.setProperty(OozieClient.LIBPATH, libPath.toString());
conf.setProperty(XOozieClient.JT, "localhost:9001");
conf.setProperty(XOozieClient.NN, "hdfs://localhost:9000");
-
+ String[] params = new String[]{"NAME=test"};
String hiveScriptFile = getTestCaseDir() + "/test";
System.out.println(hiveScriptFile);
BufferedWriter writer = new BufferedWriter(new FileWriter(hiveScriptFile));
- writer.write("CREATE EXTERNAL TABLE test (a INT);");
+ writer.write("CREATE EXTERNAL TABLE ${NAME} (a INT);");
writer.close();
assertEquals(MockDagEngineService.JOB_ID + wfCount + MockDagEngineService.JOB_ID_END,
- wc.submitScriptLanguage(conf, hiveScriptFile, null, "hive"));
+ wc.submitScriptLanguage(conf, hiveScriptFile, null, params, "hive"));
assertTrue(MockDagEngineService.started.get(wfCount));
return null;
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java Fri May 10 17:30:41 2013
@@ -54,6 +54,8 @@ public class TestSubmitHiveXCommand exte
String hiveArgsStr = "-a aaa -b bbb -c ccc -M -Da=aaa -Db=bbb -param input=abc";
String[] args = hiveArgsStr.split(" ");
MapReduceMain.setStrings(conf, XOozieClient.HIVE_OPTIONS, args);
+ String[] params = new String[]{"INPUT=/some/path", "OUTPUT=/some/other/path", "abc=xyz"};
+ MapReduceMain.setStrings(conf, XOozieClient.HIVE_SCRIPT_PARAMS, params);
SubmitHiveXCommand submitHiveCmd = new SubmitHiveXCommand(conf, "token");
String xml = submitHiveCmd.getWorkflowXml(conf);
@@ -64,7 +66,7 @@ public class TestSubmitHiveXCommand exte
sb.append("<workflow-app xmlns=\"uri:oozie:workflow:0.2\" name=\"oozie-hive\">");
sb.append("<start to=\"hive1\" />");
sb.append("<action name=\"hive1\">");
- sb.append("<hive xmlns=\"uri:oozie:hive-action:0.2\">");
+ sb.append("<hive xmlns=\"uri:oozie:hive-action:0.5\">");
sb.append("<job-tracker>jobtracker</job-tracker>");
sb.append("<name-node>namenode</name-node>");
sb.append("<configuration>");
@@ -78,6 +80,9 @@ public class TestSubmitHiveXCommand exte
sb.append("</property>");
sb.append("</configuration>");
sb.append("<script>dummy.hive</script>");
+ sb.append("<param>INPUT=/some/path</param>");
+ sb.append("<param>OUTPUT=/some/other/path</param>");
+ sb.append("<param>abc=xyz</param>");
sb.append("<argument>-a</argument>");
sb.append("<argument>aaa</argument>");
sb.append("<argument>-b</argument>");
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java Fri May 10 17:30:41 2013
@@ -55,6 +55,8 @@ public class TestSubmitPigXCommand exten
String pigArgsStr = "-a aaa -b bbb -c ccc -M -Da=aaa -Db=bbb -param input=abc";
String[] args = pigArgsStr.split(" ");
MapReduceMain.setStrings(conf, XOozieClient.PIG_OPTIONS, args);
+ String[] params = new String[]{"INPUT=/some/path", "OUTPUT=/some/other/path", "abc=xyz"};
+ MapReduceMain.setStrings(conf, XOozieClient.PIG_SCRIPT_PARAMS, params);
SubmitPigXCommand submitPigCmd = new SubmitPigXCommand(conf, "token");
String xml = submitPigCmd.getWorkflowXml(conf);
@@ -79,6 +81,9 @@ public class TestSubmitPigXCommand exten
sb.append("</property>");
sb.append("</configuration>");
sb.append("<script>dummy.pig</script>");
+ sb.append("<param>INPUT=/some/path</param>");
+ sb.append("<param>OUTPUT=/some/other/path</param>");
+ sb.append("<param>abc=xyz</param>");
sb.append("<argument>-a</argument>");
sb.append("<argument>aaa</argument>");
sb.append("<argument>-b</argument>");
Modified: oozie/trunk/docs/src/site/twiki/DG_HiveActionExtension.twiki
URL: http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/DG_HiveActionExtension.twiki?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/DG_HiveActionExtension.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/DG_HiveActionExtension.twiki Fri May 10 17:30:41 2013
@@ -149,14 +149,123 @@ property =oozie.hive.log.level=. The def
---+++ AE.A Appendix A, Hive XML-Schema
+---++++ Hive Action Schema Version 0.5
+<verbatim>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:hive="uri:oozie:hive-action:0.5" elementFormDefault="qualified"
+ targetNamespace="uri:oozie:hive-action:0.5">
+.
+ <xs:element name="hive" type="hive:ACTION"/>
+.
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="prepare" type="hive:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="hive:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="script" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="param" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="argument" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="PREPARE">
+ <xs:sequence>
+ <xs:element name="delete" type="hive:DELETE" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="mkdir" type="hive:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="DELETE">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+.
+ <xs:complexType name="MKDIR">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+.
+</xs:schema>
+</verbatim>
+
+---++++ Hive Action Schema Version 0.4
+<verbatim>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:hive="uri:oozie:hive-action:0.4" elementFormDefault="qualified"
+ targetNamespace="uri:oozie:hive-action:0.4">
+.
+ <xs:element name="hive" type="hive:ACTION"/>
+.
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="prepare" type="hive:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="hive:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="script" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="param" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="PREPARE">
+ <xs:sequence>
+ <xs:element name="delete" type="hive:DELETE" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="mkdir" type="hive:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="DELETE">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+.
+ <xs:complexType name="MKDIR">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+.
+</xs:schema>
+</verbatim>
+
---++++ Hive Action Schema Version 0.3
<verbatim>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:hive="uri:oozie:hive-action:0.3" elementFormDefault="qualified"
targetNamespace="uri:oozie:hive-action:0.3">
-
+.
<xs:element name="hive" type="hive:ACTION"/>
-
+.
<xs:complexType name="ACTION">
<xs:sequence>
<xs:element name="job-tracker" type="xs:string" minOccurs="1" maxOccurs="1"/>
@@ -170,7 +279,7 @@ property =oozie.hive.log.level=. The def
<xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
-
+.
<xs:complexType name="CONFIGURATION">
<xs:sequence>
<xs:element name="property" minOccurs="1" maxOccurs="unbounded">
@@ -184,22 +293,22 @@ property =oozie.hive.log.level=. The def
</xs:element>
</xs:sequence>
</xs:complexType>
-
+.
<xs:complexType name="PREPARE">
<xs:sequence>
<xs:element name="delete" type="hive:DELETE" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="mkdir" type="hive:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
-
+.
<xs:complexType name="DELETE">
<xs:attribute name="path" type="xs:string" use="required"/>
</xs:complexType>
-
+.
<xs:complexType name="MKDIR">
<xs:attribute name="path" type="xs:string" use="required"/>
</xs:complexType>
-
+.
</xs:schema>
</verbatim>
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Fri May 10 17:30:41 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1353 hive CLI fails with -X argument (rkanter)
OOZIE-1303 CLI API for Bulk Monitoring (mona)
OOZIE-611 distcp action does not have documentation (rkanter)
OOZIE-1318 Action Main classes should be overridable via action configuration settings (rkanter)
Modified: oozie/trunk/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java (original)
+++ oozie/trunk/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java Fri May 10 17:30:41 2013
@@ -26,9 +26,11 @@ import java.io.OutputStream;
import java.lang.reflect.Field;
import java.net.URL;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
@@ -40,11 +42,26 @@ public class HiveMain extends LauncherMa
private static final Pattern[] HIVE_JOB_IDS_PATTERNS = {
Pattern.compile("Ended Job = (job_\\S*)")
};
+ private static final Set<String> DISALLOWED_HIVE_OPTIONS = new HashSet<String>();
public static final String HIVE_L4J_PROPS = "hive-log4j.properties";
public static final String HIVE_EXEC_L4J_PROPS = "hive-exec-log4j.properties";
public static final String HIVE_SITE_CONF = "hive-site.xml";
+ static {
+ DISALLOWED_HIVE_OPTIONS.add("-d");
+ DISALLOWED_HIVE_OPTIONS.add("--define");
+ DISALLOWED_HIVE_OPTIONS.add("-e");
+ DISALLOWED_HIVE_OPTIONS.add("-f");
+ DISALLOWED_HIVE_OPTIONS.add("-H");
+ DISALLOWED_HIVE_OPTIONS.add("--help");
+ DISALLOWED_HIVE_OPTIONS.add("--hiveconf");
+ DISALLOWED_HIVE_OPTIONS.add("--hivevar");
+ DISALLOWED_HIVE_OPTIONS.add("-s");
+ DISALLOWED_HIVE_OPTIONS.add("--silent");
+ DISALLOWED_HIVE_OPTIONS.add("-D");
+ }
+
public static void main(String[] args) throws Exception {
run(HiveMain.class, args);
}
@@ -229,6 +246,13 @@ public class HiveMain extends LauncherMa
arguments.add("-f");
arguments.add(scriptPath);
+ String[] hiveArgs = MapReduceMain.getStrings(hiveConf, HiveActionExecutor.HIVE_ARGS);
+ for (String hiveArg : hiveArgs) {
+ if (DISALLOWED_HIVE_OPTIONS.contains(hiveArg)) {
+ throw new RuntimeException("Error: Hive argument " + hiveArg + " is not supported");
+ }
+ arguments.add(hiveArg);
+ }
System.out.println("Hive command arguments :");
for (String arg : arguments) {
Modified: oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java?rev=1481115&r1=1481114&r2=1481115&view=diff
==============================================================================
--- oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java (original)
+++ oozie/trunk/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java Fri May 10 17:30:41 2013
@@ -28,6 +28,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
+import java.util.Properties;
public class TestHiveMain extends MainTestCase {
private SecurityManager SECURITY_MANAGER;
@@ -96,7 +97,7 @@ public class TestHiveMain extends MainTe
SharelibUtils.addToDistributedCache("hive", fs, getFsTestCaseDir(), jobConf);
HiveActionExecutor.setHiveScript(jobConf, script.toString(), new String[]{"IN=" + inputDir.toUri().getPath(),
- "OUT=" + outputDir.toUri().getPath()});
+ "OUT=" + outputDir.toUri().getPath()}, new String[] { "-v" });
File actionXml = new File(getTestCaseDir(), "action.xml");
OutputStream os = new FileOutputStream(actionXml);
@@ -107,6 +108,8 @@ public class TestHiveMain extends MainTe
URL url = Thread.currentThread().getContextClassLoader().getResource("HiveMain.txt");
File classPathDir = new File(url.getPath()).getParentFile();
assertTrue(classPathDir.exists());
+ Properties props = jobConf.toProperties();
+ assertEquals(props.getProperty("oozie.hive.args.size"), "1");
File hiveSite = new File(classPathDir, "hive-site.xml");
File outputDataFile = new File(getTestCaseDir(), "outputdata.properties");