You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/04 19:03:07 UTC
svn commit: r1442256 - in /oozie/branches/branch-3.3: ./
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/action/hadoop/ core/src/test/resources/
docs/src/site/twiki/ examples/src/main/apps/hive/
Author: rkanter
Date: Mon Feb 4 18:03:06 2013
New Revision: 1442256
URL: http://svn.apache.org/viewvc?rev=1442256&view=rev
Log:
OOZIE-1087 Remove requirement of hive-default.xml from Hive action (rkanter)
Removed:
oozie/branches/branch-3.3/core/src/test/resources/user-hive-default.xml
oozie/branches/branch-3.3/examples/src/main/apps/hive/my-hive-default.xml
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
oozie/branches/branch-3.3/docs/src/site/twiki/DG_HiveActionExtension.twiki
oozie/branches/branch-3.3/examples/src/main/apps/hive/workflow.xml
oozie/branches/branch-3.3/release-log.txt
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java Mon Feb 4 18:03:06 2013
@@ -19,21 +19,17 @@ package org.apache.oozie.action.hadoop;
import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
-import java.io.StringReader;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.WorkflowAction;
-import org.apache.oozie.util.XConfiguration;
-import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
public class HiveActionExecutor extends JavaActionExecutor {
- private static final String OOZIE_HIVE_DEFAULTS = "oozie.hive.defaults";
public HiveActionExecutor() {
super("hive");
@@ -53,30 +49,6 @@ public class HiveActionExecutor extends
return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, HiveMain.class.getName());
}
- public Configuration setupHiveDefault(Configuration conf, Path appPath, Element actionXml)
- throws ActionExecutorException {
- try {
- //Setting up hive-default.xml file if specified by the Hive action
- Element actionConf = actionXml.getChild("configuration", actionXml.getNamespace());
- if (actionConf != null) {
- String strConf = XmlUtils.prettyPrint(actionConf).toString();
- XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
- if (inlineConf.get(OOZIE_HIVE_DEFAULTS) != null) {
- Path hiveDefaults = new Path(inlineConf.get(OOZIE_HIVE_DEFAULTS));
- // hive-default.xml will be softlinked to the working dir which is in the launcher CP.
- // the softlink is done as 'oozie-user-hive-default.xml' and Oozie HiveMain class will
- // check if the Hive being used has a hive-default.xml or not, if not it will rename
- // it as hive-default.xml before invoking hive
- addToCache(conf, appPath, hiveDefaults + "#" + HiveMain.USER_HIVE_DEFAULT_FILE, false);
- }
- }
- return conf;
- }
- catch (Exception ex) {
- throw convertException(ex);
- }
- }
-
@Override
protected Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
throws ActionExecutorException {
@@ -84,8 +56,6 @@ public class HiveActionExecutor extends
super.setupLauncherConf(conf, actionXml, appPath, context);
Namespace ns = actionXml.getNamespace();
- setupHiveDefault(conf, appPath, actionXml);
-
String script = actionXml.getChild("script", ns).getTextTrim();
String scriptName = new Path(script).getName();
addToCache(conf, appPath, script + "#" + scriptName, false);
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java Mon Feb 4 18:03:06 2013
@@ -18,18 +18,14 @@
package org.apache.oozie.action.hadoop;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.regex.Pattern;
@@ -39,8 +35,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliDriver;
public class HiveMain extends LauncherMain {
- public static final String USER_HIVE_DEFAULT_FILE = "oozie-user-hive-default.xml";
-
private static final Pattern[] HIVE_JOB_IDS_PATTERNS = {
Pattern.compile("Ended Job = (job_\\S*)")
};
@@ -167,6 +161,7 @@ public class HiveMain extends LauncherMa
System.out.println();
System.out.println("Oozie Hive action configuration");
System.out.println("=================================================================");
+ System.out.println();
Configuration hiveConf = setUpHiveSite();
@@ -181,32 +176,6 @@ public class HiveMain extends LauncherMa
throw new RuntimeException("Hive script file [" + scriptPath + "] does not exist");
}
- // check if hive-default.xml is in the classpath, if not look for oozie-user-hive-default.xml
- // in the current directory (it will be there if the Hive action has the 'oozie.hive.defaults'
- // property) and rename it to hive-default.xml
- if (Thread.currentThread().getContextClassLoader().getResource("hive-default.xml") == null) {
- File userProvidedDefault = new File(USER_HIVE_DEFAULT_FILE);
- if (userProvidedDefault.exists()) {
- if (!userProvidedDefault.renameTo(new File("hive-default.xml"))) {
- throw new RuntimeException(
- "Could not rename user provided Hive defaults file to 'hive-default.xml'");
- }
- System.out.println("Using 'hive-default.xml' defined in the Hive action");
- }
- else {
- throw new RuntimeException(
- "Hive JAR does not bundle a 'hive-default.xml' and Hive action does not define one");
- }
- }
- else {
- System.out.println("Using 'hive-default.xml' defined in the Hive JAR");
- File userProvidedDefault = new File(USER_HIVE_DEFAULT_FILE);
- if (userProvidedDefault.exists()) {
- System.out.println("WARNING: Ignoring user provided Hive defaults");
- }
- }
- System.out.println();
-
String logFile = setUpHiveLog4J(hiveConf);
// print out current directory & its contents
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java Mon Feb 4 18:03:06 2013
@@ -67,17 +67,6 @@ public class SqoopActionExecutor extends
}
@Override
- Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
- throws ActionExecutorException {
- super.setupLauncherConf(conf, actionXml, appPath, context);
-
- HiveActionExecutor hiveAE = new HiveActionExecutor();
- hiveAE.setupHiveDefault(conf, appPath, actionXml);
-
- return conf;
- }
-
- @Override
@SuppressWarnings("unchecked")
Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
throws ActionExecutorException {
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java Mon Feb 4 18:03:06 2013
@@ -116,10 +116,6 @@ public class TestHiveActionExecutor exte
"<name>oozie.hive.log.level</name>" +
"<value>DEBUG</value>" +
"</property>" +
- "<property>" +
- "<name>oozie.hive.defaults</name>" +
- "<value>user-hive-default.xml</value>" +
- "</property>" +
"</configuration>" +
"<script>" + HIVE_SCRIPT_FILENAME + "</script>" +
"</hive>";
@@ -140,10 +136,6 @@ public class TestHiveActionExecutor exte
dataWriter.write(SAMPLE_DATA_TEXT);
dataWriter.close();
- InputStream is = IOUtils.getResourceAsStream("user-hive-default.xml", -1);
- OutputStream os = fs.create(new Path(getAppPath(), "user-hive-default.xml"));
- IOUtils.copyStream(is, os);
-
Context context = createContext(getActionXml());
final RunningJob launcherJob = submitAction(context);
String launcherId = context.getAction().getExternalId();
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java Mon Feb 4 18:03:06 2013
@@ -110,10 +110,6 @@ public class TestHiveMain extends MainTe
assertTrue(classPathDir.exists());
File hiveSite = new File(classPathDir, "hive-site.xml");
- InputStream is = IOUtils.getResourceAsStream("user-hive-default.xml", -1);
- os = new FileOutputStream(new File(classPathDir, "hive-default.xml"));
- IOUtils.copyStream(is, os);
-
File outputDataFile = new File(getTestCaseDir(), "outputdata.properties");
setSystemProperty("oozie.launcher.job.id", "" + System.currentTimeMillis());
Modified: oozie/branches/branch-3.3/docs/src/site/twiki/DG_HiveActionExtension.twiki
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/docs/src/site/twiki/DG_HiveActionExtension.twiki?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/docs/src/site/twiki/DG_HiveActionExtension.twiki (original)
+++ oozie/branches/branch-3.3/docs/src/site/twiki/DG_HiveActionExtension.twiki Mon Feb 4 18:03:06 2013
@@ -118,10 +118,6 @@ expressions.
<name>mapred.compress.map.output</name>
<value>true</value>
</property>
- <property>
- <name>oozie.hive.defaults</name>
- <value>/usr/foo/hive-0.6-default.xml</value>
- </property>
</configuration>
<script>myscript.q</script>
<param>InputDir=/home/tucu/input-data</param>
@@ -136,19 +132,8 @@ expressions.
---+++ Hive Default and Site Configuration Files
-All the properties defined in the =job-xml= and inline in the =configuration=
-element become the =hive-site.xml= that Hive will use.
-
-Hive (as of Hive 0.6) does not yet include a =hive-default.xml= file, it is the responsibility
-of the user to provide one. When using Oozie Hive action, the =hive-default.xml= file must be
-copied to HDFS and in the Hive action =configuration= section it must be set in the =oozie.hive.defaults=
-property. If a relative path is given, the path will be resolved within the workflow application
-directory.
-
-*NOTE:* When Hive starts bundling a =hive-default.xml= file within its JARs, Oozie will ignore
-the =hive-default.xml= file specified in the Hive action configuration.
-
-If a =hive-site.xml= file is not specified (or available in Hive JARs), the Oozie Hive action will fail.
+Hive (as of Hive 0.8) ignores a =hive-default.xml= file. As a result, Oozie (as of Oozie 3.4) ignores the =oozie.hive.defaults=
+property that was previously required by earlier versions of Oozie for the Hive action.
---+++ Hive Action Logging
Modified: oozie/branches/branch-3.3/examples/src/main/apps/hive/workflow.xml
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/examples/src/main/apps/hive/workflow.xml?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/examples/src/main/apps/hive/workflow.xml (original)
+++ oozie/branches/branch-3.3/examples/src/main/apps/hive/workflow.xml Mon Feb 4 18:03:06 2013
@@ -32,10 +32,6 @@
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
- <property>
- <name>oozie.hive.defaults</name>
- <value>my-hive-default.xml</value>
- </property>
</configuration>
<script>script.q</script>
<param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param>
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1442256&r1=1442255&r2=1442256&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Mon Feb 4 18:03:06 2013
@@ -1,5 +1,6 @@
-- Oozie 3.3.2 (unreleased)
+OOZIE-1087 Remove requirement of hive-default.xml from Hive action (rkanter)
OOZIE-1129 Add documentation for configurable filesystem support (rkanter)
OOZIE-1084 When use IBM jdk , UT TestCallbackServlet and TestHadoopELFunctions fail (zhujinwei via rkanter)
OOZIE-1127 Missed one services.destroy() in OOZIE-1114 (rkanter)