You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2013/04/24 21:51:07 UTC
svn commit: r1471613 - in /oozie/branches/branch-4.0: ./
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/action/hadoop/
Author: virag
Date: Wed Apr 24 19:51:06 2013
New Revision: 1471613
URL: http://svn.apache.org/r1471613
Log:
OOZIE-1146 FileSystem used by prepare sections should use the configuration of the action (rohini via virag)
Modified:
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
oozie/branches/branch-4.0/release-log.txt
Modified: oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java?rev=1471613&r1=1471612&r2=1471613&view=diff
==============================================================================
--- oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java (original)
+++ oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HCatLauncherURIHandler.java Wed Apr 24 19:51:06 2013
@@ -22,6 +22,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -68,7 +69,12 @@ public class HCatLauncherURIHandler impl
}
private HCatClient getHCatClient(URI uri, Configuration conf) throws LauncherException {
- final HiveConf hiveConf = new HiveConf(conf, this.getClass());
+ // Do not use the constructor public HiveConf(Configuration other, Class<?> cls)
+ // It overwrites the values in conf with default values
+ final HiveConf hiveConf = new HiveConf();
+ for (Entry<String, String> entry : conf) {
+ hiveConf.set(entry.getKey(), entry.getValue());
+ }
String serverURI = getMetastoreConnectURI(uri);
if (!serverURI.equals("")) {
hiveConf.set("hive.metastore.local", "false");
Modified: oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java?rev=1471613&r1=1471612&r2=1471613&view=diff
==============================================================================
--- oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java (original)
+++ oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java Wed Apr 24 19:51:06 2013
@@ -23,6 +23,7 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
+import java.lang.reflect.Field;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@@ -33,6 +34,7 @@ import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliDriver;
+import org.apache.hadoop.hive.conf.HiveConf;
public class HiveMain extends LauncherMain {
private static final Pattern[] HIVE_JOB_IDS_PATTERNS = {
@@ -154,6 +156,14 @@ public class HiveMain extends LauncherMa
System.out.flush();
System.out.println("------------------------");
System.out.println();
+
+ // Reset the hiveSiteURL static variable as we just created hive-site.xml.
+ // If prepare block had a drop partition it would have been initialized to null.
+ Field declaredField = HiveConf.class.getDeclaredField("hiveSiteURL");
+ if (declaredField != null) {
+ declaredField.setAccessible(true);
+ declaredField.set(null, HiveConf.class.getClassLoader().getResource("hive-site.xml"));
+ }
return hiveConf;
}
Modified: oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1471613&r1=1471612&r2=1471613&view=diff
==============================================================================
--- oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (original)
+++ oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java Wed Apr 24 19:51:06 2013
@@ -669,7 +669,10 @@ public class LauncherMapper<K1, V1, K2,
String prepareXML = getJobConf().get(ACTION_PREPARE_XML);
if (prepareXML != null) {
if (!prepareXML.equals("")) {
- PrepareActionsDriver.doOperations(prepareXML, getJobConf());
+ Configuration actionConf = new Configuration(getJobConf());
+ String actionXml = System.getProperty("oozie.action.conf.xml");
+ actionConf.addResource(new Path("file:///", actionXml));
+ PrepareActionsDriver.doOperations(prepareXML, actionConf);
} else {
System.out.println("There are no prepare actions to execute.");
}
Modified: oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1471613&r1=1471612&r2=1471613&view=diff
==============================================================================
--- oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java (original)
+++ oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java Wed Apr 24 19:51:06 2013
@@ -847,6 +847,16 @@ public class TestJavaActionExecutor exte
"<mkdir path='" + mkdir + "'/>" +
"<delete path='" + delete + "'/>" +
"</prepare>" +
+ "<configuration>" +
+ "<property>" +
+ "<name>dfs.umaskmode</name>" +
+ "<value>026</value>" +
+ "</property>" +
+ "<property>" +
+ "<name>fs.hdfs.impl.disable.cache</name>" +
+ "<value>true</value>" +
+ "</property>" +
+ "</configuration>" +
"<main-class>" + LauncherMainTester.class.getName() + "</main-class>" +
"</java>";
Context context = createContext(actionXml, null);
@@ -867,6 +877,8 @@ public class TestJavaActionExecutor exte
assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
assertTrue(fs.exists(mkdir));
+ // Check if the action configuration is applied in the prepare block
+ assertEquals("rwxr-x--x", fs.getFileStatus(mkdir).getPermission().toString());
assertFalse(fs.exists(delete));
}
Modified: oozie/branches/branch-4.0/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-4.0/release-log.txt?rev=1471613&r1=1471612&r2=1471613&view=diff
==============================================================================
--- oozie/branches/branch-4.0/release-log.txt (original)
+++ oozie/branches/branch-4.0/release-log.txt Wed Apr 24 19:51:06 2013
@@ -1,5 +1,6 @@
-- Oozie 4.0.0 (unreleased)
+OOZIE-1146 FileSystem used by prepare sections should use the configuration of the action (rohini via virag)
OOZIE-1215 add note of using escape for oozie jobs filters in doc (egashira via rkanter)
OOZIE-1331 URIHandlerService not allowing relative path for URI's (virag)
OOZIE-1332 Flakey test TestActionCheckXCommand.testActionCheckTransientDuringMRAction (rkanter)