You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ry...@apache.org on 2015/04/18 09:29:56 UTC
[1/2] oozie git commit: OOZIE-2206 Change Reaper mode on ChildReaper
in ZKLocksService (ryota)
Repository: oozie
Updated Branches:
refs/heads/master e311d1947 -> c94ff9210
OOZIE-2206 Change Reaper mode on ChildReaper in ZKLocksService (ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/8707cad2
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/8707cad2
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/8707cad2
Branch: refs/heads/master
Commit: 8707cad2faaee83525eb972dc17a0786b67f8097
Parents: e311d19
Author: egashira <ry...@yahoo.com>
Authored: Sat Apr 18 00:09:01 2015 -0700
Committer: egashira <ry...@yahoo.com>
Committed: Sat Apr 18 00:09:01 2015 -0700
----------------------------------------------------------------------
core/src/main/java/org/apache/oozie/service/ZKLocksService.java | 2 +-
release-log.txt | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/8707cad2/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ZKLocksService.java b/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
index 7fc4d17..e3a6bcf 100644
--- a/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
+++ b/core/src/main/java/org/apache/oozie/service/ZKLocksService.java
@@ -68,7 +68,7 @@ public class ZKLocksService extends MemoryLocksService implements Service, Instr
super.init(services);
try {
zk = ZKUtils.register(this);
- reaper = new ChildReaper(zk.getClient(), LOCKS_NODE, Reaper.Mode.REAP_INDEFINITELY, getExecutorService(),
+ reaper = new ChildReaper(zk.getClient(), LOCKS_NODE, Reaper.Mode.REAP_UNTIL_GONE, getExecutorService(),
ConfigurationService.getInt(services.getConf(), REAPING_THRESHOLD) * 1000, REAPING_LEADER_PATH);
reaper.start();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/8707cad2/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 9d02e41..6fa00ca 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-2206 Change Reaper mode on ChildReaper in ZKLocksService (ryota)
OOZIE-2209 Oozie jobs to set "java.io.tmpdir" to "./tmp" (kailongs via rohini)
OOZIE-2051 Allows multiple default action configuration files per action (venkatnrangan via bzhang)
OOZIE-2170 Oozie should automatically set configs to make Spark jobs show up in the Spark History Server (rkanter)
[2/2] oozie git commit: OOZIE-2205 add option to load
default/site.xml to actionConf on compute node (ryota)
Posted by ry...@apache.org.
OOZIE-2205 add option to load default/site.xml to actionConf on compute node (ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c94ff921
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c94ff921
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c94ff921
Branch: refs/heads/master
Commit: c94ff9210998c608d05fd4de35b432a25ea7f185
Parents: 8707cad
Author: egashira <ry...@yahoo.com>
Authored: Sat Apr 18 00:13:24 2015 -0700
Committer: egashira <ry...@yahoo.com>
Committed: Sat Apr 18 00:13:24 2015 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/HiveActionExecutor.java | 11 +++++++++
.../oozie/action/hadoop/JavaActionExecutor.java | 21 +++++++++++++---
.../oozie/action/hadoop/PigActionExecutor.java | 13 ++++++++++
.../oozie/service/HadoopAccessorService.java | 1 +
core/src/main/resources/oozie-default.xml | 13 ++++++++++
release-log.txt | 1 +
.../action/hadoop/TestHiveActionExecutor.java | 26 ++++++++++++++++++++
.../action/hadoop/TestPigActionExecutor.java | 15 +++++++++++
8 files changed, 97 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
index dbb50ba..bc5b033 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
@@ -30,11 +30,15 @@ import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.action.ActionExecutor.Context;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.XOozieClient;
+import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.HadoopAccessorException;
+import org.apache.oozie.service.HadoopAccessorService;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
@@ -138,4 +142,11 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor {
return new String[]{"hive-site.xml"};
}
+ @Override
+ protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) {
+ boolean loadDefaultResources = ConfigurationService
+ .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES);
+ JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources);
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 163956e..eb2dbdb 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -82,7 +82,7 @@ import org.apache.oozie.hadoop.utils.HadoopShims;
public class JavaActionExecutor extends ActionExecutor {
- private static final String HADOOP_USER = "user.name";
+ protected static final String HADOOP_USER = "user.name";
public static final String HADOOP_JOB_TRACKER = "mapred.job.tracker";
public static final String HADOOP_JOB_TRACKER_2 = "mapreduce.jobtracker.address";
public static final String HADOOP_YARN_RM = "yarn.resourcemanager.address";
@@ -203,10 +203,20 @@ public class JavaActionExecutor extends ActionExecutor {
}
public JobConf createBaseHadoopConf(Context context, Element actionXml) {
+ return createBaseHadoopConf(context, actionXml, true);
+ }
+
+ protected JobConf createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) {
Namespace ns = actionXml.getNamespace();
String jobTracker = actionXml.getChild("job-tracker", ns).getTextTrim();
String nameNode = actionXml.getChild("name-node", ns).getTextTrim();
- JobConf conf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
+ JobConf conf = null;
+ if (loadResources) {
+ conf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
+ }
+ else {
+ conf = new JobConf(false);
+ }
conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER));
conf.set(HADOOP_JOB_TRACKER, jobTracker);
conf.set(HADOOP_JOB_TRACKER_2, jobTracker);
@@ -216,6 +226,10 @@ public class JavaActionExecutor extends ActionExecutor {
return conf;
}
+ protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) {
+ return createBaseHadoopConf(context, actionXml);
+ }
+
private void injectLauncherProperties(Configuration srcConf, Configuration launcherConf) {
for (Map.Entry<String, String> entry : srcConf) {
if (entry.getKey().startsWith("oozie.launcher.")) {
@@ -512,7 +526,6 @@ public class JavaActionExecutor extends ActionExecutor {
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_JOB_TRACKER), getType());
XConfiguration.injectDefaults(actionDefaults, actionConf);
-
has.checkSupportedFilesystem(appPath.toUri());
// Set the Java Main Class for the Java action to give to the Java launcher
@@ -1009,7 +1022,7 @@ public class JavaActionExecutor extends ActionExecutor {
Element actionXml = XmlUtils.parseXml(action.getConf());
// action job configuration
- Configuration actionConf = createBaseHadoopConf(context, actionXml);
+ Configuration actionConf = loadHadoopDefaultResources(context, actionXml);
setupActionConf(actionConf, context, actionXml, appPathRoot);
LOG.debug("Setting LibFilesArchives ");
setLibFilesArchives(context, actionXml, appPathRoot, actionConf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
index 4d6a1b4..8b2dc16 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
@@ -20,9 +20,15 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.action.ActionExecutor.Context;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.WorkflowAppService;
import org.jdom.Element;
import org.jdom.Namespace;
import org.jdom.JDOMException;
@@ -120,4 +126,11 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor {
return XOozieClient.PIG_SCRIPT;
}
+ @Override
+ protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) {
+ boolean loadDefaultResources = ConfigurationService
+ .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES);
+ JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources);
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index 102699b..6a3f045 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -68,6 +68,7 @@ public class HadoopAccessorService implements Service {
public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist";
public static final String HADOOP_CONFS = CONF_PREFIX + "hadoop.configurations";
public static final String ACTION_CONFS = CONF_PREFIX + "action.configurations";
+ public static final String ACTION_CONFS_LOAD_DEFAULT_RESOURCES = ACTION_CONFS + ".load.default.resources";
public static final String KERBEROS_AUTH_ENABLED = CONF_PREFIX + "kerberos.enabled";
public static final String KERBEROS_KEYTAB = CONF_PREFIX + "keytab.file";
public static final String KERBEROS_PRINCIPAL = CONF_PREFIX + "kerberos.principal";
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index ffff7c3..ecc0d47 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1877,6 +1877,19 @@
</description>
</property>
+ <property>
+ <name>oozie.service.HadoopAccessorService.action.configurations.load.default.resources</name>
+ <value>true</value>
+ <description>
+ true means that default and site xml files of hadoop (core-default, core-site,
+ hdfs-default, hdfs-site, mapred-default, mapred-site, yarn-default, yarn-site)
+ are parsed into actionConf on Oozie server. false means that site xml files are
+ not loaded on server, instead loaded on launcher node.
+ This is only done for pig and hive actions which handle loading those files
+ automatically from the classpath on launcher task. It defaults to true.
+ </description>
+ </property>
+
<!-- Credentials -->
<property>
<name>oozie.credentials.credentialclasses</name>
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 6fa00ca..8586dc7 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-2205 add option to load default/site.xml to actionConf on compute node (ryota)
OOZIE-2206 Change Reaper mode on ChildReaper in ZKLocksService (ryota)
OOZIE-2209 Oozie jobs to set "java.io.tmpdir" to "./tmp" (kailongs via rohini)
OOZIE-2051 Allows multiple default action configuration files per action (venkatnrangan via bzhang)
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
index c28839c..a11d409 100644
--- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
+++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
@@ -38,7 +39,9 @@ import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context;
import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.WorkflowAppService;
@@ -227,4 +230,27 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase {
return new Context(wf, action);
}
+ public void testActionConfLoadDefaultResources() throws Exception {
+ ConfigurationService.setBoolean(
+ "oozie.service.HadoopAccessorService.action.configurations.load.default.resources", false);
+ Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME);
+ Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME);
+
+ FileSystem fs = getFileSystem();
+ Path script = new Path(getAppPath(), HIVE_SCRIPT_FILENAME);
+ Writer scriptWriter = new OutputStreamWriter(fs.create(script));
+ scriptWriter.write(getHiveScript(inputDir.toString(), outputDir.toString()));
+ scriptWriter.close();
+
+ Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
+ dataWriter.write(SAMPLE_DATA_TEXT);
+ dataWriter.close();
+
+ Context context = createContext(getActionXml());
+ submitAction(context);
+ FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML));
+ XConfiguration conf = new XConfiguration();
+ conf.addResource(os);
+ assertNull(conf.get("oozie.HadoopAccessorService.created"));
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
index 039bd84..4b9a4e8 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
@@ -20,6 +20,7 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
@@ -29,6 +30,7 @@ import org.apache.hadoop.mapred.JobID;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.Services;
@@ -247,6 +249,19 @@ public class TestPigActionExecutor extends ActionExecutorTestCase {
}
+ public void testActionConfLoadDefaultResources() throws Exception {
+ ConfigurationService.setBoolean(
+ "oozie.service.HadoopAccessorService.action.configurations.load.default.resources", false);
+ String actionXml = setPigActionXml(PIG_SCRIPT, true);
+ Context context = createContext(actionXml);
+ submitAction(context);
+ FileSystem fs = getFileSystem();
+ FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML));
+ XConfiguration conf = new XConfiguration();
+ conf.addResource(os);
+ assertNull(conf.get("oozie.HadoopAccessorService.created"));
+ }
+
/*
* Test the Hadoop IDs obtained from the Pig job
*/