You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2015/09/01 23:43:41 UTC
oozie git commit: OOZIE-2347 Remove unnecessary new
Configuration()/new jobConf() calls from oozie
Repository: oozie
Updated Branches:
refs/heads/master 3f7a0c562 -> 16234592b
OOZIE-2347 Remove unnecessary new Configuration()/new jobConf() calls from oozie
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/16234592
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/16234592
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/16234592
Branch: refs/heads/master
Commit: 16234592b76b89aa1e7f5359e00d28ad138fe1ab
Parents: 3f7a0c5
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Tue Sep 1 14:45:16 2015 -0700
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Tue Sep 1 14:45:16 2015 -0700
----------------------------------------------------------------------
.../org/apache/oozie/action/hadoop/JavaActionExecutor.java | 8 +++++---
.../java/org/apache/oozie/service/HadoopAccessorService.java | 8 ++++++--
.../main/java/org/apache/oozie/service/ShareLibService.java | 5 ++---
core/src/main/java/org/apache/oozie/util/JobUtils.java | 7 ++-----
.../apache/oozie/action/hadoop/TestJavaActionExecutor.java | 2 +-
release-log.txt | 1 +
6 files changed, 17 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index bb2d3a6..5eacffe 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -642,9 +642,11 @@ public class JavaActionExecutor extends ActionExecutor {
if (confSet.contains(fileName)) {
Configuration jobXmlConf = shareLibService.getShareLibConf(actionShareLibName,
pathWithFragment);
- checkForDisallowedProps(jobXmlConf, actionLibPath.getName());
- XConfiguration.injectDefaults(jobXmlConf, conf);
- LOG.trace("Adding properties of " + actionLibPath + " to job conf");
+ if (jobXmlConf != null) {
+ checkForDisallowedProps(jobXmlConf, actionLibPath.getName());
+ XConfiguration.injectDefaults(jobXmlConf, conf);
+ LOG.trace("Adding properties of " + actionLibPath + " to job conf");
+ }
}
else {
// Filtering out duplicate jars or files
http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index b72b90f..116f382 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIden
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.security.token.Token;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.action.hadoop.JavaActionExecutor;
@@ -84,6 +83,8 @@ public class HadoopAccessorService implements Service {
protected static final String HADOOP_YARN_RM = "yarn.resourcemanager.address";
private static final Map<String, Text> mrTokenRenewers = new HashMap<String, Text>();
+ private static JobConf cachedJobConf;
+
private static final String DEFAULT_ACTIONNAME = "default";
private Set<String> jobTrackerWhitelist = new HashSet<String>();
@@ -146,6 +147,9 @@ public class HadoopAccessorService implements Service {
loadHadoopConfigs(conf);
preLoadActionConfigs(conf);
+ cachedJobConf = new JobConf();
+ //calling .size to invoke lazy loading of hadoop properties
+ cachedJobConf.size();
supportedSchemes = new HashSet<String>();
String[] schemesFromConf = ConfigurationService.getStrings(conf, SUPPORTED_FILESYSTEMS);
@@ -294,7 +298,7 @@ public class HadoopAccessorService implements Service {
* @return a JobConf with the corresponding site configuration for hostPort.
*/
public JobConf createJobConf(String hostPort) {
- JobConf jobConf = new JobConf();
+ JobConf jobConf = new JobConf(cachedJobConf);
XConfiguration.copy(getConfiguration(hostPort), jobConf);
jobConf.setBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, true);
return jobConf;
http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index b783379..81a9f2d 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -862,12 +862,11 @@ public class ShareLibService implements Service, Instrumentable {
}
public Configuration getShareLibConf(String inputKey, Path path) {
- Configuration conf = new Configuration();
if (shareLibConfigMap.containsKey(inputKey)) {
- conf = shareLibConfigMap.get(inputKey).get(path);
+ return shareLibConfigMap.get(inputKey).get(path);
}
- return conf;
+ return null;
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/main/java/org/apache/oozie/util/JobUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java
index 6bb1a00..79edcd4 100644
--- a/core/src/main/java/org/apache/oozie/util/JobUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java
@@ -145,14 +145,11 @@ public class JobUtils {
* @throws IOException
*/
public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) throws IOException {
- Configuration defaultConf = new Configuration();
- XConfiguration.copy(conf, defaultConf);
if (fs == null) {
- // it fails with conf, therefore we pass defaultConf instead
- fs = file.getFileSystem(defaultConf);
+ fs = file.getFileSystem(conf);
}
// Hadoop 0.20/1.x.
- if (defaultConf.get("yarn.resourcemanager.webapp.address") == null) {
+ if (conf.get("yarn.resourcemanager.webapp.address") == null) {
// Duplicate hadoop 1.x code to workaround MAPREDUCE-2361 in Hadoop 0.20
// Refer OOZIE-1806.
String filepath = file.toUri().getPath();
http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 079d7b8..8b867bb 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -2247,7 +2247,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// test .jar without fragment where app path is on same cluster as jar path
Path appJarPath = new Path("lib/a.jar");
Path appJarFullPath = new Path(appPath, appJarPath);
- conf.clear();
+ conf = new Configuration();
conf.set(WorkflowAppService.HADOOP_USER, getTestUser());
ae.addToCache(conf, appPath, appJarFullPath.toString(), false);
// assert that mapred.cache.files contains jar URI path (full on Hadoop-2)
http://git-wip-us.apache.org/repos/asf/oozie/blob/16234592/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 7174312..b2cc29b 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2347 Remove unnecessary new Configuration()/new jobConf() calls from oozie (puru)
OOZIE-2348 Recovery service keeps on recovering coord action of suspended jobs (puru)
OOZIE-2277 Honor oozie.action.sharelib.for.spark in Spark jobs (rkanter)
OOZIE-2322 Oozie Web UI doesn't work with Kerberos in Internet Explorer 10 or 11 and curl (rkanter)