You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2014/05/13 22:58:54 UTC
git commit: OOZIE-1825 Optimize wf_jobs protoconf storage (puru via
rohini)
Repository: oozie
Updated Branches:
refs/heads/master 83dec8b0e -> a0aa6fbdf
OOZIE-1825 Optimize wf_jobs protoconf storage (puru via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a0aa6fbd
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a0aa6fbd
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a0aa6fbd
Branch: refs/heads/master
Commit: a0aa6fbdfab47e803d36f8540ca28063907cb472
Parents: 83dec8b
Author: Rohini Palaniswamy <ro...@yahoo-inc.com>
Authored: Tue May 13 13:58:48 2014 -0700
Committer: Rohini Palaniswamy <ro...@yahoo-inc.com>
Committed: Tue May 13 13:58:48 2014 -0700
----------------------------------------------------------------------
.../oozie/service/WorkflowAppService.java | 14 +++---
.../oozie/command/wf/TestSubmitXCommand.java | 52 +++++++++++++++++++-
release-log.txt | 1 +
3 files changed, 59 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/a0aa6fbd/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
index c2f3836..05efefb 100644
--- a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
+++ b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
@@ -151,7 +151,6 @@ public abstract class WorkflowAppService implements Service {
throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
}
}
-
/**
* Create proto configuration. <p/> The proto configuration includes the user,group and the paths which need to be
* added to distributed cache. These paths include .jar,.so and the resource file paths.
@@ -168,9 +167,12 @@ public abstract class WorkflowAppService implements Service {
URI uri = new URI(jobConf.get(OozieClient.APP_PATH));
Configuration conf = has.createJobConf(uri.getAuthority());
+ XConfiguration protoConf = new XConfiguration();
+
String user = jobConf.get(OozieClient.USER_NAME);
conf.set(OozieClient.USER_NAME, user);
+ protoConf.set(OozieClient.USER_NAME, user);
FileSystem fs = has.createFileSystem(user, uri, conf);
@@ -227,7 +229,7 @@ public abstract class WorkflowAppService implements Service {
}
}
- conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()]));
+ protoConf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()]));
//Add all properties start with 'oozie.'
for (Map.Entry<String, String> entry : jobConf) {
@@ -235,14 +237,12 @@ public abstract class WorkflowAppService implements Service {
String name = entry.getKey();
String value = entry.getValue();
// if property already exists, should not overwrite
- if(conf.get(name) == null) {
- conf.set(name, value);
+ if(protoConf.get(name) == null) {
+ protoConf.set(name, value);
}
}
}
- XConfiguration retConf = new XConfiguration();
- XConfiguration.copy(conf, retConf);
- return retConf;
+ return protoConf;
}
catch (IOException ex) {
throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
http://git-wip-us.apache.org/repos/asf/oozie/blob/a0aa6fbd/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
index 5aff988..e8f77df 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
@@ -21,15 +21,19 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+import java.io.StringReader;
import java.net.URI;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.local.LocalOozie;
import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.command.CommandException;
+import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
+import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
import org.apache.oozie.service.Services;
+import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
import org.apache.oozie.store.WorkflowStore;
import org.apache.oozie.test.XDataTestCase;
@@ -237,6 +241,52 @@ public class TestSubmitXCommand extends XDataTestCase {
}
}
+ // It should not store Hadoop properties
+ public void testProtoConfStorage() throws Exception {
+ final OozieClient wfClient = LocalOozie.getClient();
+
+ Configuration conf = new XConfiguration();
+ String workflowUri = getTestCaseFileUri("workflow.xml");
+ String appXml = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='${appName}-foo'> " + "<start to='end' /> "
+ + "<end name='end' /> " + "</workflow-app>";
+
+ writeToFile(appXml, workflowUri);
+ conf.set(OozieClient.APP_PATH, workflowUri);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("appName", "var-app-name");
+ SubmitXCommand sc = new SubmitXCommand(conf);
+ final String jobId = sc.call();
+
+ waitFor(15 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED;
+ }
+ });
+ WorkflowJobBean wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId);
+ XConfiguration protoConf = new XConfiguration(new StringReader(wf.getProtoActionConf()));
+ //username, libpath, apppath
+ assertEquals(protoConf.size(), 2);
+ assertNull(protoConf.get(WorkflowAppService.APP_LIB_PATH_LIST));
+
+ new File(getTestCaseDir() + "/lib").mkdirs();
+ File.createTempFile("parentLibrary", ".jar", new File(getTestCaseDir() + "/lib"));
+ conf.set(OozieClient.APP_PATH, workflowUri);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("appName", "var-app-name");
+ sc = new SubmitXCommand(conf);
+ final String jobId1 = sc.call();
+
+ waitFor(15 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED;
+ }
+ });
+ wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId1);
+ protoConf = new XConfiguration(new StringReader(wf.getProtoActionConf()));
+ assertEquals(protoConf.size(), 3);
+ assertNotNull(protoConf.get(WorkflowAppService.APP_LIB_PATH_LIST));
+ }
+
private void writeToFile(String appXml, String appPath) throws IOException {
// TODO Auto-generated method stub
File wf = new File(URI.create(appPath));
http://git-wip-us.apache.org/repos/asf/oozie/blob/a0aa6fbd/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index c35ca7d..f5d0154 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1825 Optimize wf_jobs protoconf storage (puru via rohini)
OOZIE-1831 Oozie upgrade fails if workflow jobs are in running or suspended state (satish.mittal via rohini)
OOZIE-1690 TestShellActionExecutor#testEnvVar failed for Windows (omaliuvanchuk via rkanter)
OOZIE-1243 libtools dir should not include hadoop JARs (satish.mittal via rohini)