You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2015/07/28 19:17:46 UTC
oozie git commit: OOZIE-2308 Add support for bundle:conf() function
(kailongs via rohini)
Repository: oozie
Updated Branches:
refs/heads/master b5c1137ba -> 2d18d935d
OOZIE-2308 Add support for bundle:conf() function (kailongs via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2d18d935
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2d18d935
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2d18d935
Branch: refs/heads/master
Commit: 2d18d935d6661219613e49dc781a56da1de930b1
Parents: b5c1137
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Tue Jul 28 10:17:43 2015 -0700
Committer: Rohini Palaniswamy <ro...@apache.org>
Committed: Tue Jul 28 10:17:43 2015 -0700
----------------------------------------------------------------------
.../apache/oozie/bundle/BundleELFunctions.java | 29 ++++++++++
.../command/bundle/BundleSubmitXCommand.java | 24 ++++++--
core/src/main/resources/oozie-default.xml | 7 ++-
.../bundle/TestBundleSubmitXCommand.java | 3 +-
.../org/apache/oozie/test/XDataTestCase.java | 1 +
.../bundle-submit-job-with-functions.xml | 60 ++++++++++++++++++++
docs/src/site/twiki/BundleFunctionalSpec.twiki | 15 ++---
release-log.txt | 1 +
8 files changed, 126 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java b/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java
new file mode 100644
index 0000000..2c31ffa
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/bundle/BundleELFunctions.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.bundle;
+
+import org.apache.oozie.util.ELEvaluator;
+
+
+public class BundleELFunctions {
+ public static String bundle_conf(String property) {
+ ELEvaluator eval = ELEvaluator.getCurrent();
+ return (String) eval.getVariable(property);
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
index 5d4e0f0..3df37fa 100644
--- a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
@@ -45,12 +45,13 @@ import org.apache.oozie.command.CommandException;
import org.apache.oozie.command.PreconditionException;
import org.apache.oozie.command.SubmitTransitionXCommand;
import org.apache.oozie.executor.jpa.BundleJobQueryExecutor;
+import org.apache.oozie.service.ELService;
import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.SchemaService;
+import org.apache.oozie.service.SchemaService.SchemaName;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.UUIDService;
-import org.apache.oozie.service.SchemaService.SchemaName;
import org.apache.oozie.service.UUIDService.ApplicationType;
import org.apache.oozie.util.ConfigUtils;
import org.apache.oozie.util.DateUtils;
@@ -60,10 +61,10 @@ import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.InstrumentUtils;
import org.apache.oozie.util.LogUtils;
import org.apache.oozie.util.ParamChecker;
+import org.apache.oozie.util.ParameterVerifier;
import org.apache.oozie.util.PropertiesUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
-import org.apache.oozie.util.ParameterVerifier;
import org.jdom.Attribute;
import org.jdom.Element;
import org.jdom.JDOMException;
@@ -136,7 +137,7 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand {
}
conf = resolvedVarsConf;
- String resolvedJobXml = resolvedVars(jobXmlWithNoComment, conf);
+ String resolvedJobXml = resolvedVarsandFunctions(jobXmlWithNoComment, conf);
//verify the uniqueness of coord names
verifyCoordNameUnique(resolvedJobXml);
@@ -414,6 +415,18 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand {
return bundleBean;
}
+ public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
+ ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
+ setConfigToEval(eval, conf);
+ return eval;
+ }
+
+ private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
+ for (Map.Entry<String, String> entry : conf) {
+ eval.setVariable(entry.getKey(), entry.getValue().trim());
+ }
+ }
+
/**
* Resolve job xml with conf
*
@@ -422,9 +435,10 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand {
* @return resolved job xml
* @throws BundleJobException thrown if failed to resolve variables
*/
- private String resolvedVars(String bundleXml, Configuration conf) throws BundleJobException {
+ private String resolvedVarsandFunctions(String bundleXml, Configuration conf) throws BundleJobException {
+ ELEvaluator eval;
try {
- ELEvaluator eval = createEvaluator(conf);
+ eval = createELEvaluatorForGroup(conf, "bundle-submit");
return eval.evaluate(bundleXml, String.class);
}
catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 2f44827..12c5aa0 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -549,7 +549,7 @@
<!-- List of supported groups for ELService -->
<property>
<name>oozie.service.ELService.groups</name>
- <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout</value>
+ <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout,bundle-submit</value>
<description>List of groups for different ELServices</description>
</property>
@@ -1003,6 +1003,11 @@
</property>
<property>
+ <name>oozie.service.ELService.functions.bundle-submit</name>
+ <value>bundle:conf=org.apache.oozie.bundle.BundleELFunctions#bundle_conf</value>
+ </property>
+
+ <property>
<name>oozie.service.ELService.functions.coord-sla-submit</name>
<value>
coord:dataOut=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dataOut_echo,
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
index f793601..b883dc2 100644
--- a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
@@ -99,6 +99,7 @@ public class TestBundleSubmitXCommand extends XDataTestCase {
final XConfiguration jobConf = setUpBundle();
jobConf.set("coordName1", "coord1");
jobConf.set("coordName2", "coord2");
+ jobConf.set("coord1.starttime","2009-02-01T00:00Z");
BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf);
final BundleJobBean bundleBean = (BundleJobBean) command.getJob();
@@ -149,7 +150,7 @@ public class TestBundleSubmitXCommand extends XDataTestCase {
writeCoordXml(coordPath2, "coord-job-bundle.xml");
Path bundleAppPath = new Path(getFsTestCaseDir(), "bundle");
- String bundleAppXml = getBundleXml("bundle-submit-job.xml");
+ String bundleAppXml = getBundleXml("bundle-submit-job-with-functions.xml");
assertNotNull(bundleAppXml);
assertTrue(bundleAppXml.length() > 0);
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
index ac8f1f9..e6a7d9c 100644
--- a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
@@ -1477,6 +1477,7 @@ public abstract class XDataTestCase extends XHCatTestCase {
conf.set("nameNode", getNameNodeUri());
conf.set("coordName1", "coord1");
conf.set("coordName2", "coord2");
+ conf.set("coord1.starttime","2009-02-01T00:00Z");
BundleJobBean bundle = new BundleJobBean();
bundle.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE));
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/core/src/test/resources/bundle-submit-job-with-functions.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/bundle-submit-job-with-functions.xml b/core/src/test/resources/bundle-submit-job-with-functions.xml
new file mode 100644
index 0000000..094a06a
--- /dev/null
+++ b/core/src/test/resources/bundle-submit-job-with-functions.xml
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<bundle-app name='${appName}' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:bundle:0.2'>
+ <controls>
+ <kick-off-time>2099-02-01T00:00Z</kick-off-time>
+ </controls>
+ <coordinator name="${bundle:conf('coordName1')}" critical='true'>
+ <app-path>#app_path1</app-path>
+ <configuration>
+ <property>
+ <name>COORD_NAME</name>
+ <value>coord1</value>
+ </property>
+ <property>
+ <name>START_TIME</name>
+ <value>${bundle:conf('coord1.starttime')}</value>
+ </property>
+ <property>
+ <name>END_TIME</name>
+ <value>2009-02-20T23:59Z</value>
+ </property>
+ <property>
+ <name>testProperty</name>
+ <value>abc</value>
+ </property>
+ </configuration>
+ </coordinator>
+ <coordinator name='${coordName2}' critical='false'>
+ <app-path>#app_path2</app-path>
+ <configuration>
+ <property>
+ <name>COORD_NAME</name>
+ <value>coord2</value>
+ </property>
+ <property>
+ <name>START_TIME</name>
+ <value>2009-02-02T00:00Z</value>
+ </property>
+ <property>
+ <name>END_TIME</name>
+ <value>2009-02-23T23:59Z</value>
+ </property>
+ </configuration>
+ </coordinator>
+</bundle-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/docs/src/site/twiki/BundleFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/BundleFunctionalSpec.twiki b/docs/src/site/twiki/BundleFunctionalSpec.twiki
index b400c19..c9252c0 100644
--- a/docs/src/site/twiki/BundleFunctionalSpec.twiki
+++ b/docs/src/site/twiki/BundleFunctionalSpec.twiki
@@ -91,12 +91,13 @@ When all the coordinator jobs finish, oozie updates the bundle status accordingl
A bundle definition is defined in XML by a name, controls and one or more coordinator application specifications:
* *%BLUE% name: %ENDCOLOR%* The name for the bundle job.
- * *%BLUE% controls: %ENDCOLOR%* The control specification for the bundle.
- * *%BLUE% kick-off-time: %ENDCOLOR%* It defines when the bundle job should start and submit the coordinator applications. This field is optional and the default is *NOW* that means the job should start right-a-way.
- * *%BLUE% coordinator: %ENDCOLOR%* Coordinator application specification. There should be at least one coordinator application in any bundle.
- * *%BLUE% name: %ENDCOLOR%* Name of the coordinator application. It can be used for referring this application through bundle to control such as kill, suspend, rerun.
- * *%BLUE% app-path: %ENDCOLOR%* Path of the coordinator application definition in hdfs. This is a mandatory element.
+ * *%BLUE% controls: %ENDCOLOR%* The control specification for the bundle.
+ * *%BLUE% kick-off-time: %ENDCOLOR%* It defines when the bundle job should start and submit the coordinator applications. This field is optional and the default is *NOW* that means the job should start right-a-way.
+ * *%BLUE% coordinator: %ENDCOLOR%* Coordinator application specification. There should be at least one coordinator application in any bundle.
+ * *%BLUE% name: %ENDCOLOR%* Name of the coordinator application. It can be used for referring this application through bundle to control such as kill, suspend, rerun.
+ * *%BLUE% app-path: %ENDCOLOR%* Path of the coordinator application definition in hdfs. This is a mandatory element.
* *%BLUE% configuration: %ENDCOLOR%* A hadoop like configuration to parameterize corresponding coordinator application. This is optional.
+ * *%BLUE% Parameterization: %ENDCOLOR%* Configuration properties that are a valid Java identifier, [A-Za-z_][0-9A-Za-z_]*, are available as =${NAME}= variables within the bundle application definition. Configuration properties that are not a valid Java identifier, for example =job.tracker=, are available via the =${bundle:conf(String name)}= function. Valid Java identifier properties are available via this function as well.
*%PURPLE% Syntax: %ENDCOLOR%*
@@ -130,12 +131,12 @@ A bundle definition is defined in XML by a name, controls and one or more coordi
<controls>
<kick-off-time>${kickOffTime}</kick-off-time>
</controls>
- <coordinator name='coordJobFromBundle1' >
+ <coordinator name="${bundle:conf('coordName1')}" >
<app-path>${appPath}</app-path>
<configuration>
<property>
<name>startTime1</name>
- <value>${START_TIME}</value>
+ <value>${bundle:conf('coord1.startTime1')}</value>
</property>
<property>
<name>endTime1</name>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d18d935/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index fcf6d9d..89735d6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2308 Add support for bundle:conf() function (kailongs via rohini)
OOZIE-2315 TestOozieCLI.testshareLibUpdate_withSecurity fails with Hadoop 2 (rkanter)
OOZIE-2291 Hive2 workflow.xml.security should have "cred" in action tag instead of "hive2" tag (bzhang via rkanter)
OOZIE-2289 hive-jdbc dependency in core/pom.xml should be compile (bzhang)