You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2015/11/18 02:56:40 UTC
[1/2] oozie git commit: OOZIE-2168 Oozie flow and action names have
50 char limit (akshayrai09, me.venkatr via rkanter)
Repository: oozie
Updated Branches:
refs/heads/master f32a0fc5f -> 04bd49699
OOZIE-2168 Oozie flow and action names have 50 char limit (akshayrai09, me.venkatr via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2d031122
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2d031122
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2d031122
Branch: refs/heads/master
Commit: 2d03112213c46b2fe3180b4c0a1cbe00f26e03cb
Parents: f32a0fc
Author: Robert Kanter <rk...@cloudera.com>
Authored: Tue Nov 17 17:54:16 2015 -0800
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Tue Nov 17 17:54:16 2015 -0800
----------------------------------------------------------------------
.../org/apache/oozie/service/UUIDService.java | 23 ++++++----
.../org/apache/oozie/util/ParamChecker.java | 6 +--
.../org/apache/oozie/util/TestParamChecker.java | 48 ++++++++++++++++++++
.../wf-schema-action-name-too-long.xml | 4 +-
release-log.txt | 1 +
5 files changed, 69 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d031122/core/src/main/java/org/apache/oozie/service/UUIDService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/UUIDService.java b/core/src/main/java/org/apache/oozie/service/UUIDService.java
index b0cf0ad..313f9dd 100644
--- a/core/src/main/java/org/apache/oozie/service/UUIDService.java
+++ b/core/src/main/java/org/apache/oozie/service/UUIDService.java
@@ -41,6 +41,10 @@ public class UUIDService implements Service {
public static final String CONF_GENERATOR = CONF_PREFIX + "generator";
+ public static final int MAX_OOZIE_JOB_ID_LEN = 40;
+
+ public static final int MAX_ACTION_ID_LEN = 255;
+
protected String startTime;
private AtomicLong counter;
private String systemId;
@@ -104,8 +108,11 @@ public class UUIDService implements Service {
/**
* Create a unique ID.
*
- * @param type: Type of Id. Generally 'C' for Coordinator and 'W' for Workflow.
- * @return unique ID.
+ * @param type: Type of Id. Generally 'C' for Coordinator, 'W' for Workflow and 'B' for Bundle.
+ * @return unique ID, id = "${sequence}-${systemId}-[C|W|B]" where,
+ * sequence is ${padded_counter}-${startTime} whose length is exactly 7 + 1 + 15 = 23 characters.
+ * systemId is the value defined in the {@link #CONF_SYSTEM_ID} configuration property.
+ * Unique ID Example: 0007728-150515180312570-oozie-oozi-W
*/
public String generateId(ApplicationType type) {
StringBuilder sb = new StringBuilder();
@@ -113,9 +120,9 @@ public class UUIDService implements Service {
sb.append(getSequence());
sb.append('-').append(systemId);
sb.append('-').append(type.getType());
- // limitation due to current DB schema for action ID length (100)
- if (sb.length() > 40) {
- throw new RuntimeException(XLog.format("ID exceeds limit of 40 characters, [{0}]", sb));
+ // limited to MAX_OOZIE_JOB_ID_LEN as this partial id will be stored in the Action Id field with db schema limit of 255.
+ if (sb.length() > MAX_OOZIE_JOB_ID_LEN) {
+ throw new RuntimeException(XLog.format("ID exceeds limit of " + MAX_OOZIE_JOB_ID_LEN + " characters, [{0}]", sb));
}
return sb.toString();
}
@@ -160,9 +167,9 @@ public class UUIDService implements Service {
public String generateChildId(String id, String childName) {
id = ParamChecker.notEmpty(id, "id") + "@" + ParamChecker.notEmpty(childName, "childName");
- // limitation due to current DB schema for action ID length (100)
- if (id.length() > 95) {
- throw new RuntimeException(XLog.format("Child ID exceeds limit of 95 characters, [{0}]", id));
+ // limitation due to current DB schema for action ID length (255)
+ if (id.length() > MAX_ACTION_ID_LEN) {
+ throw new RuntimeException(XLog.format("Child ID exceeds limit of " + MAX_ACTION_ID_LEN + " characters, [{0}]", id));
}
return id;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d031122/core/src/main/java/org/apache/oozie/util/ParamChecker.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/ParamChecker.java b/core/src/main/java/org/apache/oozie/util/ParamChecker.java
index 381e7e4..8debeef 100644
--- a/core/src/main/java/org/apache/oozie/util/ParamChecker.java
+++ b/core/src/main/java/org/apache/oozie/util/ParamChecker.java
@@ -31,6 +31,8 @@ import java.text.ParseException;
*/
public class ParamChecker {
+ private static final int MAX_NODE_NAME_LEN = 128;
+
/**
* Check that a value is not null. If null throws an IllegalArgumentException.
*
@@ -106,10 +108,8 @@ public class ParamChecker {
return list;
}
- private static final int MAX_NODE_NAME_LEN = 50;
-
/**
- * Check that the given string is a valid action name [a-zA-Z_][0-9a-zA-Z_\-]* and not longer than 50 chars.
+ * Check that the given string is a valid action name [a-zA-Z_][0-9a-zA-Z_\-]* and not longer than 128 chars.
*
* @param actionName string to validate is a token.
* @return the given string.
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d031122/core/src/test/java/org/apache/oozie/util/TestParamChecker.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/util/TestParamChecker.java b/core/src/test/java/org/apache/oozie/util/TestParamChecker.java
index 4b4f317..76f2a9c 100644
--- a/core/src/test/java/org/apache/oozie/util/TestParamChecker.java
+++ b/core/src/test/java/org/apache/oozie/util/TestParamChecker.java
@@ -239,4 +239,52 @@ public class TestParamChecker extends XTestCase {
}
}
+ public void testValidateActionName() {
+ String actionName = "actionName";
+ ParamChecker.validateActionName(actionName);
+
+ actionName = "actionName01";
+ ParamChecker.validateActionName(actionName);
+
+ actionName = "actionName01_02";
+ ParamChecker.validateActionName(actionName);
+
+ actionName = "actionName01_02-test";
+ ParamChecker.validateActionName(actionName);
+
+ // actionName with = 128 chars
+ StringBuilder sb = new StringBuilder();
+ sb.append("a");
+ for (int i = 0; i < 127; i++) {
+ sb.append(i % 10);
+ }
+ ParamChecker.validateActionName(sb.toString());
+
+ try {
+ actionName = "1actionName";
+ ParamChecker.validateActionName(actionName);
+ fail();
+ } catch (IllegalArgumentException ex) {
+ }
+
+ try {
+ actionName = "-actionName";
+ ParamChecker.validateActionName(actionName);
+ fail();
+ } catch (IllegalArgumentException ex) {
+ }
+
+ try {
+ // actionName with > 128 chars
+ sb.setLength(0);
+ sb.append("ab");
+ for (int i = 0; i < 128; i++) {
+ sb.append(i);
+ }
+ ParamChecker.validateActionName(sb.toString());
+ fail();
+ } catch (IllegalArgumentException ex) {
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d031122/core/src/test/resources/wf-schema-action-name-too-long.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/wf-schema-action-name-too-long.xml b/core/src/test/resources/wf-schema-action-name-too-long.xml
index a8e939f..ae095ef 100644
--- a/core/src/test/resources/wf-schema-action-name-too-long.xml
+++ b/core/src/test/resources/wf-schema-action-name-too-long.xml
@@ -16,6 +16,6 @@
limitations under the License.
-->
<workflow-app xmlns="uri:oozie:workflow:0.1" name="test-wf">
- <start to="a1234567890123456789012345678901234567890123456789x"/>
- <end name="a1234567890123456789012345678901234567890123456789x"/>
+ <start to="a1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567x"/>
+ <end name="a1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567x"/>
</workflow-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2d031122/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index e6da494..d1467d2 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2168 Oozie flow and action names have 50 char limit (akshayrai09, me.venkatr via rkanter)
OOZIE-2346 Add sub-workflow information like the super-parent id and workflow depth into the 'oozie.job.info' property (akshayrai09 via puru)
OOZIE-2303 Typo in documentation (lars_francke via rohini)
OOZIE-2328 Coordinator endtime change should check if the last action is in database (kailongs via puru)
[2/2] oozie git commit: OOZIE-2401 Typo in twiki docs with
FilesAchives instead of FilesArchives (anthony@cloudera.com via rkanter)
Posted by rk...@apache.org.
OOZIE-2401 Typo in twiki docs with FilesAchives instead of FilesArchives (anthony@cloudera.com via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/04bd4969
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/04bd4969
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/04bd4969
Branch: refs/heads/master
Commit: 04bd4969968821cf24592d3ad7becaf5755ab871
Parents: 2d03112
Author: Robert Kanter <rk...@cloudera.com>
Authored: Tue Nov 17 17:56:30 2015 -0800
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Tue Nov 17 17:56:30 2015 -0800
----------------------------------------------------------------------
docs/src/site/twiki/DG_Hive2ActionExtension.twiki | 2 +-
docs/src/site/twiki/DG_HiveActionExtension.twiki | 2 +-
docs/src/site/twiki/DG_ShellActionExtension.twiki | 2 +-
docs/src/site/twiki/DG_SqoopActionExtension.twiki | 2 +-
docs/src/site/twiki/WorkflowFunctionalSpec.twiki | 6 +++---
release-log.txt | 1 +
6 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/04bd4969/docs/src/site/twiki/DG_Hive2ActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_Hive2ActionExtension.twiki b/docs/src/site/twiki/DG_Hive2ActionExtension.twiki
index 4cd588e..094307f 100644
--- a/docs/src/site/twiki/DG_Hive2ActionExtension.twiki
+++ b/docs/src/site/twiki/DG_Hive2ActionExtension.twiki
@@ -29,7 +29,7 @@ in the =job-xml= file.
As with Hadoop =map-reduce= jobs, it is possible to add files and
archives in order to make them available to Beeline. Refer to the
-[WorkflowFunctionalSpec#FilesAchives][Adding Files and Archives for the Job]
+[WorkflowFunctionalSpec#FilesArchives][Adding Files and Archives for the Job]
section for more information about this feature.
Oozie Hive 2 action supports Hive scripts with parameter variables, their
http://git-wip-us.apache.org/repos/asf/oozie/blob/04bd4969/docs/src/site/twiki/DG_HiveActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_HiveActionExtension.twiki b/docs/src/site/twiki/DG_HiveActionExtension.twiki
index 93e122b..a0d7b57 100644
--- a/docs/src/site/twiki/DG_HiveActionExtension.twiki
+++ b/docs/src/site/twiki/DG_HiveActionExtension.twiki
@@ -35,7 +35,7 @@ must not be present in the inline configuration.
As with Hadoop =map-reduce= jobs, it is possible to add files and
archives in order to make them available to the Hive job. Refer to the
-[WorkflowFunctionalSpec#FilesAchives][Adding Files and Archives for the Job]
+[WorkflowFunctionalSpec#FilesArchives][Adding Files and Archives for the Job]
section for more information about this feature.
Oozie Hive action supports Hive scripts with parameter variables, their
http://git-wip-us.apache.org/repos/asf/oozie/blob/04bd4969/docs/src/site/twiki/DG_ShellActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_ShellActionExtension.twiki b/docs/src/site/twiki/DG_ShellActionExtension.twiki
index dc856f5..4d8a6b1 100644
--- a/docs/src/site/twiki/DG_ShellActionExtension.twiki
+++ b/docs/src/site/twiki/DG_ShellActionExtension.twiki
@@ -35,7 +35,7 @@ must not be present in the inline configuration.
As with Hadoop =map-reduce= jobs, it is possible to add files and
archives in order to make them available to the Shell job. Refer to the
-[WorkflowFunctionalSpec#FilesAchives][Adding Files and Archives for the Job]
+[WorkflowFunctionalSpec#FilesArchives][Adding Files and Archives for the Job]
section for more information about this feature.
The output (STDOUT) of the Shell job can be made available to the workflow job after the Shell job ends. This information
http://git-wip-us.apache.org/repos/asf/oozie/blob/04bd4969/docs/src/site/twiki/DG_SqoopActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_SqoopActionExtension.twiki b/docs/src/site/twiki/DG_SqoopActionExtension.twiki
index 457e899..1e7f2a8 100644
--- a/docs/src/site/twiki/DG_SqoopActionExtension.twiki
+++ b/docs/src/site/twiki/DG_SqoopActionExtension.twiki
@@ -36,7 +36,7 @@ must not be present in the inline configuration.
As with Hadoop =map-reduce= jobs, it is possible to add files and
archives in order to make them available to the Sqoop job. Refer to the
-[WorkflowFunctionalSpec#FilesAchives][Adding Files and Archives for the Job]
+[WorkflowFunctionalSpec#FilesArchives][Adding Files and Archives for the Job]
section for more information about this feature.
*Syntax:*
http://git-wip-us.apache.org/repos/asf/oozie/blob/04bd4969/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
index 2ce56bf..f9ddbb5 100644
--- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
+++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
@@ -587,7 +587,7 @@ Streaming and inline property values can be parameterized (templatized) using EL
The Hadoop =mapred.job.tracker= and =fs.default.name= properties must not be present in the job-xml and inline
configuration.
-#FilesAchives
+#FilesArchives
---+++++ 3.2.2.1 Adding Files and Archives for the Job
The =file=, =archive= elements make available, to map-reduce jobs, files and archives. If the specified path is
@@ -948,7 +948,7 @@ The Hadoop =mapred.job.tracker= and =fs.default.name= properties must not be pre
configuration.
As with Hadoop map-reduce jobs, it is possible to add files and archives to be available to the Pig job, refer to
-section [#FilesAchives][Adding Files and Archives for the Job].
+section [#FilesArchives][Adding Files and Archives for the Job].
*Syntax for Pig actions in Oozie schema 0.2:*
<verbatim>
@@ -1468,7 +1468,7 @@ The Hadoop =mapred.job.tracker= (=job-tracker=) and =fs.default.name= (=name-nod
the =job-xml= and in the inline configuration.
As with =map-reduce= and =pig= actions, it is possible to add files and archives to be available to the Java
-application. Refer to section [#FilesAchives][Adding Files and Archives for the Job].
+application. Refer to section [#FilesArchives][Adding Files and Archives for the Job].
The =capture-output= element can be used to propagate values back into Oozie context, which can then be accessed via
EL-functions. This needs to be written out as a java properties format file. The filename is obtained via a System
http://git-wip-us.apache.org/repos/asf/oozie/blob/04bd4969/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d1467d2..6a045b0 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2401 Typo in twiki docs with FilesAchives instead of FilesArchives (anthony@cloudera.com via rkanter)
OOZIE-2168 Oozie flow and action names have 50 char limit (akshayrai09, me.venkatr via rkanter)
OOZIE-2346 Add sub-workflow information like the super-parent id and workflow depth into the 'oozie.job.info' property (akshayrai09 via puru)
OOZIE-2303 Typo in documentation (lars_francke via rohini)