You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2017/06/20 07:25:08 UTC
oozie git commit: OOZIE-2769 Extend FS action to allow setrep on a
file (Artem Ervits via gezapeti)
Repository: oozie
Updated Branches:
refs/heads/master a5c839a3a -> 72bce837d
OOZIE-2769 Extend FS action to allow setrep on a file (Artem Ervits via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/72bce837
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/72bce837
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/72bce837
Branch: refs/heads/master
Commit: 72bce837d038e0c61435c77c56fadd81dcd05cbb
Parents: a5c839a
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Tue Jun 20 09:25:01 2017 +0200
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Tue Jun 20 09:25:01 2017 +0200
----------------------------------------------------------------------
.../src/main/resources/oozie-workflow-0.5.xsd | 6 +++++
.../oozie/action/hadoop/FsActionExecutor.java | 24 ++++++++++++++++++++
.../action/hadoop/TestFsActionExecutor.java | 13 +++++++++++
core/src/test/resources/wf-schema-valid.xml | 5 +++-
.../src/site/twiki/WorkflowFunctionalSpec.twiki | 8 ++++++-
release-log.txt | 1 +
6 files changed, 55 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/client/src/main/resources/oozie-workflow-0.5.xsd
----------------------------------------------------------------------
diff --git a/client/src/main/resources/oozie-workflow-0.5.xsd b/client/src/main/resources/oozie-workflow-0.5.xsd
index 8fe2b47..87f38a3 100644
--- a/client/src/main/resources/oozie-workflow-0.5.xsd
+++ b/client/src/main/resources/oozie-workflow-0.5.xsd
@@ -215,6 +215,7 @@
<xs:element name="chmod" type="workflow:CHMOD"/>
<xs:element name="touchz" type="workflow:TOUCHZ"/>
<xs:element name="chgrp" type="workflow:CHGRP"/>
+ <xs:element name="setrep" type="workflow:SETREP"/>
</xs:choice>
</xs:sequence>
</xs:complexType>
@@ -305,6 +306,11 @@
<xs:attribute name="dir-files" type="xs:string"/>
</xs:complexType>
+ <xs:complexType name="SETREP">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ <xs:attribute name="replication-factor" type="xs:short" use="required"/>
+ </xs:complexType>
+
<xs:complexType name="TOUCHZ">
<xs:attribute name="path" type="xs:string" use="required"/>
</xs:complexType>
http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
index e1150f7..85d3efa 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
@@ -223,6 +223,16 @@ public class FsActionExecutor extends ActionExecutor {
chgrp(context, fsConf, nameNodePath, path, context.getWorkflow().getUser(),
group, dirFiles, recursive);
}
+ else {
+ if (command.equals("setrep")) {
+ Path path = getPath(commandElement, "path");
+ String replicationFactor =
+ commandElement.getAttributeValue("replication-factor");
+ if (commandElement.getAttributeValue("replication-factor") != null) {
+ setrep(context, path, Short.parseShort(replicationFactor));
+ }
+ }
+ }
}
}
}
@@ -654,6 +664,20 @@ public class FsActionExecutor extends ActionExecutor {
}
}
+ void setrep(Context context, Path path, short replicationFactor)
+ throws ActionExecutorException, HadoopAccessorException {
+ try {
+ path = resolveToFullPath(null, path, true);
+ FileSystem fs = getFileSystemFor(path, context, null);
+
+ if (fs.isFile(path)) {
+ fs.setReplication(path, replicationFactor);
+ }
+ } catch (IOException ex) {
+ convertException(ex);
+ }
+ }
+
public boolean supportsConfigurationJobXML() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
index 99e4d91..6939b57 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
@@ -1214,4 +1214,17 @@ public void testChmodRecursive() throws Exception {
assertEquals(ActionExecutorException.ErrorType.ERROR, ((ActionExecutorException) e).getErrorType());
}
}
+
+ public void testSetRep() throws Exception {
+ FsActionExecutor ae = new FsActionExecutor();
+ FileSystem fs = getFileSystem();
+ Path path = new Path(getFsTestCaseDir(), "dir1");
+ Context context = createContext("<fs/>");
+ fs.mkdirs(path);
+ Path file = new Path(path + "/file");
+ fs.createNewFile(file);
+ final short replicationFactor = (short) 2;
+ ae.setrep(context, path, replicationFactor);
+ assertEquals(replicationFactor, fs.getFileStatus(file).getReplication());
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/core/src/test/resources/wf-schema-valid.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/wf-schema-valid.xml b/core/src/test/resources/wf-schema-valid.xml
index 71bbcc8..085634f 100644
--- a/core/src/test/resources/wf-schema-valid.xml
+++ b/core/src/test/resources/wf-schema-valid.xml
@@ -15,7 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<workflow-app xmlns="uri:oozie:workflow:0.1" name="test-wf">
+<workflow-app xmlns="uri:oozie:workflow:0.5" name="test-wf">
<start to="a"/>
<decision name="a">
<switch>
@@ -100,6 +100,9 @@
<delete path='/tmp2'/>
<mkdir path='/tmp3'/>
<delete path='/tmp3'/>
+ <mkdir path='/tmp4'/>
+ <touchz path='/tmp4/file'/>
+ <setrep path='/tmp4/file' replication-factor='2'/>
</fs>
<ok to="z"/>
<error to="b"/>
http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
index 6655696..5a75b99 100644
--- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
+++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
@@ -1139,7 +1139,7 @@ All the above elements can be parameterized (templatized) using EL expressions.
---++++ 3.2.4 Fs (HDFS) action
The =fs= action allows to manipulate files and directories in HDFS from a workflow application. The supported commands
-are =move=, =delete=, =mkdir=, =chmod=, =touchz= and =chgrp=.
+are =move=, =delete=, =mkdir=, =chmod=, =touchz=, =setrep= and =chgrp=.
The FS commands are executed synchronously from within the FS action, the workflow job will wait until the specified
file commands are completed before continuing to the next action.
@@ -1174,6 +1174,8 @@ executed. Thus there is less chance of an error occurring while the =fs= action
<touchz path='[PATH]' />
...
<chgrp path='[PATH]' group='[GROUP]' dir-files='false' />
+ ...
+ <setrep path='[PATH]' replication-factor='2'/>
</fs>
<ok to="[NODE-NAME]"/>
<error to="[NODE-NAME]"/>
@@ -1218,6 +1220,9 @@ within the directory. To apply the =chgrp= command to the directory, without aff
the =dir-files= attribute must be set to =false=.
To apply the =chgrp= command recursively to all levels within a directory, put a =recursive= element inside the <chgrp> element.
+The =setrep= command changes replication factor of an hdfs file(s). Changing RF of directories or symlinks is not
+supported; this action requires an argument for RF.
+
*Example:*
<verbatim>
@@ -1230,6 +1235,7 @@ To apply the =chgrp= command recursively to all levels within a directory, put a
<move source='${jobInput}' target='archives/${wf:id()}/processed-input'/>
<chmod path='${jobOutput}' permissions='-rwxrw-rw-' dir-files='true'><recursive/></chmod>
<chgrp path='${jobOutput}' group='testgroup' dir-files='true'><recursive/></chgrp>
+ <setrep path='archives/${wf:id()/filename(s)}' replication-factor='2'/>
</fs>
<ok to="myotherjob"/>
<error to="errorcleanup"/>
http://git-wip-us.apache.org/repos/asf/oozie/blob/72bce837/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 10c3d56..05ed49c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-2769 Extend FS action to allow setrep on a file (Artem Ervits via gezapeti)
OOZIE-2815 amend - Oozie not always display job log (andras.piros via gezapeti)
OOZIE-2946 Include find-sec-bugs plugin (Jan Hentschel via rkanter)
OOZIE-2950 TestOozieCLI.testSlaEvents is not testing SLAServlet. (gezapeti)