You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2013/04/30 22:55:12 UTC
svn commit: r1477823 - in /oozie/trunk: ./ client/src/main/resources/
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/main/java/org/apache/oozie/service/
core/src/test/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie...
Author: virag
Date: Tue Apr 30 20:55:06 2013
New Revision: 1477823
URL: http://svn.apache.org/r1477823
Log:
OOZIE-1297 Add chgrp in FS action (ryota via virag)
Added:
oozie/trunk/client/src/main/resources/oozie-workflow-0.4.5.xsd
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
oozie/trunk/release-log.txt
Added: oozie/trunk/client/src/main/resources/oozie-workflow-0.4.5.xsd
URL: http://svn.apache.org/viewvc/oozie/trunk/client/src/main/resources/oozie-workflow-0.4.5.xsd?rev=1477823&view=auto
==============================================================================
--- oozie/trunk/client/src/main/resources/oozie-workflow-0.4.5.xsd (added)
+++ oozie/trunk/client/src/main/resources/oozie-workflow-0.4.5.xsd Tue Apr 30 20:55:06 2013
@@ -0,0 +1,339 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:workflow="uri:oozie:workflow:0.4.5"
+ elementFormDefault="qualified" targetNamespace="uri:oozie:workflow:0.4.5">
+
+ <xs:element name="workflow-app" type="workflow:WORKFLOW-APP"/>
+
+ <xs:simpleType name="IDENTIFIER">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="([a-zA-Z_]([\-_a-zA-Z0-9])*){1,39}"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:complexType name="WORKFLOW-APP">
+ <xs:sequence>
+ <xs:element name="parameters" type="workflow:PARAMETERS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="global" type="workflow:GLOBAL" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="credentials" type="workflow:CREDENTIALS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="start" type="workflow:START" minOccurs="1" maxOccurs="1"/>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="decision" type="workflow:DECISION" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="fork" type="workflow:FORK" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="join" type="workflow:JOIN" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="kill" type="workflow:KILL" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="action" type="workflow:ACTION" minOccurs="1" maxOccurs="1"/>
+ </xs:choice>
+ <xs:element name="end" type="workflow:END" minOccurs="1" maxOccurs="1"/>
+ <xs:any namespace="uri:oozie:sla:0.1" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="PARAMETERS">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="GLOBAL">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="workflow:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="START">
+ <xs:attribute name="to" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="END">
+ <xs:attribute name="name" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="DECISION">
+ <xs:sequence>
+ <xs:element name="switch" type="workflow:SWITCH" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:element name="switch" type="workflow:SWITCH"/>
+
+ <xs:complexType name="SWITCH">
+ <xs:sequence>
+ <xs:sequence>
+ <xs:element name="case" type="workflow:CASE" minOccurs="1" maxOccurs="unbounded"/>
+ <xs:element name="default" type="workflow:DEFAULT" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="CASE">
+ <xs:simpleContent>
+ <xs:extension base="xs:string">
+ <xs:attribute name="to" type="workflow:IDENTIFIER" use="required"/>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+
+ <xs:complexType name="DEFAULT">
+ <xs:attribute name="to" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="FORK_TRANSITION">
+ <xs:attribute name="start" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="FORK">
+ <xs:sequence>
+ <xs:element name="path" type="workflow:FORK_TRANSITION" minOccurs="2" maxOccurs="unbounded"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="JOIN">
+ <xs:attribute name="name" type="workflow:IDENTIFIER" use="required"/>
+ <xs:attribute name="to" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:element name="kill" type="workflow:KILL"/>
+
+ <xs:complexType name="KILL">
+ <xs:sequence>
+ <xs:element name="message" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="ACTION_TRANSITION">
+ <xs:attribute name="to" type="workflow:IDENTIFIER" use="required"/>
+ </xs:complexType>
+
+ <xs:element name="map-reduce" type="workflow:MAP-REDUCE"/>
+ <xs:element name="pig" type="workflow:PIG"/>
+ <xs:element name="sub-workflow" type="workflow:SUB-WORKFLOW"/>
+ <xs:element name="fs" type="workflow:FS"/>
+ <xs:element name="java" type="workflow:JAVA"/>
+
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:choice minOccurs="1" maxOccurs="1">
+ <xs:element name="map-reduce" type="workflow:MAP-REDUCE" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="pig" type="workflow:PIG" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="sub-workflow" type="workflow:SUB-WORKFLOW" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="fs" type="workflow:FS" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="java" type="workflow:JAVA" minOccurs="1" maxOccurs="1"/>
+ <xs:any namespace="##other" minOccurs="1" maxOccurs="1"/>
+ </xs:choice>
+ <xs:element name="ok" type="workflow:ACTION_TRANSITION" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="error" type="workflow:ACTION_TRANSITION" minOccurs="1" maxOccurs="1"/>
+ <xs:any namespace="uri:oozie:sla:0.1" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="workflow:IDENTIFIER" use="required"/>
+ <xs:attribute name="cred" type="xs:string"/>
+ <xs:attribute name="retry-max" type="xs:string"/>
+ <xs:attribute name="retry-interval" type="xs:string"/>
+ </xs:complexType>
+
+ <xs:complexType name="MAP-REDUCE">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="prepare" type="workflow:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:choice minOccurs="0" maxOccurs="1">
+ <xs:element name="streaming" type="workflow:STREAMING" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="pipes" type="workflow:PIPES" minOccurs="0" maxOccurs="1"/>
+ </xs:choice>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="workflow:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="PIG">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="prepare" type="workflow:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="workflow:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="script" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="param" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="argument" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="SUB-WORKFLOW">
+ <xs:sequence>
+ <xs:element name="app-path" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="propagate-configuration" type="workflow:FLAG" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="configuration" type="workflow:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="FS">
+ <xs:sequence>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="workflow:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="delete" type="workflow:DELETE"/>
+ <xs:element name="mkdir" type="workflow:MKDIR"/>
+ <xs:element name="move" type="workflow:MOVE"/>
+ <xs:element name="chmod" type="workflow:CHMOD"/>
+ <xs:element name="touchz" type="workflow:TOUCHZ"/>
+ <xs:element name="chgrp" type="workflow:CHGRP"/>
+ </xs:choice>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="JAVA">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="prepare" type="workflow:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="configuration" type="workflow:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="main-class" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:choice minOccurs="0" maxOccurs="1">
+ <xs:element name="java-opts" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="java-opt" type="xs:string" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:choice>
+ <xs:element name="arg" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="capture-output" type="workflow:FLAG" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="FLAG"/>
+
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="STREAMING">
+ <xs:sequence>
+ <xs:element name="mapper" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="reducer" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="record-reader" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="record-reader-mapping" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="env" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="PIPES">
+ <xs:sequence>
+ <xs:element name="map" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="reduce" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="inputformat" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="partitioner" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="writer" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="program" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="PREPARE">
+ <xs:sequence>
+ <xs:element name="delete" type="workflow:DELETE" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="mkdir" type="workflow:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="DELETE">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="MKDIR">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="MOVE">
+ <xs:attribute name="source" type="xs:string" use="required"/>
+ <xs:attribute name="target" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="CHMOD">
+ <xs:sequence>
+ <xs:element name="recursive" type="workflow:FLAG" minOccurs="0" maxOccurs="1"></xs:element>
+ </xs:sequence>
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ <xs:attribute name="permissions" type="xs:string" use="required"/>
+ <xs:attribute name="dir-files" type="xs:string"/>
+ </xs:complexType>
+
+ <xs:complexType name="TOUCHZ">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="CHGRP">
+ <xs:sequence>
+ <xs:element name="recursive" type="workflow:FLAG" minOccurs="0" maxOccurs="1"></xs:element>
+ </xs:sequence>
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ <xs:attribute name="group" type="xs:string" use="required"/>
+ <xs:attribute name="dir-files" type="xs:string"/>
+ </xs:complexType>
+
+ <xs:complexType name="CREDENTIALS">
+ <xs:sequence minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="credential" type="workflow:CREDENTIAL"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="CREDENTIAL">
+ <xs:sequence minOccurs="0" maxOccurs="unbounded" >
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ <xs:attribute name="type" type="xs:string" use="required"/>
+ </xs:complexType>
+</xs:schema>
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java?rev=1477823&r1=1477822&r2=1477823&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java Tue Apr 30 20:55:06 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,8 +20,11 @@ package org.apache.oozie.action.hadoop;
import java.io.IOException;
import java.io.StringReader;
import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -93,7 +96,7 @@ public class FsActionExecutor extends Ac
String nameNodeSchemeAuthority = nameNode.toUri().getScheme() + "://" + nameNode.toUri().getAuthority();
fullPath = new Path(nameNodeSchemeAuthority + path.toString());
} else {
- throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS011",
+ throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS011",
"Path [{0}] cannot be relative", path);
}
} else {
@@ -128,7 +131,7 @@ public class FsActionExecutor extends Ac
if (!recovery) {
fs.mkdirs(getRecoveryPath(context));
}
-
+
Path nameNodePath = null;
Element nameNodeElement = element.getChild("name-node", element.getNamespace());
if (nameNodeElement != null) {
@@ -139,7 +142,7 @@ public class FsActionExecutor extends Ac
validatePath(nameNodePath, true);
}
}
-
+
XConfiguration fsConf = new XConfiguration();
Path appPath = new Path(context.getWorkflow().getAppPath());
// app path could be a file
@@ -147,7 +150,7 @@ public class FsActionExecutor extends Ac
appPath = appPath.getParent();
}
JavaActionExecutor.parseJobXmlAndConfiguration(context, element, appPath, fsConf);
-
+
for (Element commandElement : (List<Element>) element.getChildren()) {
String command = commandElement.getName();
if (command.equals("mkdir")) {
@@ -157,13 +160,13 @@ public class FsActionExecutor extends Ac
else {
if (command.equals("delete")) {
Path path = getPath(commandElement, "path");
- delete(context, fsConf,nameNodePath, path);
+ delete(context, fsConf, nameNodePath, path);
}
else {
if (command.equals("move")) {
Path source = getPath(commandElement, "source");
Path target = getPath(commandElement, "target");
- move(context, fsConf,nameNodePath, source, target, recovery);
+ move(context, fsConf, nameNodePath, source, target, recovery);
}
else {
if (command.equals("chmod")) {
@@ -172,12 +175,24 @@ public class FsActionExecutor extends Ac
String str = commandElement.getAttributeValue("dir-files");
boolean dirFiles = (str == null) || Boolean.parseBoolean(str);
String permissionsMask = commandElement.getAttributeValue("permissions").trim();
- chmod(context, fsConf,nameNodePath, path, permissionsMask, dirFiles, recursive);
+ chmod(context, fsConf, nameNodePath, path, permissionsMask, dirFiles, recursive);
}
else {
if (command.equals("touchz")) {
Path path = getPath(commandElement, "path");
- touchz(context, fsConf,nameNodePath, path);
+ touchz(context, fsConf, nameNodePath, path);
+ }
+ else {
+ if (command.equals("chgrp")) {
+ Path path = getPath(commandElement, "path");
+ boolean recursive = commandElement.getChild("recursive",
+ commandElement.getNamespace()) != null;
+ String group = commandElement.getAttributeValue("group");
+ String str = commandElement.getAttributeValue("dir-files");
+ boolean dirFiles = (str == null) || Boolean.parseBoolean(str);
+ chgrp(context, fsConf, nameNodePath, path, context.getWorkflow().getUser(),
+ group, dirFiles, recursive);
+ }
}
}
}
@@ -190,6 +205,73 @@ public class FsActionExecutor extends Ac
}
}
+ void chgrp(Context context, XConfiguration fsConf, Path nameNodePath, Path path, String user, String group,
+ boolean dirFiles, boolean recursive) throws ActionExecutorException {
+
+ HashMap<String, String> argsMap = new HashMap<String, String>();
+ argsMap.put("user", user);
+ argsMap.put("group", group);
+ try {
+ FileSystem fs = getFileSystemFor(path, context, fsConf);
+ recursiveFsOperation("chgrp", fs, nameNodePath, path, argsMap, dirFiles, recursive, true);
+ }
+ catch (Exception ex) {
+ throw convertException(ex);
+ }
+ }
+
+ private void recursiveFsOperation(String op, FileSystem fs, Path nameNodePath, Path path,
+ Map<String, String> argsMap, boolean dirFiles, boolean recursive, boolean isRoot)
+ throws ActionExecutorException {
+
+ try {
+ path = resolveToFullPath(nameNodePath, path, true);
+ if (!fs.exists(path)) {
+ throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS009", op
+ + ", path [{0}] does not exist", path);
+ }
+ FileStatus pathStatus = fs.getFileStatus(path);
+ List<Path> paths = new ArrayList<Path>();
+
+ if (dirFiles && pathStatus.isDir()) {
+ if (isRoot) {
+ paths.add(path);
+ }
+ FileStatus[] filesStatus = fs.listStatus(path);
+ for (int i = 0; i < filesStatus.length; i++) {
+ Path p = filesStatus[i].getPath();
+ paths.add(p);
+ if (recursive && filesStatus[i].isDir()) {
+ recursiveFsOperation(op, fs, null, p, argsMap, dirFiles, recursive, false);
+ }
+ }
+ }
+ else {
+ paths.add(path);
+ }
+ for (Path p : paths) {
+ doFsOperation(op, fs, p, argsMap);
+ }
+ }
+ catch (Exception ex) {
+ throw convertException(ex);
+ }
+ }
+
+ private void doFsOperation(String op, FileSystem fs, Path p, Map<String, String> argsMap)
+ throws ActionExecutorException, IOException {
+ if (op.equals("chmod")) {
+ String permissions = argsMap.get("permissions");
+ FsPermission newFsPermission = createShortPermission(permissions, p);
+ fs.setPermission(p, newFsPermission);
+ }
+ else if (op.equals("chgrp")) {
+ String user = argsMap.get("user");
+ String group = argsMap.get("group");
+ fs.setOwner(p, user, group);
+ }
+ }
+
/**
* @param path
* @param context
@@ -258,7 +340,7 @@ public class FsActionExecutor extends Ac
*
* @param context
* @param fsConf
- * @param nameNodePath
+ * @param nameNodePath
* @param path
* @throws ActionExecutorException
*/
@@ -321,14 +403,14 @@ public class FsActionExecutor extends Ac
* Move source to target
*
* @param context
- * @param fsConf
+ * @param fsConf
* @param nameNodePath
* @param source
* @param target
* @param recovery
* @throws ActionExecutorException
*/
- public void move(Context context, XConfiguration fsConf, Path nameNodePath, Path source, Path target, boolean recovery)
+ public void move(Context context, XConfiguration fsConf, Path nameNodePath, Path source, Path target, boolean recovery)
throws ActionExecutorException {
try {
source = resolveToFullPath(nameNodePath, source, true);
@@ -354,39 +436,14 @@ public class FsActionExecutor extends Ac
chmod(context, null, null, path, permissions, dirFiles, recursive);
}
- void chmod(Context context, XConfiguration fsConf, Path nameNodePath, Path path, String permissions, boolean dirFiles,
- boolean recursive) throws ActionExecutorException {
+ void chmod(Context context, XConfiguration fsConf, Path nameNodePath, Path path, String permissions,
+ boolean dirFiles, boolean recursive) throws ActionExecutorException {
+
+ HashMap<String, String> argsMap = new HashMap<String, String>();
+ argsMap.put("permissions", permissions);
try {
- path = resolveToFullPath(nameNodePath, path, true);
FileSystem fs = getFileSystemFor(path, context, fsConf);
-
- if (!fs.exists(path)) {
- throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS009",
- "chmod, path [{0}] does not exist", path);
- }
-
- FileStatus pathStatus = fs.getFileStatus(path);
-
- Path[] paths;
- if (dirFiles && pathStatus.isDir()) {
- FileStatus[] filesStatus = fs.listStatus(path);
- paths = new Path[filesStatus.length];
- for (int i = 0; i < filesStatus.length; i++) {
- paths[i] = filesStatus[i].getPath();
- if (recursive && filesStatus[i].isDir()){
- chmod(context, fsConf, nameNodePath, paths[i], permissions, dirFiles, recursive);
- }
- }
- }
- else {
- paths = new Path[]{path};
- }
-
- FsPermission newFsPermission = createShortPermission(permissions, path);
- fs.setPermission(path, newFsPermission);
- for (Path p : paths) {
- fs.setPermission(p, newFsPermission);
- }
+ recursiveFsOperation("chmod", fs, nameNodePath, path, argsMap, dirFiles, recursive, true);
}
catch (Exception ex) {
throw convertException(ex);
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java?rev=1477823&r1=1477822&r2=1477823&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java Tue Apr 30 20:55:06 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -59,13 +59,14 @@ public class SchemaService implements Se
private Schema slaSchema;
- private static final String OOZIE_WORKFLOW_XSD[] = {
- "oozie-workflow-0.1.xsd",
- "oozie-workflow-0.2.xsd",
- "oozie-workflow-0.2.5.xsd",
+ private static final String OOZIE_WORKFLOW_XSD[] = {
+ "oozie-workflow-0.1.xsd",
+ "oozie-workflow-0.2.xsd",
+ "oozie-workflow-0.2.5.xsd",
"oozie-workflow-0.3.xsd",
- "oozie-workflow-0.4.xsd"};
- private static final String OOZIE_COORDINATOR_XSD[] = { "oozie-coordinator-0.1.xsd", "oozie-coordinator-0.2.xsd",
+ "oozie-workflow-0.4.xsd",
+ "oozie-workflow-0.4.5.xsd"};
+ private static final String OOZIE_COORDINATOR_XSD[] = { "oozie-coordinator-0.1.xsd", "oozie-coordinator-0.2.xsd",
"oozie-coordinator-0.3.xsd", "oozie-coordinator-0.4.xsd"};
private static final String OOZIE_BUNDLE_XSD[] = { "oozie-bundle-0.1.xsd", "oozie-bundle-0.2.xsd" };
private static final String OOZIE_SLA_SEMANTIC_XSD[] = { "gms-oozie-sla-0.1.xsd" };
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java?rev=1477823&r1=1477822&r2=1477823&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java Tue Apr 30 20:55:06 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -62,7 +62,7 @@ public class TestFsActionExecutor extend
return new Context(wf, action);
}
-
+
public void testValidatePath() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
ae.validatePath(new Path("hdfs://x/bla"), true);
@@ -147,7 +147,7 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS001", ex.getErrorCode());
+ assertEquals("FS001", ex.getErrorCode());
}
Services.get().destroy();
@@ -174,7 +174,7 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS011", ex.getErrorCode());
+ assertEquals("FS011", ex.getErrorCode());
}
try {
@@ -195,7 +195,7 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS007", ex.getErrorCode());
+ assertEquals("FS007", ex.getErrorCode());
}
try {
@@ -203,7 +203,7 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS007", ex.getErrorCode());
+ assertEquals("FS007", ex.getErrorCode());
}
}
@@ -244,7 +244,7 @@ public class TestFsActionExecutor extend
FsActionExecutor ae = new FsActionExecutor();
FileSystem fs = getFileSystem();
- Path source = new Path(getFsTestCaseDir(), "source");
+ Path source = new Path(getFsTestCaseDir(), "source");
Path target = new Path(getFsTestCaseDir(), "target");
Context context = createContext("<fs/>");
@@ -261,13 +261,13 @@ public class TestFsActionExecutor extend
try {
ae.move(context, new Path(source+"/newfile1"), destPath, false);
- fail();
+ fail();
}
catch (ActionExecutorException ex) {
assertEquals("FS006", ex.getErrorCode());
}
- fs.mkdirs(source);
+ fs.mkdirs(source);
fs.createNewFile(new Path(source+"/newfile"));
Path complexTarget = new Path(target+"/a/b");
fs.mkdirs(complexTarget);
@@ -466,7 +466,7 @@ public void testChmodRecursive() throws
assertEquals("rwxrwx---", fs.getFileStatus(child3).getPermission().toString());
assertEquals("rwxrwx---", fs.getFileStatus(grandchild3).getPermission().toString());
}
-
+
public void testDoOperationsWithNameNodeElement() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
FileSystem fs = getFileSystem();
@@ -598,7 +598,7 @@ public void testChmodRecursive() throws
assertNotSame("rwxrwx---", fs.getFileStatus(child2).getPermission().toString());
}
-
+
public void testSubmitWithNameNode() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
FileSystem fs = getFileSystem();
@@ -628,7 +628,7 @@ public void testChmodRecursive() throws
Path newFile2X = new Path(newFile2.toUri().getPath());
fs.createNewFile(newFile1);
- String actionXml = MessageFormat.format("<fs><name-node>{0}</name-node>" +
+ String actionXml = MessageFormat.format("<fs><name-node>{0}</name-node>" +
"<mkdir path=''{1}''/>" +
"<delete path=''{2}''/>" +
"<move source=''{3}'' target=''{4}''/>" +
@@ -765,4 +765,54 @@ public void testChmodRecursive() throws
assertEquals("r--------", ae.createShortPermission("-r--------", null).toString());
}
+
+ public void testChgrp() throws Exception {
+ FsActionExecutor ae = new FsActionExecutor();
+ FileSystem fs = getFileSystem();
+
+ Path path = new Path(getFsTestCaseDir(), "dir");
+ Path child = new Path(path, "child");
+ Path grandchild = new Path(child, "grandchild");
+ fs.mkdirs(grandchild);
+ String testUser = getTestUser();
+ String testGroup = getTestGroup();
+ String testGroup2 = getTestGroup2();
+
+ fs.setOwner(path, testUser, testGroup);
+ fs.setOwner(child, testUser, testGroup);
+ fs.setOwner(grandchild, testUser, testGroup);
+ assertEquals(testGroup, fs.getFileStatus(path).getGroup().toString());
+ assertEquals(testGroup, fs.getFileStatus(child).getGroup().toString());
+ assertEquals(testGroup, fs.getFileStatus(grandchild).getGroup().toString());
+
+ Context context = createContext("<fs/>");
+
+ // Test case where dir-files=false, recursive=false
+ ae.chgrp(context, null, null, path, testUser, testGroup2, false, false);
+ assertEquals(testGroup2, fs.getFileStatus(path).getGroup().toString());
+ assertEquals(testGroup, fs.getFileStatus(child).getGroup().toString());
+ assertEquals(testGroup, fs.getFileStatus(grandchild).getGroup().toString());
+
+ // Test case where dir-files=true, recursive=false
+
+ fs.setOwner(path, testUser, testGroup);// revert to testgroup
+ fs.setOwner(child, testUser, testGroup);// revert to testgroup
+ fs.setOwner(grandchild, testUser, testGroup);// revert to testgroup
+
+ ae.chgrp(context, null, null, path, testUser, testGroup2, true, false);
+ assertEquals(testGroup2, fs.getFileStatus(path).getGroup().toString());
+ assertEquals(testGroup2, fs.getFileStatus(child).getGroup().toString());
+ assertEquals(testGroup, fs.getFileStatus(grandchild).getGroup().toString());
+
+ // Test case where dir-files=true, recursive=true
+
+ fs.setOwner(path, testUser, testGroup);// revert to testgroup
+ fs.setOwner(child, testUser, testGroup);// revert to testgroup
+ fs.setOwner(grandchild, testUser, testGroup);// revert to testgroup
+
+ ae.chgrp(context, null, null, path, testUser, testGroup2, true, true);
+ assertEquals(testGroup2, fs.getFileStatus(path).getGroup().toString());
+ assertEquals(testGroup2, fs.getFileStatus(child).getGroup().toString());
+ assertEquals(testGroup2, fs.getFileStatus(grandchild).getGroup().toString());
+ }
}
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java?rev=1477823&r1=1477822&r2=1477823&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java Tue Apr 30 20:55:06 2013
@@ -215,6 +215,12 @@ public abstract class XTestCase extends
public static final String TEST_GROUP_PROP = "oozie.test.group";
/**
+ * System property that specifies the test groiup used by the tests.
+ * The default value of this property is <tt>testg</tt>.
+ */
+ public static final String TEST_GROUP_PROP2 = "oozie.test.group2";
+
+ /**
* System property that specifies the wait time, in seconds, between testcases before
* triggering a shutdown. The default value is 10 sec.
*/
@@ -412,6 +418,15 @@ public abstract class XTestCase extends
}
/**
+ * Return the alternate test group.
+ *
+ * @return the test group.
+ */
+ protected static String getTestGroup2() {
+ return System.getProperty(TEST_GROUP_PROP, "testg2");
+ }
+
+ /**
* Return the test working directory.
* <p/>
* It returns <code>${oozie.test.dir}/oozietests/TESTCLASSNAME/TESTMETHODNAME</code>.
@@ -739,7 +754,7 @@ public abstract class XTestCase extends
conf.set("mapred.tasktracker.map.tasks.maximum", "4");
conf.set("mapred.tasktracker.reduce.tasks.maximum", "4");
- String [] userGroups = new String[] { getTestGroup() };
+ String[] userGroups = new String[] { getTestGroup(), getTestGroup2() };
UserGroupInformation.createUserForTesting(oozieUser, userGroups);
UserGroupInformation.createUserForTesting(getTestUser(), userGroups);
UserGroupInformation.createUserForTesting(getTestUser2(), userGroups);
Modified: oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki?rev=1477823&r1=1477822&r2=1477823&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki Tue Apr 30 20:55:06 2013
@@ -1024,7 +1024,7 @@ All the above elements can be parameteri
---++++ 3.2.4 Fs (HDFS) action
The =fs= action allows to manipulate files and directories in HDFS from a workflow application. The supported commands
-are =move=, =delete= and =mkdir=.
+are =move=, =delete=, =mkdir=, =chmod=, =touchz= and =chgrp=.
The FS commands are executed synchronously from within the FS action, the workflow job will wait until the specified
file commands are completed before continuing to the next action.
@@ -1055,6 +1055,8 @@ executed. Thus there is less chance of a
<chmod path='[PATH]' permissions='[PERMISSIONS]' dir-files='false' />
...
<touchz path='[PATH]' />
+ ...
+ <chmod path='[PATH]' group='[GROUP]' dir-files='false' />
</fs>
<ok to="[NODE-NAME]"/>
<error to="[NODE-NAME]"/>
@@ -1086,6 +1088,12 @@ recursively to all levels within a direc
The =touchz= command creates a zero length file in the specified path if none exists. If one already exists, then touchz will perform a touch operation.
Touchz works only for absolute paths.
+The =chgrp= command changes the group for the specified path.
+When doing a =chgrp= command on a directory, by default the command is applied to the directory and the files one level
+within the directory. To apply the =chgrp= command to the directory, without affecting the files within it,
+the =dir-files= attribute must be set to =false=.
+To apply the =chgrp= command recursively to all levels within a directory, put a =recursive= element inside the <chgrp> element.
+
*Example:*
<verbatim>
@@ -1097,6 +1105,7 @@ Touchz works only for absolute paths.
<mkdir path='archives/${wf:id()}'/>
<move source='${jobInput}' target='archives/${wf:id()}/processed-input'/>
<chmod path='${jobOutput}' permissions='-rwxrw-rw-' dir-files='true'><recursive/></chmod>
+ <chgrp path='${jobOutput}' group='testgroup' dir-files='true'><recursive/></chgrp>
</fs>
<ok to="myotherjob"/>
<error to="errorcleanup"/>
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1477823&r1=1477822&r2=1477823&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Tue Apr 30 20:55:06 2013
@@ -1,6 +1,7 @@
-- Oozie 4.1.0 release (trunk - unreleased)
-OOZIE-1329 fix coverage org.apache.oozie.tools (agorshkov)
+OOZIE-1297 Add chgrp in FS action (ryota via virag)
+OOZIE-1329 fix coverage org.apache.oozie.tools (agorshkov via virag)
OOZIE-1351 Oozie jobs with state PAUSEDWITHERROR should change to SUSPENDEDWITHERROR state when suspended (bowenzhangusa via virag)
OOZIE-1346 Modularize hbase credentials to separate the populating of jobconf and obtaining token for job (virag)
OOZIE-1341 Have Action Main classes in Oozie webapp (virag)