You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2014/06/16 12:58:33 UTC
[2/2] git commit: FALCON-280 Validate the ACL in Feed entity with the user submitting the entity. Contributed by Jean-Baptiste Onofré
FALCON-280 Validate the ACL in Feed entity with the user submitting the entity. Contributed by Jean-Baptiste Onofré
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/b68946a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/b68946a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/b68946a0
Branch: refs/heads/master
Commit: b68946a02d6eff9314e242f73d7e97855c01957d
Parents: b501baa
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Mon Jun 16 16:28:17 2014 +0530
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Mon Jun 16 16:28:17 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../apache/falcon/entity/CatalogStorage.java | 6 +++
.../apache/falcon/entity/FileSystemStorage.java | 47 ++++++++++++++++++++
.../java/org/apache/falcon/entity/Storage.java | 10 +++++
.../falcon/entity/parser/FeedEntityParser.java | 17 +++++++
.../falcon/entity/FileSystemStorageTest.java | 24 ++++++++++
.../resources/config/cluster/cluster-0.1.xml | 2 +-
.../workflow/engine/OozieWorkflowEngine.java | 4 +-
8 files changed, 110 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e6ca795..0b007e2 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,9 @@ Trunk (Unreleased)
NEW FEATURES
IMPROVEMENTS
+ FALCON-280 Validate the ACL in Feed entity with the user submitting the entity
+ (Jean-Baptiste Onofré via Venkatesh Seetharam)
+
FALCON-445 Propagate hive credentials defined in cluster entity to
hive-site.xml (Sowmya Ramesh via Venkatesh Seetharam)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
index 37a05cb..f7b592d 100644
--- a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
@@ -338,6 +338,12 @@ public class CatalogStorage implements Storage {
}
@Override
+ public void validateACL(String owner, String group, String permissions) throws FalconException {
+ // This is not supported in Hive today as authorization is not enforced on table and
+ // partition listing
+ }
+
+ @Override
public String toString() {
return "CatalogStorage{"
+ "catalogUrl='" + catalogUrl + '\''
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index c2b86b6..9b76d3b 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -23,8 +23,14 @@ import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.entity.v0.feed.LocationType;
import org.apache.falcon.entity.v0.feed.Locations;
+import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.security.CurrentUser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.net.URI;
import java.net.URISyntaxException;
@@ -36,6 +42,8 @@ import java.util.List;
*/
public class FileSystemStorage implements Storage {
+ private static final Logger LOG = LoggerFactory.getLogger(FileSystemStorage.class);
+
public static final String FEED_PATH_SEP = "#";
public static final String LOCATION_TYPE_SEP = "=";
@@ -227,6 +235,45 @@ public class FileSystemStorage implements Storage {
}
@Override
+ public void validateACL(String owner, String group, String permissions) throws FalconException {
+ try {
+ FileSystem fileSystem = HadoopClientFactory.get().createProxiedFileSystem(getConf());
+ for (Location location : getLocations()) {
+ String pathString = getRelativePath(location);
+ Path path = new Path(pathString);
+ if (fileSystem.exists(path)) {
+ FileStatus fileStatus = fileSystem.getFileStatus(path);
+ if (!fileStatus.getOwner().equals(owner)) {
+ LOG.error("Feed ACL owner {} doesn't match the actual file owner {}",
+ owner, fileStatus.getOwner());
+ throw new FalconException("Feed ACL owner " + owner + " doesn't match the actual file owner "
+ + fileStatus.getOwner());
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Can't validate ACL on storage {}", getStorageUrl(), e);
+ throw new FalconException("Can't validate storage ACL (URI " + getStorageUrl() + ")", e);
+ }
+ }
+
+ private Configuration getConf() {
+ Configuration conf = new Configuration();
+ conf.set(HadoopClientFactory.FS_DEFAULT_NAME_KEY, storageUrl);
+ return conf;
+ }
+
+ private String getRelativePath(Location location) {
+ // if the path contains variables, locate on the "parent" path (just before first variable usage)
+ int index = location.getPath().indexOf(DOLLAR_EXPR_START_REGEX);
+ String pathString = location.getPath();
+ if (index != -1) {
+ pathString = pathString.substring(index);
+ }
+ return pathString;
+ }
+
+ @Override
public String toString() {
return "FileSystemStorage{"
+ "storageUrl='" + storageUrl + '\''
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/Storage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/Storage.java b/common/src/main/java/org/apache/falcon/entity/Storage.java
index 60c87c5..b49410e 100644
--- a/common/src/main/java/org/apache/falcon/entity/Storage.java
+++ b/common/src/main/java/org/apache/falcon/entity/Storage.java
@@ -72,4 +72,14 @@ public interface Storage {
* @throws FalconException an exception
*/
boolean isIdentical(Storage toCompareAgainst) throws FalconException;
+
+ /**
+ * Check the permission on the storage, regarding owner/group/permission coming from ACL.
+ *
+ * @param owner the owner defined in the ACL.
+ * @param group the group defined in the ACL.
+ * @param permissions the permissions defined in the ACL.
+ * @throws FalconException if the permissions are not valid.
+ */
+ void validateACL(String owner, String group, String permissions) throws FalconException;
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
index 7f87bdc..d5a8a76 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java
@@ -314,6 +314,9 @@ public class FeedEntityParser extends EntityParser<Feed> {
* Does not matter for FileSystem storage.
*/
private void validateFeedStorage(Feed feed) throws FalconException {
+ validateUser(feed);
+ validateACL(feed);
+
final Storage.TYPE baseFeedStorageType = FeedHelper.getStorageType(feed);
validateMultipleSourcesExist(feed, baseFeedStorageType);
validateUniformStorageType(feed, baseFeedStorageType);
@@ -405,4 +408,18 @@ public class FeedEntityParser extends EntityParser<Feed> {
throw new ValidationException(buffer.toString());
}
}
+
+ private void validateACL(Feed feed) throws FalconException {
+ for (Cluster cluster : feed.getClusters().getClusters()) {
+ org.apache.falcon.entity.v0.cluster.Cluster clusterEntity =
+ EntityUtil.getEntity(EntityType.CLUSTER, cluster.getName());
+ if (!EntityUtil.responsibleFor(clusterEntity.getColo())) {
+ continue;
+ }
+
+ final Storage storage = FeedHelper.createStorage(cluster, feed);
+ storage.validateACL(feed.getACL().getOwner(), feed.getACL().getGroup(),
+ feed.getACL().getPermission());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
index e095f8e..33e74fa 100644
--- a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
@@ -148,6 +148,30 @@ public class FileSystemStorageTest {
Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), absoluteUrl);
}
+ @Test
+ public void testValidateACL() throws Exception {
+ final Location location = new Location();
+ location.setPath("/foo/bar");
+ location.setType(LocationType.DATA);
+ List<Location> locations = new ArrayList<Location>();
+ locations.add(location);
+
+ FileSystemStorage storage = new FileSystemStorage("jail://global:00", locations);
+ storage.validateACL(USER, USER, "rrr");
+ }
+
+ @Test
+ public void testValidateACLWithTimeVariables() throws Exception {
+ final Location location = new Location();
+ location.setPath("/foo/bar/${YEAR}/${MONTH}/${DAY}");
+ location.setType(LocationType.DATA);
+ List<Location> locations = new ArrayList<Location>();
+ locations.add(location);
+
+ FileSystemStorage storage = new FileSystemStorage("jail://global:00", locations);
+ storage.validateACL(USER, USER, "rrr");
+ }
+
@DataProvider(name = "locationTestWithRelativePathDataProvider")
private Object[][] createLocationTestDataWithRelativePath() {
return new Object[][] {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/common/src/test/resources/config/cluster/cluster-0.1.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/config/cluster/cluster-0.1.xml b/common/src/test/resources/config/cluster/cluster-0.1.xml
index 658711d..0f5fb47 100644
--- a/common/src/test/resources/config/cluster/cluster-0.1.xml
+++ b/common/src/test/resources/config/cluster/cluster-0.1.xml
@@ -22,7 +22,7 @@
<interfaces>
<interface type="readonly" endpoint="hftp://localhost:50010"
version="0.20.2"/>
- <interface type="write" endpoint="hdfs://localhost:8020"
+ <interface type="write" endpoint="jail://testCluster:00"
version="0.20.2"/>
<interface type="execute" endpoint="localhost:8021" version="0.20.2"/>
<interface type="workflow" endpoint="http://localhost:11000/oozie/"
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/b68946a0/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
index 096be40..7166b13 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
@@ -1123,8 +1123,8 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
updateCoords(cluster, oldBundle, EntityUtil.getParallel(oldEntity), effectiveTime);
}
- if (oldBundleStatus != Job.Status.SUSPENDED &&
- oldBundleStatus != Job.Status.PREPSUSPENDED) {
+ if (oldBundleStatus != Job.Status.SUSPENDED
+ && oldBundleStatus != Job.Status.PREPSUSPENDED) {
//resume coords
resumeCoords(cluster, oldBundle);
}