You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2014/11/14 03:54:51 UTC
[6/8] incubator-falcon git commit: FALCON-889 Windows azure
replication fails with "wasb" as the scheme to an HDFS file system.
Contributed by Chris Nauroth
FALCON-889 Windows azure replication fails with "wasb" as the scheme to an HDFS file system. Contributed by Chris Nauroth
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/60d33b61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/60d33b61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/60d33b61
Branch: refs/heads/master
Commit: 60d33b615d90cdaea2a869c24b0157e9f5a798d4
Parents: fc3094a
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Thu Nov 13 18:19:11 2014 -0800
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Thu Nov 13 18:19:11 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../apache/falcon/entity/FileSystemStorage.java | 3 +-
.../entity/parser/FeedEntityParserTest.java | 50 +++++++++++++++++++-
3 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 443eb65..f20ef0d 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -144,6 +144,9 @@ Trunk (Unreleased)
OPTIMIZATIONS
BUG FIXES
+ FALCON-889 Windows azure replication fails with "wasb" as the scheme to an
+ HDFS file system (Chris Nauroth via Venkatesh Seetharam)
+
FALCON-885 RequestID is coming as null for all instance API calls
(Venkatesh Seetharam)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index f7bd321..bbe274b 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -273,10 +273,11 @@ public class FileSystemStorage extends Configured implements Storage {
@Override
public void validateACL(AccessControlList acl) throws FalconException {
try {
- FileSystem fileSystem = HadoopClientFactory.get().createProxiedFileSystem(getConf());
for (Location location : getLocations()) {
String pathString = getRelativePath(location);
Path path = new Path(pathString);
+ FileSystem fileSystem =
+ HadoopClientFactory.get().createProxiedFileSystem(path.toUri(), getConf());
if (fileSystem.exists(path)) {
FileStatus fileStatus = fileSystem.getFileStatus(path);
Set<String> groups = CurrentUser.getGroupNames();
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
index 8c67a7d..8e773cb 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java
@@ -30,7 +30,15 @@ import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.feed.*;
+import org.apache.falcon.entity.v0.feed.ActionType;
+import org.apache.falcon.entity.v0.feed.ClusterType;
+import org.apache.falcon.entity.v0.feed.Feed;
+import org.apache.falcon.entity.v0.feed.Location;
+import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.falcon.entity.v0.feed.Locations;
+import org.apache.falcon.entity.v0.feed.Partition;
+import org.apache.falcon.entity.v0.feed.Partitions;
+import org.apache.falcon.entity.v0.feed.Validity;
import org.apache.falcon.group.FeedGroupMapTest;
import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.util.StartupProperties;
@@ -874,4 +882,44 @@ public class FeedEntityParserTest extends AbstractTestBase {
}
}
}
+
+ @Test
+ public void testValidateACLForArchiveReplication() throws Exception {
+ StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
+ Assert.assertTrue(Boolean.valueOf(
+ StartupProperties.get().getProperty("falcon.security.authorization.enabled")));
+
+ CurrentUser.authenticate(USER);
+ try {
+ InputStream stream = this.getClass().getResourceAsStream(FEED_XML);
+
+ // need a new parser since it caches authorization enabled flag
+ FeedEntityParser feedEntityParser =
+ (FeedEntityParser) EntityParserFactory.getParser(EntityType.FEED);
+ Feed feed = feedEntityParser.parse(stream);
+
+ org.apache.falcon.entity.v0.feed.Cluster feedCluster =
+ FeedHelper.getCluster(feed, "backupCluster");
+ Location location = new Location();
+ location.setType(LocationType.DATA);
+ location.setPath(
+ "s3://falcontesting@hwxasvtesting.blob.core.windows.net/{YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}");
+ Locations locations = new Locations();
+ locations.getLocations().add(location);
+ feedCluster.setLocations(locations);
+
+ Assert.assertNotNull(feed);
+ Assert.assertNotNull(feed.getACL());
+ feed.getACL().setOwner(USER);
+ feed.getACL().setGroup(getPrimaryGroupName());
+
+ try {
+ feedEntityParser.validate(feed);
+ } catch (IllegalArgumentException e) {
+ // this is normal since AWS Secret Access Key is not specified as the password of a s3 URL
+ }
+ } finally {
+ StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false");
+ }
+ }
}