You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2014/03/11 01:07:33 UTC
[2/2] git commit: FALCON-257 File system storage wont work with
relative paths. Contributed by Venkatesh Seetharam
FALCON-257 File system storage wont work with relative paths. Contributed by Venkatesh Seetharam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/3158c032
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/3158c032
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/3158c032
Branch: refs/heads/master
Commit: 3158c0321d86c474a6edc117a008fc518215663e
Parents: 5445e10
Author: Venkatesh Seetharam <ve...@hortonworks.com>
Authored: Mon Mar 10 10:50:22 2014 -0700
Committer: Venkatesh Seetharam <ve...@hortonworks.com>
Committed: Mon Mar 10 17:07:37 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../apache/falcon/entity/FileSystemStorage.java | 26 ++++-
.../falcon/entity/FileSystemStorageTest.java | 105 +++++++++++++++++++
3 files changed, 133 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3158c032/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b17437b..64bb353 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -66,6 +66,9 @@ Trunk (Unreleased)
FALCON-287 Record lineage information in post processing (Venkatesh Seetharam)
+ FALCON-257 File system storage wont work with relative paths
+ (Venkatesh Seetharam)
+
OPTIMIZATIONS
FALCON-123 Improve build speeds in falcon. (Srikanth Sundarrajan via Shwetha GS)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3158c032/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index 4bcf271..c2b86b6 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -23,6 +23,7 @@ import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.entity.v0.feed.LocationType;
import org.apache.falcon.entity.v0.feed.Locations;
+import org.apache.falcon.security.CurrentUser;
import org.apache.hadoop.fs.Path;
import java.net.URI;
@@ -166,7 +167,30 @@ public class FileSystemStorage implements Storage {
}
// normalize the path so trailing and double '/' are removed
- return storageUrl + new Path(locationForType.getPath());
+ Path locationPath = new Path(locationForType.getPath());
+ locationPath = locationPath.makeQualified(getDefaultUri(), getWorkingDir());
+
+ if (isRelativePath(locationPath)) {
+ locationPath = new Path(storageUrl + locationPath);
+ }
+
+ return locationPath.toString();
+ }
+
+ private boolean isRelativePath(Path locationPath) {
+ return locationPath.toUri().getAuthority() == null && isStorageUrlATemplate();
+ }
+
+ private boolean isStorageUrlATemplate() {
+ return storageUrl.startsWith(FILE_SYSTEM_URL);
+ }
+
+ private URI getDefaultUri() {
+ return new Path(isStorageUrlATemplate() ? "/" : storageUrl).toUri();
+ }
+
+ public Path getWorkingDir() {
+ return new Path(CurrentUser.getSubject() == null ? "/" : "/user/" + CurrentUser.getUser());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3158c032/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
index 1d56a78..e095f8e 100644
--- a/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/FileSystemStorageTest.java
@@ -20,7 +20,9 @@ package org.apache.falcon.entity;
import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.falcon.security.CurrentUser;
import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -32,6 +34,13 @@ import java.util.List;
*/
public class FileSystemStorageTest {
+ private static final String USER = "falcon";
+
+ @BeforeClass
+ public void setUp() {
+ CurrentUser.authenticate(USER);
+ }
+
@Test
public void testGetType() throws Exception {
final Location location = new Location();
@@ -80,6 +89,102 @@ public class FileSystemStorageTest {
}
@Test
+ public void testFSHomeDir() {
+ final Location location = new Location();
+ location.setPath("foo/bar"); // relative path
+ location.setType(LocationType.DATA);
+ List<Location> locations = new ArrayList<Location>();
+ locations.add(location);
+
+ FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+ Assert.assertEquals(storage.getWorkingDir().toString(), "/user/falcon");
+ }
+
+ @Test
+ public void testGetUriTemplateForDataWithRelativePath() throws Exception {
+ final Location location = new Location();
+ location.setPath("foo/bar"); // relative path
+ location.setType(LocationType.DATA);
+ List<Location> locations = new ArrayList<Location>();
+ locations.add(location);
+
+ FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+ Assert.assertEquals(storage.getUriTemplate(LocationType.DATA),
+ "hdfs://localhost:41020/user/" + USER + "/foo/bar");
+
+ storage = new FileSystemStorage("hdfs://localhost:41020/", locations);
+ Assert.assertEquals(storage.getUriTemplate(LocationType.DATA),
+ "hdfs://localhost:41020/user/" + USER + "/foo/bar");
+ }
+
+ @Test
+ public void testGetUriTemplateForDataWithAbsolutePath() throws Exception {
+ final Location location = new Location();
+ location.setPath("/foo/bar"); // absolute path
+ location.setType(LocationType.DATA);
+ List<Location> locations = new ArrayList<Location>();
+ locations.add(location);
+
+ FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+ Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), "hdfs://localhost:41020/foo/bar");
+
+ storage = new FileSystemStorage("hdfs://localhost:41020/", locations);
+ Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), "hdfs://localhost:41020/foo/bar");
+ }
+
+ @Test
+ public void testGetUriTemplateForDataWithAbsoluteURL() throws Exception {
+ final String absoluteUrl = "s3://host:1000/foo/bar";
+ final Location location = new Location();
+ location.setPath(absoluteUrl); // absolute url
+ location.setType(LocationType.DATA);
+ List<Location> locations = new ArrayList<Location>();
+ locations.add(location);
+
+ FileSystemStorage storage = new FileSystemStorage("hdfs://localhost:41020", locations);
+ Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), absoluteUrl);
+
+ storage = new FileSystemStorage("hdfs://localhost:41020/", locations);
+ Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), absoluteUrl);
+ }
+
+ @DataProvider(name = "locationTestWithRelativePathDataProvider")
+ private Object[][] createLocationTestDataWithRelativePath() {
+ return new Object[][] {
+ {"hdfs://h:0", "localDC/rc/billing/ua2", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"hdfs://h:0/", "localDC/rc/billing/ua2", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"hdfs://h:0", "localDC/rc/billing/ua2/", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"hdfs://h:0/", "localDC/rc/billing/ua2/", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"hdfs://h:0", "localDC/rc/billing/ua2//", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"hdfs://h:0/", "localDC/rc/billing/ua2//", "hdfs://h:0/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}", "localDC/rc/billing/ua2", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}/", "localDC/rc/billing/ua2", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}", "localDC/rc/billing/ua2/", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}/", "localDC/rc/billing/ua2/", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}", "localDC/rc/billing/ua2//", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}/", "localDC/rc/billing/ua2//", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}/", "localDC/rc/billing/ua2//", "${nameNode}/user/" + USER + "/localDC/rc/billing/ua2"},
+ {"${nameNode}", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+ {"${nameNode}/", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+ {"hdfs://h:0", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+ {"hdfs://h:0/", "s3://h:p/localDC/rc/billing/ua2//", "s3://h:p/localDC/rc/billing/ua2"},
+ };
+ }
+
+ @Test (dataProvider = "locationTestWithRelativePathDataProvider")
+ public void testGetUriTemplateWithRelativePath(String storageUrl, String path,
+ String expected) throws Exception {
+ final Location location = new Location();
+ location.setPath(path);
+ location.setType(LocationType.DATA);
+ List<Location> locations = new ArrayList<Location>();
+ locations.add(location);
+
+ FileSystemStorage storage = new FileSystemStorage(storageUrl, locations);
+ Assert.assertEquals(storage.getUriTemplate(LocationType.DATA), expected);
+ }
+
+ @Test
public void testGetUriTemplate() throws Exception {
final Location dataLocation = new Location();
dataLocation.setPath("/data/foo/bar");