You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2007/09/23 23:47:09 UTC
svn commit: r578607 - in /lucene/hadoop/branches/branch-0.14: CHANGES.txt
src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Author: tomwhite
Date: Sun Sep 23 14:47:07 2007
New Revision: 578607
URL: http://svn.apache.org/viewvc?rev=578607&view=rev
Log:
HADOOP-1783. Make S3 FileSystem return Paths fully-qualified with scheme and host.
Modified:
lucene/hadoop/branches/branch-0.14/CHANGES.txt
lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
Modified: lucene/hadoop/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/CHANGES.txt?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/CHANGES.txt (original)
+++ lucene/hadoop/branches/branch-0.14/CHANGES.txt Sun Sep 23 14:47:07 2007
@@ -1,6 +1,13 @@
Hadoop Change Log
+Release 0.14.2 (unreleased changes)
+
+ BUG FIXES
+
+ HADOOP-1783. Make S3 FileSystem return Paths fully-qualified with
+ scheme and host. (tomwhite)
+
Release 0.14.1 - 2007-09-04
Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Sun Sep 23 14:47:07 2007
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -169,8 +170,11 @@
} else if (inode.isFile()) {
return new Path[] { absolutePath };
} else { // directory
- Set<Path> paths = store.listSubPaths(absolutePath);
- return paths.toArray(new Path[0]);
+ ArrayList<Path> ret = new ArrayList<Path>();
+ for (Path p : store.listSubPaths(absolutePath)) {
+ ret.add(p.makeQualified(this));
+ }
+ return ret.toArray(new Path[0]);
}
}
Modified: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java Sun Sep 23 14:47:07 2007
@@ -56,7 +56,7 @@
}
public void deleteINode(Path path) throws IOException {
- inodes.remove(path);
+ inodes.remove(normalize(path));
}
public void deleteBlock(Block block) throws IOException {
@@ -64,7 +64,7 @@
}
public boolean inodeExists(Path path) throws IOException {
- return inodes.containsKey(path);
+ return inodes.containsKey(normalize(path));
}
public boolean blockExists(long blockId) throws IOException {
@@ -72,7 +72,7 @@
}
public INode retrieveINode(Path path) throws IOException {
- return inodes.get(path);
+ return inodes.get(normalize(path));
}
public File retrieveBlock(Block block, long byteRangeStart) throws IOException {
@@ -101,10 +101,11 @@
}
public Set<Path> listSubPaths(Path path) throws IOException {
+ Path normalizedPath = normalize(path);
// This is inefficient but more than adequate for testing purposes.
Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(path).keySet()) {
- if (path.equals(p.getParent())) {
+ for (Path p : inodes.tailMap(normalizedPath).keySet()) {
+ if (normalizedPath.equals(p.getParent())) {
subPaths.add(p);
}
}
@@ -112,13 +113,14 @@
}
public Set<Path> listDeepSubPaths(Path path) throws IOException {
- String pathString = path.toUri().getPath();
+ Path normalizedPath = normalize(path);
+ String pathString = normalizedPath.toUri().getPath();
if (!pathString.endsWith("/")) {
pathString += "/";
}
// This is inefficient but more than adequate for testing purposes.
Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(path).keySet()) {
+ for (Path p : inodes.tailMap(normalizedPath).keySet()) {
if (p.toUri().getPath().startsWith(pathString)) {
subPaths.add(p);
}
@@ -127,7 +129,7 @@
}
public void storeINode(Path path, INode inode) throws IOException {
- inodes.put(path, inode);
+ inodes.put(normalize(path), inode);
}
public void storeBlock(Block block, File file) throws IOException {
@@ -146,6 +148,13 @@
}
}
blocks.put(block.getId(), out.toByteArray());
+ }
+
+ private Path normalize(Path path) {
+ if (!path.isAbsolute()) {
+ throw new IllegalArgumentException("Path must be absolute: " + path);
+ }
+ return new Path(path.toUri().getPath());
}
public void purge() throws IOException {
Modified: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java?rev=578607&r1=578606&r2=578607&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/fs/s3/S3FileSystemBaseTest.java Sun Sep 23 14:47:07 2007
@@ -111,17 +111,21 @@
Path[] paths = s3FileSystem.listPaths(new Path("/"));
assertEquals(1, paths.length);
- assertEquals(new Path("/test"), paths[0]);
+ assertEquals(new Path("/test").makeQualified(s3FileSystem), paths[0]);
paths = s3FileSystem.listPaths(new Path("/test"));
assertEquals(1, paths.length);
- assertEquals(new Path("/test/hadoop"), paths[0]);
+ assertEquals(new Path("/test/hadoop").makeQualified(s3FileSystem),
+ paths[0]);
paths = s3FileSystem.listPaths(new Path("/test/hadoop"));
assertEquals(3, paths.length);
- assertEquals(new Path("/test/hadoop/a"), paths[0]);
- assertEquals(new Path("/test/hadoop/b"), paths[1]);
- assertEquals(new Path("/test/hadoop/c"), paths[2]);
+ assertEquals(new Path("/test/hadoop/a").makeQualified(s3FileSystem),
+ paths[0]);
+ assertEquals(new Path("/test/hadoop/b").makeQualified(s3FileSystem),
+ paths[1]);
+ assertEquals(new Path("/test/hadoop/c").makeQualified(s3FileSystem),
+ paths[2]);
paths = s3FileSystem.listPaths(new Path("/test/hadoop/a"));
assertEquals(0, paths.length);