You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2012/02/03 23:55:52 UTC
svn commit: r1240383 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/fs/viewfs/
src/test/java/org/apache/hadoop/fs/
src/test/java/org/apache/hadoop/fs/viewfs/
Author: bobby
Date: Fri Feb 3 22:55:52 2012
New Revision: 1240383
URL: http://svn.apache.org/viewvc?rev=1240383&view=rev
Log:
HADOOP-8013 ViewFileSystem does not honor setVerifyChecksum (Dayrn Sharp via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1240383&r1=1240382&r2=1240383&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Feb 3 22:55:52 2012
@@ -83,17 +83,20 @@ Trunk (unreleased changes)
kerberos. (jitendra)
BUG FIXES
+ HADOOP-8013. ViewFileSystem does not honor setVerifyChecksum
+ (Daryn Sharp via bobby)
+
HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc
(Jon Eagles via bobby)
HADOOP-8001 ChecksumFileSystem's rename doesn't correctly handle checksum
- files. (Daryn Sharp via bobby)
+ files. (Daryn Sharp via bobby)
HADOOP-8006 TestFSInputChecker is failing in trunk.
- (Daryn Sharp via bobby)
+ (Daryn Sharp via bobby)
HADOOP-7998. CheckFileSystem does not correctly honor setVerifyChecksum
- (Daryn Sharp via bobby)
+ (Daryn Sharp via bobby)
HADOOP-7851. Configuration.getClasses() never returns the default value.
(Uma Maheswara Rao G via amarrk)
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1240383&r1=1240382&r2=1240383&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Fri Feb 3 22:55:52 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import java.io.*;
import java.net.URI;
+import java.net.URISyntaxException;
import java.util.EnumSet;
import java.util.List;
@@ -51,6 +52,7 @@ import org.apache.hadoop.util.Progressab
public class FilterFileSystem extends FileSystem {
protected FileSystem fs;
+ private String swapScheme;
/*
* so that extending classes can define it
@@ -77,7 +79,11 @@ public class FilterFileSystem extends Fi
* @param conf the configuration
*/
public void initialize(URI name, Configuration conf) throws IOException {
- fs.initialize(name, conf);
+ super.initialize(name, conf);
+ String scheme = name.getScheme();
+ if (!scheme.equals(fs.getUri().getScheme())) {
+ swapScheme = scheme;
+ }
}
/** Returns a URI whose scheme and authority identify this FileSystem.*/
@@ -96,7 +102,19 @@ public class FilterFileSystem extends Fi
/** Make sure that a path specifies a FileSystem. */
public Path makeQualified(Path path) {
- return fs.makeQualified(path);
+ Path fqPath = fs.makeQualified(path);
+ // swap in our scheme if the filtered fs is using a different scheme
+ if (swapScheme != null) {
+ try {
+ // NOTE: should deal with authority, but too much other stuff is broken
+ fqPath = new Path(
+ new URI(swapScheme, fqPath.toUri().getSchemeSpecificPart(), null)
+ );
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ return fqPath;
}
///////////////////////////////////////////////////////////////
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=1240383&r1=1240382&r2=1240383&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java Fri Feb 3 22:55:52 2012
@@ -24,6 +24,7 @@ import java.util.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
/****************************************************************
* Implement the FileSystem API for the checksumed local filesystem.
@@ -34,21 +35,26 @@ import org.apache.hadoop.classification.
public class LocalFileSystem extends ChecksumFileSystem {
static final URI NAME = URI.create("file:///");
static private Random rand = new Random();
- FileSystem rfs;
public LocalFileSystem() {
this(new RawLocalFileSystem());
}
public FileSystem getRaw() {
- return rfs;
+ return getRawFileSystem();
}
public LocalFileSystem(FileSystem rawLocalFileSystem) {
super(rawLocalFileSystem);
- rfs = rawLocalFileSystem;
}
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
+ // ctor didn't initialize the filtered fs
+ getRawFileSystem().initialize(uri, conf);
+ }
+
/** Convert a path to a File. */
public File pathToFile(Path path) {
return ((RawLocalFileSystem)fs).pathToFile(path);
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1240383&r1=1240382&r2=1240383&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Fri Feb 3 22:55:52 2012
@@ -19,8 +19,6 @@ package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
-import java.net.URISyntaxException;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -82,37 +80,16 @@ class ChRootedFileSystem extends FilterF
/**
* Constructor
- * @param fs base file system
- * @param theRoot chRoot for this file system
- * @throws URISyntaxException
+ * @param uri base file system
+ * @param conf configuration
+ * @throws IOException
*/
- public ChRootedFileSystem(final FileSystem fs, final Path theRoot)
- throws URISyntaxException {
- super(fs);
- makeQualified(theRoot); //check that root is a valid path for fs
- // Would like to call myFs.checkPath(theRoot);
- // but not public
- chRootPathPart = new Path(theRoot.toUri().getPath());
+ public ChRootedFileSystem(final URI uri, Configuration conf)
+ throws IOException {
+ super(FileSystem.get(uri, conf));
+ chRootPathPart = new Path(uri.getPath());
chRootPathPartString = chRootPathPart.toUri().getPath();
- try {
- initialize(fs.getUri(), fs.getConf());
- } catch (IOException e) { // This exception should not be thrown
- throw new RuntimeException("This should not occur");
- }
-
- /*
- * We are making URI include the chrootedPath: e.g. file:///chrootedPath.
- * This is questionable since Path#makeQualified(uri, path) ignores
- * the pathPart of a uri. Since this class is internal we can ignore
- * this issue but if we were to make it external then this needs
- * to be resolved.
- */
- // Handle the two cases:
- // scheme:/// and scheme://authority/
- myUri = new URI(fs.getUri().toString() +
- (fs.getUri().getAuthority() == null ? "" : Path.SEPARATOR) +
- chRootPathPart.toString().substring(1));
-
+ myUri = uri;
workingDir = getHomeDirectory();
// We don't use the wd of the myFs
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1240383&r1=1240382&r2=1240383&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Fri Feb 3 22:55:52 2012
@@ -168,8 +168,7 @@ public class ViewFileSystem extends File
protected
FileSystem getTargetFileSystem(final URI uri)
throws URISyntaxException, IOException {
- return new ChRootedFileSystem(FileSystem.get(uri, config),
- new Path(uri.getPath()));
+ return new ChRootedFileSystem(uri, config);
}
@Override
@@ -464,8 +463,11 @@ public class ViewFileSystem extends File
@Override
public void setVerifyChecksum(final boolean verifyChecksum) {
- // This is a file system level operations, however ViewFileSystem
- // points to many file systems. Noop for ViewFileSystem.
+ List<InodeTree.MountPoint<FileSystem>> mountPoints =
+ fsState.getMountPoints();
+ for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
+ mount.target.targetFileSystem.setVerifyChecksum(verifyChecksum);
+ }
}
public MountPoint[] getMountPoints() {
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java?rev=1240383&r1=1240382&r2=1240383&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java Fri Feb 3 22:55:52 2012
@@ -72,14 +72,15 @@ public final class FileSystemTestHelper
public static String getAbsoluteTestRootDir(FileSystem fSys)
throws IOException {
- if (absTestRootDir == null) {
+ // NOTE: can't cache because of different filesystems!
+ //if (absTestRootDir == null)
if (TEST_ROOT_DIR.startsWith("/")) {
absTestRootDir = TEST_ROOT_DIR;
} else {
absTestRootDir = fSys.getWorkingDirectory().toString() + "/"
+ TEST_ROOT_DIR;
}
- }
+ //}
return absTestRootDir;
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java?rev=1240383&r1=1240382&r2=1240383&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Fri Feb 3 22:55:52 2012
@@ -51,7 +51,7 @@ public class TestChRootedFileSystem {
// ChRoot to the root of the testDirectory
- fSys = new ChRootedFileSystem(fSysTarget, chrootedTo);
+ fSys = new ChRootedFileSystem(chrootedTo.toUri(), conf);
}
@After