You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tu...@apache.org on 2013/07/11 02:10:01 UTC
svn commit: r1502065 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common:
./ src/test/java/org/apache/hadoop/fs/
Author: tucu
Date: Thu Jul 11 00:10:00 2013
New Revision: 1502065
URL: http://svn.apache.org/r1502065
Log:
HADOOP-9355. Abstract symlink tests to use either FileContext or FileSystem. (Andrew Wang via Colin Patrick McCabe)
Added:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1502065&r1=1502064&r2=1502065&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt Thu Jul 11 00:10:00 2013
@@ -139,6 +139,12 @@ Release 2.1.0-beta - 2013-07-02
HADOOP-9661. Allow metrics sources to be extended. (sandyr via tucu)
+ HADOOP-9370. Write FSWrapper class to wrap FileSystem and FileContext for
+ better test coverage. (Andrew Wang via Colin Patrick McCabe)
+
+ HADOOP-9355. Abstract symlink tests to use either FileContext or
+ FileSystem. (Andrew Wang via Colin Patrick McCabe)
+
OPTIMIZATIONS
HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
Added: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java?rev=1502065&view=auto
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java (added)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java Thu Jul 11 00:10:00 2013
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.fs.Options.CreateOpts;
+
+/**
+ * Abstraction of filesystem functionality with additional helper methods
+ * commonly used in tests. This allows generic tests to be written which apply
+ * to the two filesystem abstractions in Hadoop: {@link FileSystem} and
+ * {@link FileContext}.
+ */
+public abstract class FSTestWrapper implements FSWrapper {
+
+ //
+ // Test helper methods taken from FileContextTestHelper
+ //
+
+ protected static final int DEFAULT_BLOCK_SIZE = 1024;
+ protected static final int DEFAULT_NUM_BLOCKS = 2;
+
+ protected String testRootDir = null;
+ protected String absTestRootDir = null;
+
+ public FSTestWrapper(String testRootDir) {
+ // Use default test dir if not provided
+ if (testRootDir == null || testRootDir.isEmpty()) {
+ testRootDir = System.getProperty("test.build.data", "build/test/data");
+ }
+ // salt test dir with some random digits for safe parallel runs
+ this.testRootDir = testRootDir + "/"
+ + RandomStringUtils.randomAlphanumeric(10);
+ }
+
+ public static byte[] getFileData(int numOfBlocks, long blockSize) {
+ byte[] data = new byte[(int) (numOfBlocks * blockSize)];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = (byte) (i % 10);
+ }
+ return data;
+ }
+
+ public Path getTestRootPath() {
+ return makeQualified(new Path(testRootDir));
+ }
+
+ public Path getTestRootPath(String pathString) {
+ return makeQualified(new Path(testRootDir, pathString));
+ }
+
+ // the getAbsolutexxx method is needed because the root test dir
+ // can be messed up by changing the working dir.
+
+ public String getAbsoluteTestRootDir() throws IOException {
+ if (absTestRootDir == null) {
+ if (testRootDir.startsWith("/")) {
+ absTestRootDir = testRootDir;
+ } else {
+ absTestRootDir = getWorkingDirectory().toString() + "/"
+ + testRootDir;
+ }
+ }
+ return absTestRootDir;
+ }
+
+ public Path getAbsoluteTestRootPath() throws IOException {
+ return makeQualified(new Path(getAbsoluteTestRootDir()));
+ }
+
+ abstract public FSTestWrapper getLocalFSWrapper()
+ throws UnsupportedFileSystemException, IOException;
+
+ abstract public Path getDefaultWorkingDirectory() throws IOException;
+
+ /*
+ * Create files with numBlocks blocks each with block size blockSize.
+ */
+ abstract public long createFile(Path path, int numBlocks,
+ CreateOpts... options) throws IOException;
+
+ abstract public long createFile(Path path, int numBlocks, int blockSize)
+ throws IOException;
+
+ abstract public long createFile(Path path) throws IOException;
+
+ abstract public long createFile(String name) throws IOException;
+
+ abstract public long createFileNonRecursive(String name) throws IOException;
+
+ abstract public long createFileNonRecursive(Path path) throws IOException;
+
+ abstract public void appendToFile(Path path, int numBlocks,
+ CreateOpts... options) throws IOException;
+
+ abstract public boolean exists(Path p) throws IOException;
+
+ abstract public boolean isFile(Path p) throws IOException;
+
+ abstract public boolean isDir(Path p) throws IOException;
+
+ abstract public boolean isSymlink(Path p) throws IOException;
+
+ abstract public void writeFile(Path path, byte b[]) throws IOException;
+
+ abstract public byte[] readFile(Path path, int len) throws IOException;
+
+ abstract public FileStatus containsPath(Path path, FileStatus[] dirList)
+ throws IOException;
+
+ abstract public FileStatus containsPath(String path, FileStatus[] dirList)
+ throws IOException;
+
+ enum fileType {
+ isDir, isFile, isSymlink
+ };
+
+ abstract public void checkFileStatus(String path, fileType expectedType)
+ throws IOException;
+
+ abstract public void checkFileLinkStatus(String path, fileType expectedType)
+ throws IOException;
+}
Added: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java?rev=1502065&view=auto
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java (added)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java Thu Jul 11 00:10:00 2013
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.EnumSet;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
+
+/**
+ * Abstraction of filesystem operations that is essentially an interface
+ * extracted from {@link FileContext}.
+ */
+public interface FSWrapper {
+
+ abstract public void setWorkingDirectory(final Path newWDir)
+ throws IOException;
+
+ abstract public Path getWorkingDirectory();
+
+ abstract public Path makeQualified(final Path path);
+
+ abstract public FSDataOutputStream create(final Path f,
+ final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, ParentNotDirectoryException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public void mkdir(final Path dir, final FsPermission permission,
+ final boolean createParent) throws AccessControlException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, UnsupportedFileSystemException, IOException;
+
+ abstract public boolean delete(final Path f, final boolean recursive)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public FSDataInputStream open(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public boolean setReplication(final Path f, final short replication)
+ throws AccessControlException, FileNotFoundException,
+ IOException;
+
+ abstract public void rename(final Path src, final Path dst,
+ final Options.Rename... options) throws AccessControlException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, UnsupportedFileSystemException, IOException;
+
+ abstract public void setPermission(final Path f, final FsPermission permission)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public void setOwner(final Path f, final String username,
+ final String groupname) throws AccessControlException,
+ UnsupportedFileSystemException, FileNotFoundException,
+ IOException;
+
+ abstract public void setTimes(final Path f, final long mtime, final long atime)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public FileChecksum getFileChecksum(final Path f)
+ throws AccessControlException, FileNotFoundException, IOException;
+
+ abstract public FileStatus getFileStatus(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public FileStatus getFileLinkStatus(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public Path getLinkTarget(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public BlockLocation[] getFileBlockLocations(final Path f,
+ final long start, final long len) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException;
+
+ abstract public void createSymlink(final Path target, final Path link,
+ final boolean createParent) throws AccessControlException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, UnsupportedFileSystemException, IOException;
+
+ abstract public RemoteIterator<FileStatus> listStatusIterator(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+
+ abstract public FileStatus[] listStatus(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException;
+}
Added: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java?rev=1502065&view=auto
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java (added)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java Thu Jul 11 00:10:00 2013
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.util.EnumSet;
+
+import org.apache.hadoop.fs.Options.CreateOpts;
+import org.apache.hadoop.fs.Options.CreateOpts.BlockSize;
+import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.AccessControlException;
+import org.junit.Assert;
+
+/**
+ * Helper class for unit tests.
+ */
+public final class FileContextTestWrapper extends FSTestWrapper {
+
+ private final FileContext fc;
+
+ public FileContextTestWrapper(FileContext context) {
+ this(context, null);
+ }
+
+ public FileContextTestWrapper(FileContext context, String rootDir) {
+ super(rootDir);
+ this.fc = context;
+ }
+
+ public FSTestWrapper getLocalFSWrapper()
+ throws UnsupportedFileSystemException {
+ return new FileContextTestWrapper(FileContext.getLocalFSFileContext());
+ }
+
+ public Path getDefaultWorkingDirectory() throws IOException {
+ return getTestRootPath("/user/" + System.getProperty("user.name"))
+ .makeQualified(fc.getDefaultFileSystem().getUri(),
+ fc.getWorkingDirectory());
+ }
+
+ /*
+ * Create files with numBlocks blocks each with block size blockSize.
+ */
+ public long createFile(Path path, int numBlocks, CreateOpts... options)
+ throws IOException {
+ BlockSize blockSizeOpt =
+ (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
+ long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
+ : DEFAULT_BLOCK_SIZE;
+ FSDataOutputStream out =
+ fc.create(path, EnumSet.of(CreateFlag.CREATE), options);
+ byte[] data = getFileData(numBlocks, blockSize);
+ out.write(data, 0, data.length);
+ out.close();
+ return data.length;
+ }
+
+ public long createFile(Path path, int numBlocks, int blockSize)
+ throws IOException {
+ return createFile(path, numBlocks, CreateOpts.blockSize(blockSize),
+ CreateOpts.createParent());
+ }
+
+ public long createFile(Path path) throws IOException {
+ return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.createParent());
+ }
+
+ public long createFile(String name) throws IOException {
+ Path path = getTestRootPath(name);
+ return createFile(path);
+ }
+
+ public long createFileNonRecursive(String name) throws IOException {
+ Path path = getTestRootPath(name);
+ return createFileNonRecursive(path);
+ }
+
+ public long createFileNonRecursive(Path path) throws IOException {
+ return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.donotCreateParent());
+ }
+
+ public void appendToFile(Path path, int numBlocks, CreateOpts... options)
+ throws IOException {
+ BlockSize blockSizeOpt =
+ (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
+ long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
+ : DEFAULT_BLOCK_SIZE;
+ FSDataOutputStream out;
+ out = fc.create(path, EnumSet.of(CreateFlag.APPEND));
+ byte[] data = getFileData(numBlocks, blockSize);
+ out.write(data, 0, data.length);
+ out.close();
+ }
+
+ public boolean exists(Path p) throws IOException {
+ return fc.util().exists(p);
+ }
+
+ public boolean isFile(Path p) throws IOException {
+ try {
+ return fc.getFileStatus(p).isFile();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ public boolean isDir(Path p) throws IOException {
+ try {
+ return fc.getFileStatus(p).isDirectory();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ public boolean isSymlink(Path p) throws IOException {
+ try {
+ return fc.getFileLinkStatus(p).isSymlink();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ public void writeFile(Path path, byte b[]) throws IOException {
+ FSDataOutputStream out =
+ fc.create(path,EnumSet.of(CreateFlag.CREATE), CreateOpts.createParent());
+ out.write(b);
+ out.close();
+ }
+
+ public byte[] readFile(Path path, int len) throws IOException {
+ DataInputStream dis = fc.open(path);
+ byte[] buffer = new byte[len];
+ IOUtils.readFully(dis, buffer, 0, len);
+ dis.close();
+ return buffer;
+ }
+
+ public FileStatus containsPath(Path path, FileStatus[] dirList)
+ throws IOException {
+ for(int i = 0; i < dirList.length; i ++) {
+ if (path.equals(dirList[i].getPath()))
+ return dirList[i];
+ }
+ return null;
+ }
+
+ public FileStatus containsPath(String path, FileStatus[] dirList)
+ throws IOException {
+ return containsPath(new Path(path), dirList);
+ }
+
+ public void checkFileStatus(String path, fileType expectedType)
+ throws IOException {
+ FileStatus s = fc.getFileStatus(new Path(path));
+ Assert.assertNotNull(s);
+ if (expectedType == fileType.isDir) {
+ Assert.assertTrue(s.isDirectory());
+ } else if (expectedType == fileType.isFile) {
+ Assert.assertTrue(s.isFile());
+ } else if (expectedType == fileType.isSymlink) {
+ Assert.assertTrue(s.isSymlink());
+ }
+ Assert.assertEquals(fc.makeQualified(new Path(path)), s.getPath());
+ }
+
+ public void checkFileLinkStatus(String path, fileType expectedType)
+ throws IOException {
+ FileStatus s = fc.getFileLinkStatus(new Path(path));
+ Assert.assertNotNull(s);
+ if (expectedType == fileType.isDir) {
+ Assert.assertTrue(s.isDirectory());
+ } else if (expectedType == fileType.isFile) {
+ Assert.assertTrue(s.isFile());
+ } else if (expectedType == fileType.isSymlink) {
+ Assert.assertTrue(s.isSymlink());
+ }
+ Assert.assertEquals(fc.makeQualified(new Path(path)), s.getPath());
+ }
+
+ //
+ // FileContext wrappers
+ //
+
+ @Override
+ public Path makeQualified(Path path) {
+ return fc.makeQualified(path);
+ }
+
+ @Override
+ public void mkdir(Path dir, FsPermission permission, boolean createParent)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, ParentNotDirectoryException,
+ UnsupportedFileSystemException, IOException {
+ fc.mkdir(dir, permission, createParent);
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ return fc.delete(f, recursive);
+ }
+
+ @Override
+ public FileStatus getFileLinkStatus(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fc.getFileLinkStatus(f);
+ }
+
+ @Override
+ public void createSymlink(Path target, Path link, boolean createParent)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, ParentNotDirectoryException,
+ UnsupportedFileSystemException, IOException {
+ fc.createSymlink(target, link, createParent);
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newWDir) throws IOException {
+ fc.setWorkingDirectory(newWDir);
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return fc.getWorkingDirectory();
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fc.getFileStatus(f);
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, EnumSet<CreateFlag> createFlag,
+ CreateOpts... opts) throws AccessControlException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, UnsupportedFileSystemException, IOException {
+ return fc.create(f, createFlag, opts);
+ }
+
+ @Override
+ public FSDataInputStream open(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fc.open(f);
+ }
+
+ @Override
+ public boolean setReplication(final Path f, final short replication)
+ throws AccessControlException, FileNotFoundException,
+ IOException {
+ return fc.setReplication(f, replication);
+ }
+
+ @Override
+ public Path getLinkTarget(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fc.getLinkTarget(f);
+ }
+
+ @Override
+ public void rename(Path src, Path dst, Rename... options)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, ParentNotDirectoryException,
+ UnsupportedFileSystemException, IOException {
+ fc.rename(src, dst, options);
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(Path f, long start, long len)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ return fc.getFileBlockLocations(f, start, len);
+ }
+
+ @Override
+ public FileChecksum getFileChecksum(Path f) throws AccessControlException,
+ FileNotFoundException, IOException {
+ return fc.getFileChecksum(f);
+ }
+
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ return fc.listStatus(f);
+ }
+
+ @Override
+ public void setPermission(final Path f, final FsPermission permission)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ fc.setPermission(f, permission);
+ }
+
+ @Override
+ public void setOwner(final Path f, final String username,
+ final String groupname) throws AccessControlException,
+ UnsupportedFileSystemException, FileNotFoundException,
+ IOException {
+ fc.setOwner(f, username, groupname);
+ }
+
+ @Override
+ public void setTimes(Path f, long mtime, long atime)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ fc.setTimes(f, mtime, atime);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fc.util().listStatus(f);
+ }
+}
Added: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java?rev=1502065&view=auto
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java (added)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java Thu Jul 11 00:10:00 2013
@@ -0,0 +1,393 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.EnumSet;
+
+import org.apache.hadoop.fs.Options.CreateOpts;
+import org.apache.hadoop.fs.Options.CreateOpts.BlockSize;
+import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.util.Progressable;
+import org.junit.Assert;
+
+/**
+ * Helper class for unit tests.
+ */
+public final class FileSystemTestWrapper extends FSTestWrapper {
+
+ private final FileSystem fs;
+
+ public FileSystemTestWrapper(FileSystem fs) {
+ this(fs, null);
+ }
+
+ public FileSystemTestWrapper(FileSystem fs, String rootDir) {
+ super(rootDir);
+ this.fs = fs;
+ }
+
+ public FSTestWrapper getLocalFSWrapper()
+ throws IOException {
+ return new FileSystemTestWrapper(FileSystem.getLocal(fs.getConf()));
+ }
+
+ public Path getDefaultWorkingDirectory() throws IOException {
+ return getTestRootPath("/user/" + System.getProperty("user.name"))
+ .makeQualified(fs.getUri(),
+ fs.getWorkingDirectory());
+ }
+
+ /*
+ * Create files with numBlocks blocks each with block size blockSize.
+ */
+ public long createFile(Path path, int numBlocks, CreateOpts... options)
+ throws IOException {
+ BlockSize blockSizeOpt =
+ (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
+ long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
+ : DEFAULT_BLOCK_SIZE;
+ FSDataOutputStream out =
+ create(path, EnumSet.of(CreateFlag.CREATE), options);
+ byte[] data = getFileData(numBlocks, blockSize);
+ out.write(data, 0, data.length);
+ out.close();
+ return data.length;
+ }
+
+ public long createFile(Path path, int numBlocks, int blockSize)
+ throws IOException {
+ return createFile(path, numBlocks, CreateOpts.blockSize(blockSize),
+ CreateOpts.createParent());
+ }
+
+ public long createFile(Path path) throws IOException {
+ return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.createParent());
+ }
+
+ public long createFile(String name) throws IOException {
+ Path path = getTestRootPath(name);
+ return createFile(path);
+ }
+
+ public long createFileNonRecursive(String name) throws IOException {
+ Path path = getTestRootPath(name);
+ return createFileNonRecursive(path);
+ }
+
+ public long createFileNonRecursive(Path path) throws IOException {
+ return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.donotCreateParent());
+ }
+
+ public void appendToFile(Path path, int numBlocks, CreateOpts... options)
+ throws IOException {
+ BlockSize blockSizeOpt =
+ (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
+ long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
+ : DEFAULT_BLOCK_SIZE;
+ FSDataOutputStream out;
+ out = fs.append(path);
+ byte[] data = getFileData(numBlocks, blockSize);
+ out.write(data, 0, data.length);
+ out.close();
+ }
+
+ public boolean exists(Path p) throws IOException {
+ return fs.exists(p);
+ }
+
+ public boolean isFile(Path p) throws IOException {
+ try {
+ return fs.getFileStatus(p).isFile();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ public boolean isDir(Path p) throws IOException {
+ try {
+ return fs.getFileStatus(p).isDirectory();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ public boolean isSymlink(Path p) throws IOException {
+ throw new UnsupportedFileSystemException(
+ "FileSystem does not support symlinks");
+ }
+
+ public void writeFile(Path path, byte b[]) throws IOException {
+ FSDataOutputStream out =
+ create(path,EnumSet.of(CreateFlag.CREATE), CreateOpts.createParent());
+ out.write(b);
+ out.close();
+ }
+
+ public byte[] readFile(Path path, int len) throws IOException {
+ DataInputStream dis = fs.open(path);
+ byte[] buffer = new byte[len];
+ IOUtils.readFully(dis, buffer, 0, len);
+ dis.close();
+ return buffer;
+ }
+
+ public FileStatus containsPath(Path path, FileStatus[] dirList)
+ throws IOException {
+ for(int i = 0; i < dirList.length; i ++) {
+ if (path.equals(dirList[i].getPath()))
+ return dirList[i];
+ }
+ return null;
+ }
+
+ public FileStatus containsPath(String path, FileStatus[] dirList)
+ throws IOException {
+ return containsPath(new Path(path), dirList);
+ }
+
+ public void checkFileStatus(String path, fileType expectedType)
+ throws IOException {
+ FileStatus s = fs.getFileStatus(new Path(path));
+ Assert.assertNotNull(s);
+ if (expectedType == fileType.isDir) {
+ Assert.assertTrue(s.isDirectory());
+ } else if (expectedType == fileType.isFile) {
+ Assert.assertTrue(s.isFile());
+ } else if (expectedType == fileType.isSymlink) {
+ Assert.assertTrue(s.isSymlink());
+ }
+ Assert.assertEquals(fs.makeQualified(new Path(path)), s.getPath());
+ }
+
+ public void checkFileLinkStatus(String path, fileType expectedType)
+ throws IOException {
+ throw new UnsupportedFileSystemException(
+ "FileSystem does not support symlinks");
+ }
+
+ //
+ // FileContext wrappers
+ //
+
+ @Override
+ public Path makeQualified(Path path) {
+ return fs.makeQualified(path);
+ }
+
+ @Override
+ public void mkdir(Path dir, FsPermission permission, boolean createParent)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, ParentNotDirectoryException,
+ UnsupportedFileSystemException, IOException {
+ // Note that there is no "mkdir" in FileSystem, it always does
+ // "mkdir -p" (creating parent directories).
+ fs.mkdirs(dir, permission);
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ return fs.delete(f, recursive);
+ }
+
+ @Override
+ public FileStatus getFileLinkStatus(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ throw new UnsupportedFileSystemException(
+ "FileSystem does not support symlinks");
+ }
+
+ @Override
+ public void createSymlink(Path target, Path link, boolean createParent)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, ParentNotDirectoryException,
+ UnsupportedFileSystemException, IOException {
+ throw new UnsupportedFileSystemException(
+ "FileSystem does not support symlinks");
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newWDir) throws IOException {
+ fs.setWorkingDirectory(newWDir);
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return fs.getWorkingDirectory();
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fs.getFileStatus(f);
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, EnumSet<CreateFlag> createFlag,
+ CreateOpts... opts) throws AccessControlException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, UnsupportedFileSystemException, IOException {
+
+ // Need to translate the FileContext-style options into FileSystem-style
+
+ // Permissions with umask
+ CreateOpts.Perms permOpt = (CreateOpts.Perms) CreateOpts.getOpt(
+ CreateOpts.Perms.class, opts);
+ FsPermission umask = FsPermission.getUMask(fs.getConf());
+ FsPermission permission = (permOpt != null) ? permOpt.getValue()
+ : FsPermission.getFileDefault().applyUMask(umask);
+ permission = permission.applyUMask(umask);
+ // Overwrite
+ boolean overwrite = createFlag.contains(CreateFlag.OVERWRITE);
+ // bufferSize
+ int bufferSize = fs.getConf().getInt(
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
+ CreateOpts.BufferSize bufOpt = (CreateOpts.BufferSize) CreateOpts.getOpt(
+ CreateOpts.BufferSize.class, opts);
+ bufferSize = (bufOpt != null) ? bufOpt.getValue() : bufferSize;
+ // replication
+ short replication = fs.getDefaultReplication(f);
+ CreateOpts.ReplicationFactor repOpt =
+ (CreateOpts.ReplicationFactor) CreateOpts.getOpt(
+ CreateOpts.ReplicationFactor.class, opts);
+ replication = (repOpt != null) ? repOpt.getValue() : replication;
+ // blockSize
+ long blockSize = fs.getDefaultBlockSize(f);
+ CreateOpts.BlockSize blockOpt = (CreateOpts.BlockSize) CreateOpts.getOpt(
+ CreateOpts.BlockSize.class, opts);
+ blockSize = (blockOpt != null) ? blockOpt.getValue() : blockSize;
+ // Progressable
+ Progressable progress = null;
+ CreateOpts.Progress progressOpt = (CreateOpts.Progress) CreateOpts.getOpt(
+ CreateOpts.Progress.class, opts);
+ progress = (progressOpt != null) ? progressOpt.getValue() : progress;
+ return fs.create(f, permission, overwrite, bufferSize, replication,
+ blockSize, progress);
+ }
+
+ @Override
+ public FSDataInputStream open(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fs.open(f);
+ }
+
+ @Override
+ public Path getLinkTarget(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ throw new UnsupportedFileSystemException(
+ "FileSystem does not support symlinks");
+ }
+
+ @Override
+ public boolean setReplication(final Path f, final short replication)
+ throws AccessControlException, FileNotFoundException,
+ IOException {
+ return fs.setReplication(f, replication);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void rename(Path src, Path dst, Rename... options)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, ParentNotDirectoryException,
+ UnsupportedFileSystemException, IOException {
+ fs.rename(src, dst, options);
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(Path f, long start, long len)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ return fs.getFileBlockLocations(f, start, len);
+ }
+
+ @Override
+ public FileChecksum getFileChecksum(Path f) throws AccessControlException,
+ FileNotFoundException, IOException {
+ return fs.getFileChecksum(f);
+ }
+
+ private class FakeRemoteIterator<E> implements RemoteIterator<E> {
+
+ private E[] elements;
+ private int count;
+
+ FakeRemoteIterator(E[] elements) {
+ this.elements = elements;
+ count = 0;
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return count < elements.length;
+ }
+
+ @Override
+ public E next() throws IOException {
+ if (hasNext()) {
+ return elements[count++];
+ }
+ return null;
+ }
+ }
+
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ // Fake the RemoteIterator, because FileSystem has no such thing
+ FileStatus[] statuses = fs.listStatus(f);
+ return new FakeRemoteIterator<FileStatus>(statuses);
+ }
+
+ @Override
+ public void setPermission(final Path f, final FsPermission permission)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ fs.setPermission(f, permission);
+ }
+
+ @Override
+ public void setOwner(final Path f, final String username,
+ final String groupname) throws AccessControlException,
+ UnsupportedFileSystemException, FileNotFoundException,
+ IOException {
+ fs.setOwner(f, username, groupname);
+ }
+
+ @Override
+ public void setTimes(Path f, long mtime, long atime)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ fs.setTimes(f, mtime, atime);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ return fs.listStatus(f);
+ }
+}