You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/11/24 00:42:49 UTC
[1/3] hadoop git commit: HADOOP-13605. Clean up FileSystem javadocs,
logging; improve diagnostics on FS load. Contributed by Steve Loughran
Repository: hadoop
Updated Branches:
refs/heads/trunk de4894936 -> 860d49aa6
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index b18b5f6..201d397 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -419,7 +419,7 @@ If the filesystem is not location aware, it SHOULD return
BlockLocation(["localhost:9866"] ,
["localhost"],
["/default/localhost"]
- 0, F.getLen())
+ 0, f.getLen())
] ;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
index f232735..b84d66a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDefaultUri.java
@@ -21,14 +21,14 @@ import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Test;
+import static org.apache.hadoop.test.LambdaTestUtils.*;
/**
* Test default URI related APIs in {@link FileSystem}.
@@ -69,15 +69,12 @@ public class TestDefaultUri {
}
@Test
- public void tetGetDefaultUriNoSchemeTrailingSlash() {
+ public void tetGetDefaultUriNoSchemeTrailingSlash() throws Exception {
conf.set(FS_DEFAULT_NAME_KEY, "nn_host/");
- try {
- FileSystem.getDefaultUri(conf);
- fail("Expect IAE: No scheme in default FS");
- } catch (IllegalArgumentException e) {
- GenericTestUtils.assertExceptionContains(
- "No scheme in default FS", e);
- }
+ intercept(IllegalArgumentException.class,
+ "No scheme in default FS",
+ () -> FileSystem.getDefaultUri(conf));
+
}
@Test
@@ -88,28 +85,19 @@ public class TestDefaultUri {
}
@Test
- public void tetFsGetNoScheme() throws IOException {
+ public void tetFsGetNoScheme() throws Exception {
// Bare host name or address indicates hdfs scheme
conf.set(FS_DEFAULT_NAME_KEY, "nn_host");
- try {
- FileSystem.get(conf);
- fail("Expect IOE: No FileSystem for scheme: hdfs");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains(
- "No FileSystem for scheme: hdfs", e);
- }
+ intercept(UnsupportedFileSystemException.class, "hdfs",
+ () -> FileSystem.get(conf));
}
@Test
- public void tetFsGetNoSchemeTrailingSlash() throws IOException {
+ public void tetFsGetNoSchemeTrailingSlash() throws Exception {
// Bare host name or address with trailing slash is invalid
conf.set(FS_DEFAULT_NAME_KEY, "nn_host/");
- try {
- FileSystem.get(conf);
- fail("Expect IAE: No scheme in default FS");
- } catch (IllegalArgumentException e) {
- GenericTestUtils.assertExceptionContains(
- "No scheme in default FS", e);
- }
+ intercept(IllegalArgumentException.class,
+ "No scheme in default FS",
+ () -> FileSystem.get(conf));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
index 07b07dc..69ef71e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
@@ -95,16 +95,14 @@ public class TestFileSystemCaching {
try {
fs = FileSystem.get(URI.create("//host"), conf);
fail("got fs with auth but no scheme");
- } catch (Exception e) {
- assertEquals("No FileSystem for scheme: null", e.getMessage());
+ } catch (UnsupportedFileSystemException e) {
}
-
+
// no scheme, different auth
try {
fs = FileSystem.get(URI.create("//host2"), conf);
fail("got fs with auth but no scheme");
- } catch (Exception e) {
- assertEquals("No FileSystem for scheme: null", e.getMessage());
+ } catch (UnsupportedFileSystemException e) {
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HADOOP-13605. Clean up FileSystem javadocs,
logging; improve diagnostics on FS load. Contributed by Steve Loughran
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/860d49aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 9e98455..f581f61 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -75,6 +75,8 @@ import org.apache.htrace.core.TraceScope;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
@@ -88,79 +90,118 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
* <p>
*
* All user code that may potentially use the Hadoop Distributed
- * File System should be written to use a FileSystem object. The
- * Hadoop DFS is a multi-machine system that appears as a single
- * disk. It's useful because of its fault tolerance and potentially
- * very large capacity.
- *
+ * File System should be written to use a FileSystem object or its
+ * successor, {@link FileContext}.
+ *
* <p>
* The local implementation is {@link LocalFileSystem} and distributed
- * implementation is DistributedFileSystem.
+ * implementation is DistributedFileSystem. There are other implementations
+ * for object stores and (outside the Apache Hadoop codebase),
+ * third party filesystems.
+ * <p>
+ * Notes
+ * <ol>
+ * <li>The behaviour of the filesystem is
+ * <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/filesystem.html">
+ * specified in the Hadoop documentation. </a>
+ * However, the normative specification of the behavior of this class is
+ * actually HDFS: if HDFS does not behave the way these Javadocs or
+ * the specification in the Hadoop documentations define, assume that
+ * the documentation is incorrect.
+ * </li>
+ * <li>The term {@code FileSystem} refers to an instance of this class.</li>
+ * <li>The acronym "FS" is used as an abbreviation of FileSystem.</li>
+ * <li>The term {@code filesystem} refers to the distributed/local filesystem
+ * itself, rather than the class used to interact with it.</li>
+ * <li>The term "file" refers to a file in the remote filesystem,
+ * rather than instances of {@code java.io.File}.</li>
+ * </ol>
*****************************************************************/
+@SuppressWarnings("DeprecatedIsStillUsed")
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class FileSystem extends Configured implements Closeable {
- public static final String FS_DEFAULT_NAME_KEY =
+ public static final String FS_DEFAULT_NAME_KEY =
CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
- public static final String DEFAULT_FS =
+ public static final String DEFAULT_FS =
CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
+ /**
+ * This log is widely used in the org.apache.hadoop.fs code and tests,
+ * so must be considered something to only be changed with care.
+ */
+ @InterfaceAudience.Private
public static final Log LOG = LogFactory.getLog(FileSystem.class);
/**
- * Priority of the FileSystem shutdown hook.
+ * The SLF4J logger to use in logging within the FileSystem class itself.
+ */
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(FileSystem.class);
+
+ /**
+ * Priority of the FileSystem shutdown hook: {@value}.
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 10;
+ /**
+ * Prefix for trash directory: {@value}.
+ */
public static final String TRASH_PREFIX = ".Trash";
public static final String USER_HOME_PREFIX = "/user";
- /** FileSystem cache */
+ /** FileSystem cache. */
static final Cache CACHE = new Cache();
/** The key this instance is stored under in the cache. */
private Cache.Key key;
- /** Recording statistics per a FileSystem class */
- private static final Map<Class<? extends FileSystem>, Statistics>
- statisticsTable =
- new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
-
+ /** Recording statistics per a FileSystem class. */
+ private static final Map<Class<? extends FileSystem>, Statistics>
+ statisticsTable = new IdentityHashMap<>();
+
/**
* The statistics for this file system.
*/
protected Statistics statistics;
/**
- * A cache of files that should be deleted when filesystem is closed
+ * A cache of files that should be deleted when the FileSystem is closed
* or the JVM is exited.
*/
- private Set<Path> deleteOnExit = new TreeSet<Path>();
-
+ private final Set<Path> deleteOnExit = new TreeSet<>();
+
+ /**
+ * Should symbolic links be resolved by {@link FileSystemLinkResolver}.
+ * Set to the value of
+ * {@link CommonConfigurationKeysPublic#FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY}
+ */
boolean resolveSymlinks;
/**
- * This method adds a file system for testing so that we can find it later. It
- * is only for testing.
+ * This method adds a FileSystem instance to the cache so that it can
+ * be retrieved later. It is only for testing.
* @param uri the uri to store it under
* @param conf the configuration to store it under
- * @param fs the file system to store
- * @throws IOException
+ * @param fs the FileSystem to store
+ * @throws IOException if the current user cannot be determined.
*/
+ @VisibleForTesting
static void addFileSystemForTesting(URI uri, Configuration conf,
FileSystem fs) throws IOException {
CACHE.map.put(new Cache.Key(uri, conf), fs);
}
/**
- * Get a filesystem instance based on the uri, the passed
- * configuration and the user
+ * Get a FileSystem instance based on the uri, the passed in
+ * configuration and the user.
* @param uri of the filesystem
* @param conf the configuration to use
* @param user to perform the get as
* @return the filesystem instance
- * @throws IOException
- * @throws InterruptedException
+ * @throws IOException failure to load
+ * @throws InterruptedException If the {@code UGI.doAs()} call was
+ * somehow interrupted.
*/
public static FileSystem get(final URI uri, final Configuration conf,
final String user) throws IOException, InterruptedException {
@@ -177,14 +218,15 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Returns the configured filesystem implementation.
+ * Returns the configured FileSystem implementation.
* @param conf the configuration to use
*/
public static FileSystem get(Configuration conf) throws IOException {
return get(getDefaultUri(conf), conf);
}
-
- /** Get the default filesystem URI from a configuration.
+
+ /**
+ * Get the default FileSystem URI from a configuration.
* @param conf the configuration to use
* @return the uri of the default filesystem
*/
@@ -196,7 +238,8 @@ public abstract class FileSystem extends Configured implements Closeable {
return uri;
}
- /** Set the default filesystem URI in a configuration.
+ /**
+ * Set the default FileSystem URI in a configuration.
* @param conf the configuration to alter
* @param uri the new default filesystem uri
*/
@@ -204,7 +247,7 @@ public abstract class FileSystem extends Configured implements Closeable {
conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
}
- /** Set the default filesystem URI in a configuration.
+ /** Set the default FileSystem URI in a configuration.
* @param conf the configuration to alter
* @param uri the new default filesystem uri
*/
@@ -212,10 +255,21 @@ public abstract class FileSystem extends Configured implements Closeable {
setDefaultUri(conf, URI.create(fixName(uri)));
}
- /** Called after a new FileSystem instance is constructed.
- * @param name a uri whose authority section names the host, port, etc.
+ /**
+ * Initialize a FileSystem.
+ *
+ * Called after the new FileSystem instance is constructed, and before it
+ * is ready for use.
+ *
+ * FileSystem implementations overriding this method MUST forward it to
+ * their superclass, though the order in which it is done, and whether
+ * to alter the configuration before the invocation are options of the
+ * subclass.
+ * @param name a URI whose authority section names the host, port, etc.
* for this FileSystem
* @param conf the configuration
+ * @throws IOException on any failure to initialize this instance.
+ * @throws IllegalArgumentException if the URI is considered invalid.
*/
public void initialize(URI name, Configuration conf) throws IOException {
final String scheme;
@@ -226,27 +280,34 @@ public abstract class FileSystem extends Configured implements Closeable {
}
statistics = getStatistics(scheme, getClass());
resolveSymlinks = conf.getBoolean(
- CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
- CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
+ CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+ CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
}
/**
- * Return the protocol scheme for the FileSystem.
- * <p/>
+ * Return the protocol scheme for this FileSystem.
+ * <p>
* This implementation throws an <code>UnsupportedOperationException</code>.
*
- * @return the protocol scheme for the FileSystem.
+ * @return the protocol scheme for this FileSystem.
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public String getScheme() {
- throw new UnsupportedOperationException("Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation");
+ throw new UnsupportedOperationException("Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
}
- /** Returns a URI whose scheme and authority identify this FileSystem.*/
+ /**
+ * Returns a URI which identifies this FileSystem.
+ *
+ * @return the URI of this filesystem.
+ */
public abstract URI getUri();
-
+
/**
* Return a canonicalized form of this FileSystem's URI.
- *
+ *
* The default implementation simply calls {@link #canonicalizeUri(URI)}
* on the filesystem's own URI, so subclasses typically only need to
* implement that method.
@@ -256,16 +317,17 @@ public abstract class FileSystem extends Configured implements Closeable {
protected URI getCanonicalUri() {
return canonicalizeUri(getUri());
}
-
+
/**
* Canonicalize the given URI.
- *
- * This is filesystem-dependent, but may for example consist of
+ *
+ * This is implementation-dependent, and may for example consist of
* canonicalizing the hostname using DNS and adding the default
* port if not specified.
- *
+ *
* The default implementation simply fills in the default port if
- * not specified and if the filesystem has a default port.
+ * not specified and if {@link #getDefaultPort()} returns a
+ * default port.
*
* @return URI
* @see NetUtils#getCanonicalUri(URI, int)
@@ -283,12 +345,12 @@ public abstract class FileSystem extends Configured implements Closeable {
uri);
}
}
-
+
return uri;
}
-
+
/**
- * Get the default port for this file system.
+ * Get the default port for this FileSystem.
* @return the default port or 0 if there isn't one
*/
protected int getDefaultPort() {
@@ -301,25 +363,25 @@ public abstract class FileSystem extends Configured implements Closeable {
absOrFqPath.checkNotSchemeWithRelative();
absOrFqPath.checkNotRelative();
- // Uses the default file system if not fully qualified
+ // Uses the default FileSystem if not fully qualified
return get(absOrFqPath.toUri(), conf);
}
/**
- * Get a canonical service name for this file system. The token cache is
- * the only user of the canonical service name, and uses it to lookup this
- * filesystem's service tokens.
- * If file system provides a token of its own then it must have a canonical
- * name, otherwise canonical name can be null.
- *
- * Default Impl: If the file system has child file systems
- * (such as an embedded file system) then it is assumed that the fs has no
+ * Get a canonical service name for this FileSystem.
+ * The token cache is the only user of the canonical service name,
+ * and uses it to lookup this FileSystem's service tokens.
+ * If the file system provides a token of its own then it must have a
+ * canonical name, otherwise the canonical name can be null.
+ *
+ * Default implementation: If the FileSystem has child file systems
+ * (such as an embedded file system) then it is assumed that the FS has no
* tokens of its own and hence returns a null name; otherwise a service
* name is built using Uri and port.
- *
+ *
* @return a service string that uniquely identifies this file system, null
* if the filesystem does not implement tokens
- * @see SecurityUtil#buildDTServiceName(URI, int)
+ * @see SecurityUtil#buildDTServiceName(URI, int)
*/
@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
public String getCanonicalServiceName() {
@@ -328,28 +390,29 @@ public abstract class FileSystem extends Configured implements Closeable {
: null;
}
- /** @deprecated call #getUri() instead.*/
+ /** @deprecated call {@link #getUri()} instead.*/
@Deprecated
public String getName() { return getUri().toString(); }
- /** @deprecated call #get(URI,Configuration) instead. */
+ /** @deprecated call {@link #get(URI, Configuration)} instead. */
@Deprecated
public static FileSystem getNamed(String name, Configuration conf)
throws IOException {
return get(URI.create(fixName(name)), conf);
}
-
+
/** Update old-format filesystem names, for back-compatibility. This should
* eventually be replaced with a checkName() method that throws an exception
- * for old-format names. */
+ * for old-format names.
+ */
private static String fixName(String name) {
// convert old-format name to new-format name
if (name.equals("local")) { // "local" is now "file:///".
- LOG.warn("\"local\" is a deprecated filesystem name."
+ LOGGER.warn("\"local\" is a deprecated filesystem name."
+" Use \"file:///\" instead.");
name = "file:///";
} else if (name.indexOf('/')==-1) { // unqualified is "hdfs://"
- LOG.warn("\""+name+"\" is a deprecated filesystem name."
+ LOGGER.warn("\""+name+"\" is a deprecated filesystem name."
+" Use \"hdfs://"+name+"/\" instead.");
name = "hdfs://"+name;
}
@@ -357,19 +420,36 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Get the local file system.
- * @param conf the configuration to configure the file system with
+ * Get the local FileSystem.
+ * @param conf the configuration to configure the FileSystem with
+ * if it is newly instantiated.
* @return a LocalFileSystem
+ * @throws IOException if somehow the local FS cannot be instantiated.
*/
public static LocalFileSystem getLocal(Configuration conf)
throws IOException {
return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
}
- /** Returns the FileSystem for this URI's scheme and authority. The scheme
- * of the URI determines a configuration property name,
- * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
- * The entire URI is passed to the FileSystem instance's initialize method.
+ /**
+ * Get a FileSystem for this URI's scheme and authority.
+ * <ol>
+ * <li>
+ * If the configuration has the property
+ * {@code "fs.$SCHEME.impl.disable.cache"} set to true,
+ * a new instance will be created, initialized with the supplied URI and
+ * configuration, then returned without being cached.
+ * </li>
+ * <li>
+ * If the there is a cached FS instance matching the same URI, it will
+ * be returned.
+ * </li>
+ * <li>
+ * Otherwise: a new FS instance will be created, initialized with the
+ * configuration and URI, cached and returned to the caller.
+ * </li>
+ * </ol>
+ * @throws IOException if the FileSystem cannot be instantiated.
*/
public static FileSystem get(URI uri, Configuration conf) throws IOException {
String scheme = uri.getScheme();
@@ -386,9 +466,9 @@ public abstract class FileSystem extends Configured implements Closeable {
return get(defaultUri, conf); // return default
}
}
-
String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
if (conf.getBoolean(disableCacheName, false)) {
+ LOGGER.debug("Bypassing cache to create filesystem {}", uri);
return createFileSystem(uri, conf);
}
@@ -396,14 +476,15 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Returns the FileSystem for this URI's scheme and authority and the
- * passed user. Internally invokes {@link #newInstance(URI, Configuration)}
+ * Returns the FileSystem for this URI's scheme and authority and the
+ * given user. Internally invokes {@link #newInstance(URI, Configuration)}
* @param uri of the filesystem
* @param conf the configuration to use
* @param user to perform the get as
* @return filesystem instance
- * @throws IOException
- * @throws InterruptedException
+ * @throws IOException if the FileSystem cannot be instantiated.
+ * @throws InterruptedException If the {@code UGI.doAs()} call was
+ * somehow interrupted.
*/
public static FileSystem newInstance(final URI uri, final Configuration conf,
final String user) throws IOException, InterruptedException {
@@ -414,47 +495,56 @@ public abstract class FileSystem extends Configured implements Closeable {
return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
@Override
public FileSystem run() throws IOException {
- return newInstance(uri,conf);
+ return newInstance(uri, conf);
}
});
}
- /** Returns the FileSystem for this URI's scheme and authority. The scheme
- * of the URI determines a configuration property name,
- * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
+
+ /**
+ * Returns the FileSystem for this URI's scheme and authority.
* The entire URI is passed to the FileSystem instance's initialize method.
* This always returns a new FileSystem object.
+ * @param uri FS URI
+ * @param config configuration to use
+ * @return the new FS instance
+ * @throws IOException FS creation or initialization failure.
*/
- public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
+ public static FileSystem newInstance(URI uri, Configuration config)
+ throws IOException {
String scheme = uri.getScheme();
String authority = uri.getAuthority();
if (scheme == null) { // no scheme: use default FS
- return newInstance(conf);
+ return newInstance(config);
}
if (authority == null) { // no authority
- URI defaultUri = getDefaultUri(conf);
+ URI defaultUri = getDefaultUri(config);
if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
&& defaultUri.getAuthority() != null) { // & default has authority
- return newInstance(defaultUri, conf); // return default
+ return newInstance(defaultUri, config); // return default
}
}
- return CACHE.getUnique(uri, conf);
+ return CACHE.getUnique(uri, config);
}
- /** Returns a unique configured filesystem implementation.
+ /**
+ * Returns a unique configured FileSystem implementation for the default
+ * filesystem of the supplied configuration.
* This always returns a new FileSystem object.
* @param conf the configuration to use
+ * @return the new FS instance
+ * @throws IOException FS creation or initialization failure.
*/
public static FileSystem newInstance(Configuration conf) throws IOException {
return newInstance(getDefaultUri(conf), conf);
}
/**
- * Get a unique local file system object
- * @param conf the configuration to configure the file system with
- * @return a LocalFileSystem
- * This always returns a new FileSystem object.
+ * Get a unique local FileSystem object.
+ * @param conf the configuration to configure the FileSystem with
+ * @return a new LocalFileSystem object.
+ * @throws IOException FS creation or initialization failure.
*/
public static LocalFileSystem newInstanceLocal(Configuration conf)
throws IOException {
@@ -462,65 +552,70 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Close all cached filesystems. Be sure those filesystems are not
- * used anymore.
- *
- * @throws IOException
+ * Close all cached FileSystem instances. After this operation, they
+ * may not be used in any operations.
+ *
+ * @throws IOException a problem arose closing one or more filesystem.
*/
public static void closeAll() throws IOException {
CACHE.closeAll();
}
/**
- * Close all cached filesystems for a given UGI. Be sure those filesystems
- * are not used anymore.
+ * Close all cached FileSystem instances for a given UGI.
+ * Be sure those filesystems are not used anymore.
* @param ugi user group info to close
- * @throws IOException
+ * @throws IOException a problem arose closing one or more filesystem.
*/
- public static void closeAllForUGI(UserGroupInformation ugi)
+ public static void closeAllForUGI(UserGroupInformation ugi)
throws IOException {
CACHE.closeAll(ugi);
}
- /**
- * Make sure that a path specifies a FileSystem.
- * @param path to use
+ /**
+ * Qualify a path to one which uses this FileSystem and, if relative,
+ * made absolute.
+ * @param path to qualify.
+ * @return this path if it contains a scheme and authority and is absolute, or
+ * a new path that includes a path and authority and is fully qualified
+ * @see Path#makeQualified(URI, Path)
+ * @throws IllegalArgumentException if the path has a schema/URI different
+ * from this FileSystem.
*/
public Path makeQualified(Path path) {
checkPath(path);
return path.makeQualified(this.getUri(), this.getWorkingDirectory());
}
-
+
/**
- * Get a new delegation token for this file system.
+ * Get a new delegation token for this FileSystem.
* This is an internal method that should have been declared protected
* but wasn't historically.
* Callers should use {@link #addDelegationTokens(String, Credentials)}
- *
+ *
* @param renewer the account name that is allowed to renew the token.
- * @return a new delegation token
- * @throws IOException
+ * @return a new delegation token or null if the FS does not support tokens.
+ * @throws IOException on any problem obtaining a token
*/
@InterfaceAudience.Private()
public Token<?> getDelegationToken(String renewer) throws IOException {
return null;
}
-
+
/**
* Obtain all delegation tokens used by this FileSystem that are not
- * already present in the given Credentials. Existing tokens will neither
+ * already present in the given Credentials. Existing tokens will neither
* be verified as valid nor having the given renewer. Missing tokens will
* be acquired and added to the given Credentials.
- *
- * Default Impl: works for simple fs with its own token
- * and also for an embedded fs whose tokens are those of its
- * children file system (i.e. the embedded fs has not tokens of its
- * own).
- *
+ *
+ * Default Impl: works for simple FS with its own token
+ * and also for an embedded FS whose tokens are those of its
+ * child FileSystems (i.e. the embedded FS has no tokens of its own).
+ *
* @param renewer the user allowed to renew the delegation tokens
* @param credentials cache in which to add new delegation tokens
* @return list of new delegation tokens
- * @throws IOException
+ * @throws IOException problems obtaining a token
*/
@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
public Token<?>[] addDelegationTokens(
@@ -528,18 +623,18 @@ public abstract class FileSystem extends Configured implements Closeable {
if (credentials == null) {
credentials = new Credentials();
}
- final List<Token<?>> tokens = new ArrayList<Token<?>>();
+ final List<Token<?>> tokens = new ArrayList<>();
collectDelegationTokens(renewer, credentials, tokens);
return tokens.toArray(new Token<?>[tokens.size()]);
}
-
+
/**
- * Recursively obtain the tokens for this FileSystem and all descended
- * FileSystems as determined by getChildFileSystems().
+ * Recursively obtain the tokens for this FileSystem and all descendant
+ * FileSystems as determined by {@link #getChildFileSystems()}.
* @param renewer the user allowed to renew the delegation tokens
* @param credentials cache in which to add the new delegation tokens
* @param tokens list in which to add acquired tokens
- * @throws IOException
+ * @throws IOException problems obtaining a token
*/
private void collectDelegationTokens(final String renewer,
final Credentials credentials,
@@ -570,30 +665,34 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Get all the immediate child FileSystems embedded in this FileSystem.
* It does not recurse and get grand children. If a FileSystem
- * has multiple child FileSystems, then it should return a unique list
+ * has multiple child FileSystems, then it must return a unique list
* of those FileSystems. Default is to return null to signify no children.
- *
- * @return FileSystems used by this FileSystem
+ *
+ * @return FileSystems that are direct children of this FileSystem,
+ * or null for "no children"
*/
@InterfaceAudience.LimitedPrivate({ "HDFS" })
@VisibleForTesting
public FileSystem[] getChildFileSystems() {
return null;
}
-
- /** create a file with the provided permission
+
+ /**
+ * Create a file with the provided permission.
+ *
* The permission of the file is set to be the provided permission as in
* setPermission, not permission&~umask
- *
- * It is implemented using two RPCs. It is understood that it is inefficient,
+ *
+ * The HDFS implementation is implemented using two RPCs.
+ * It is understood that it is inefficient,
* but the implementation is thread-safe. The other option is to change the
* value of umask in configuration to be 0, but it is not thread-safe.
- *
- * @param fs file system handle
+ *
+ * @param fs FileSystem
* @param file the name of the file to be created
* @param permission the permission of the file
* @return an output stream
- * @throws IOException
+ * @throws IOException IO failure
*/
public static FSDataOutputStream create(FileSystem fs,
Path file, FsPermission permission) throws IOException {
@@ -604,20 +703,21 @@ public abstract class FileSystem extends Configured implements Closeable {
return out;
}
- /** create a directory with the provided permission
+ /**
+ * Create a directory with the provided permission.
* The permission of the directory is set to be the provided permission as in
* setPermission, not permission&~umask
- *
+ *
* @see #create(FileSystem, Path, FsPermission)
- *
- * @param fs file system handle
+ *
+ * @param fs FileSystem handle
* @param dir the name of the directory to be created
* @param permission the permission of the directory
* @return true if the directory creation succeeds; false otherwise
- * @throws IOException
+ * @throws IOException A problem creating the directories.
*/
public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
- throws IOException {
+ throws IOException {
// create the directory using the default permission
boolean result = fs.mkdirs(dir);
// set its permission to be the supplied one
@@ -633,9 +733,16 @@ public abstract class FileSystem extends Configured implements Closeable {
super(null);
}
- /**
+ /**
* Check that a Path belongs to this FileSystem.
+ *
+ * The base implementation performs case insensitive equality checks
+ * of the URIs' schemes and authorities. Subclasses may implement slightly
+ * different checks.
* @param path to check
+ * @throws IllegalArgumentException if the path is not considered to be
+ * part of this FileSystem.
+ *
*/
protected void checkPath(Path path) {
URI uri = path.toUri();
@@ -667,25 +774,37 @@ public abstract class FileSystem extends Configured implements Closeable {
return;
}
}
- throw new IllegalArgumentException("Wrong FS: "+path+
- ", expected: "+this.getUri());
+ throw new IllegalArgumentException("Wrong FS: " + path +
+ ", expected: " + this.getUri());
}
/**
- * Return an array containing hostnames, offset and size of
- * portions of the given file. For a nonexistent
- * file or regions, null will be returned.
+ * Return an array containing hostnames, offset and size of
+ * portions of the given file. For nonexistent
+ * file or regions, {@code null} is returned.
*
- * This call is most helpful with DFS, where it returns
- * hostnames of machines that contain the given file.
+ * <pre>
+ * if f == null :
+ * result = null
+ * elif f.getLen() <= start:
+ * result = []
+ * else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
+ * </pre>
+ * This call is most helpful with and distributed filesystem
+ * where the hostnames of machines that contain blocks of the given file
+ * can be determined.
*
- * The FileSystem will simply return an elt containing 'localhost'.
+ * The default implementation returns an array containing one element:
+ * <pre>
+ * BlockLocation( { "localhost:50010" }, { "localhost" }, 0, file.getLen())
+ * </pre>>
*
* @param file FilesStatus to get data from
* @param start offset into the given file
* @param len length for which to get locations for
+ * @throws IOException IO failure
*/
- public BlockLocation[] getFileBlockLocations(FileStatus file,
+ public BlockLocation[] getFileBlockLocations(FileStatus file,
long start, long len) throws IOException {
if (file == null) {
return null;
@@ -704,24 +823,28 @@ public abstract class FileSystem extends Configured implements Closeable {
return new BlockLocation[] {
new BlockLocation(name, host, 0, file.getLen()) };
}
-
/**
- * Return an array containing hostnames, offset and size of
- * portions of the given file. For a nonexistent
- * file or regions, null will be returned.
+ * Return an array containing hostnames, offset and size of
+ * portions of the given file. For a nonexistent
+ * file or regions, {@code null} is returned.
*
- * This call is most helpful with DFS, where it returns
- * hostnames of machines that contain the given file.
+ * This call is most helpful with location-aware distributed
+ * filesystems, where it returns hostnames of machines that
+ * contain the given file.
*
- * The FileSystem will simply return an elt containing 'localhost'.
+ * A FileSystem will normally return the equivalent result
+ * of passing the {@code FileStatus} of the path to
+ * {@link #getFileBlockLocations(FileStatus, long, long)}
*
* @param p path is used to identify an FS since an FS could have
* another FS that it could be delegating the call to
* @param start offset into the given file
* @param len length for which to get locations for
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException IO failure
*/
- public BlockLocation[] getFileBlockLocations(Path p,
+ public BlockLocation[] getFileBlockLocations(Path p,
long start, long len) throws IOException {
if (p == null) {
throw new NullPointerException();
@@ -729,46 +852,47 @@ public abstract class FileSystem extends Configured implements Closeable {
FileStatus file = getFileStatus(p);
return getFileBlockLocations(file, start, len);
}
-
+
/**
- * Return a set of server default configuration values
+ * Return a set of server default configuration values.
* @return server default configuration values
- * @throws IOException
+ * @throws IOException IO failure
* @deprecated use {@link #getServerDefaults(Path)} instead
*/
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
- Configuration conf = getConf();
- // CRC32 is chosen as default as it is available in all
+ Configuration config = getConf();
+ // CRC32 is chosen as default as it is available in all
// releases that support checksum.
// The client trash configuration is ignored.
- return new FsServerDefaults(getDefaultBlockSize(),
- conf.getInt("io.bytes.per.checksum", 512),
- 64 * 1024,
+ return new FsServerDefaults(getDefaultBlockSize(),
+ config.getInt("io.bytes.per.checksum", 512),
+ 64 * 1024,
getDefaultReplication(),
- conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT),
+ config.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT),
false,
FS_TRASH_INTERVAL_DEFAULT,
DataChecksum.Type.CRC32);
}
/**
- * Return a set of server default configuration values
+ * Return a set of server default configuration values.
* @param p path is used to identify an FS since an FS could have
* another FS that it could be delegating the call to
* @return server default configuration values
- * @throws IOException
+ * @throws IOException IO failure
*/
public FsServerDefaults getServerDefaults(Path p) throws IOException {
return getServerDefaults();
}
/**
- * Return the fully-qualified path of path f resolving the path
- * through any symlinks or mount point
+ * Return the fully-qualified path of path, resolving the path
+ * through any symlinks or mount point.
* @param p path to be resolved
- * @return fully qualified path
- * @throws FileNotFoundException
+ * @return fully qualified path
+ * @throws FileNotFoundException if the path is not present
+ * @throws IOException for any other error
*/
public Path resolvePath(final Path p) throws IOException {
checkPath(p);
@@ -779,13 +903,15 @@ public abstract class FileSystem extends Configured implements Closeable {
* Opens an FSDataInputStream at the indicated Path.
* @param f the file name to open
* @param bufferSize the size of the buffer to be used.
+ * @throws IOException IO failure
*/
public abstract FSDataInputStream open(Path f, int bufferSize)
throws IOException;
-
+
/**
* Opens an FSDataInputStream at the indicated Path.
* @param f the file to open
+ * @throws IOException IO failure
*/
public FSDataInputStream open(Path f) throws IOException {
return open(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@@ -796,6 +922,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create an FSDataOutputStream at the indicated Path.
* Files are overwritten by default.
* @param f the file to create
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f) throws IOException {
return create(f, true);
@@ -806,10 +933,11 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param f the file to create
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an exception will be thrown.
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f, boolean overwrite)
throws IOException {
- return create(f, overwrite,
+ return create(f, overwrite,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
getDefaultReplication(f),
@@ -822,10 +950,11 @@ public abstract class FileSystem extends Configured implements Closeable {
* Files are overwritten by default.
* @param f the file to create
* @param progress to report progress
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f, Progressable progress)
+ public FSDataOutputStream create(Path f, Progressable progress)
throws IOException {
- return create(f, true,
+ return create(f, true,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
getDefaultReplication(f),
@@ -837,10 +966,11 @@ public abstract class FileSystem extends Configured implements Closeable {
* Files are overwritten by default.
* @param f the file to create
* @param replication the replication factor
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f, short replication)
throws IOException {
- return create(f, true,
+ return create(f, true,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
replication,
@@ -854,65 +984,70 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param f the file to create
* @param replication the replication factor
* @param progress to report progress
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f, short replication,
+ public FSDataOutputStream create(Path f, short replication,
Progressable progress) throws IOException {
- return create(f, true,
+ return create(f, true,
getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT),
replication, getDefaultBlockSize(f), progress);
}
-
+
/**
* Create an FSDataOutputStream at the indicated Path.
- * @param f the file name to create
- * @param overwrite if a file with this name already exists, then if true,
+ * @param f the file to create
+ * @param overwrite if a path with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f,
+ public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize
) throws IOException {
- return create(f, overwrite, bufferSize,
+ return create(f, overwrite, bufferSize,
getDefaultReplication(f),
getDefaultBlockSize(f));
}
-
+
/**
- * Create an FSDataOutputStream at the indicated Path with write-progress
- * reporting.
+ * Create an {@link FSDataOutputStream} at the indicated Path
+ * with write-progress reporting.
+ *
+ * The frequency of callbacks is implementation-specific; it may be "none".
* @param f the path of the file to open
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f,
+ public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
Progressable progress
) throws IOException {
- return create(f, overwrite, bufferSize,
+ return create(f, overwrite, bufferSize,
getDefaultReplication(f),
getDefaultBlockSize(f), progress);
}
-
-
+
+
/**
* Create an FSDataOutputStream at the indicated Path.
* @param f the file name to open
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
- * @param replication required block replication for the file.
+ * @param replication required block replication for the file.
+ * @throws IOException IO failure
*/
- public FSDataOutputStream create(Path f,
- boolean overwrite,
- int bufferSize,
- short replication,
- long blockSize
- ) throws IOException {
+ public FSDataOutputStream create(Path f,
+ boolean overwrite,
+ int bufferSize,
+ short replication,
+ long blockSize) throws IOException {
return create(f, overwrite, bufferSize, replication, blockSize, null);
}
@@ -923,7 +1058,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
- * @param replication required block replication for the file.
+ * @param replication required block replication for the file.
+ * @throws IOException IO failure
*/
public FSDataOutputStream create(Path f,
boolean overwrite,
@@ -948,7 +1084,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public abstract FSDataOutputStream create(Path f,
@@ -958,7 +1094,7 @@ public abstract class FileSystem extends Configured implements Closeable {
short replication,
long blockSize,
Progressable progress) throws IOException;
-
+
/**
* Create an FSDataOutputStream at the indicated Path with write-progress
* reporting.
@@ -969,7 +1105,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream create(Path f,
@@ -982,10 +1118,10 @@ public abstract class FileSystem extends Configured implements Closeable {
return create(f, permission, flags, bufferSize, replication,
blockSize, progress, null);
}
-
+
/**
* Create an FSDataOutputStream at the indicated Path with a custom
- * checksum option
+ * checksum option.
* @param f the file name to open
* @param permission file permission
* @param flags {@link CreateFlag}s to use for this stream.
@@ -995,7 +1131,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param progress the progress reporter
* @param checksumOpt checksum parameter. If null, the values
* found in conf will be used.
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream create(Path f,
@@ -1009,27 +1145,31 @@ public abstract class FileSystem extends Configured implements Closeable {
// Checksum options are ignored by default. The file systems that
// implement checksum need to override this method. The full
// support is currently only available in DFS.
- return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
+ return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
bufferSize, replication, blockSize, progress);
}
- /*.
+ /**
* This create has been added to support the FileContext that processes
- * the permission
- * with umask before calling this method.
+ * the permission with umask before calling this method.
* This a temporary method added to support the transition from FileSystem
* to FileContext for user applications.
+ * @throws IOException IO failure
*/
@Deprecated
protected FSDataOutputStream primitiveCreate(Path f,
- FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
- short replication, long blockSize, Progressable progress,
- ChecksumOpt checksumOpt) throws IOException {
+ FsPermission absolutePermission,
+ EnumSet<CreateFlag> flag,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ ChecksumOpt checksumOpt) throws IOException {
boolean pathExists = exists(f);
CreateFlag.validate(f, pathExists, flag);
-
- // Default impl assumes that permissions do not matter and
+
+ // Default impl assumes that permissions do not matter and
// nor does the bytesPerChecksum hence
// calling the regular create is good enough.
// FSs that implement permissions should override this.
@@ -1037,25 +1177,27 @@ public abstract class FileSystem extends Configured implements Closeable {
if (pathExists && flag.contains(CreateFlag.APPEND)) {
return append(f, bufferSize, progress);
}
-
+
return this.create(f, absolutePermission,
flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
blockSize, progress);
}
-
+
/**
* This version of the mkdirs method assumes that the permission is absolute.
* It has been added to support the FileContext that processes the permission
* with umask before calling this method.
* This a temporary method added to support the transition from FileSystem
* to FileContext for user applications.
+ * @param f path
+ * @param absolutePermission permissions
+ * @return true if the directory was actually created.
+ * @throws IOException IO failure
+ * @see #mkdirs(Path, FsPermission)
*/
@Deprecated
protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
throws IOException {
- // Default impl is to assume that permissions do not matter and hence
- // calling the regular mkdirs is good enough.
- // FSs that implement permissions should override this.
return this.mkdirs(f, absolutePermission);
}
@@ -1068,10 +1210,10 @@ public abstract class FileSystem extends Configured implements Closeable {
* to FileContext for user applications.
*/
@Deprecated
- protected void primitiveMkdir(Path f, FsPermission absolutePermission,
+ protected void primitiveMkdir(Path f, FsPermission absolutePermission,
boolean createParent)
throws IOException {
-
+
if (!createParent) { // parent must exist.
// since the this.mkdirs makes parent dirs automatically
// we must throw exception if parent does not exist.
@@ -1103,7 +1245,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream createNonRecursive(Path f,
@@ -1126,7 +1268,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
@@ -1149,7 +1291,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param replication required block replication for the file.
* @param blockSize block size
* @param progress the progress reporter
- * @throws IOException
+ * @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
*/
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
@@ -1162,8 +1304,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Creates the given Path as a brand-new zero-length file. If
* create fails, or if it already existed, return false.
- *
+ * <i>Important: the default implementation is not atomic</i>
* @param f path to use for create
+ * @throws IOException IO failure
*/
public boolean createNewFile(Path f) throws IOException {
if (exists(f)) {
@@ -1177,21 +1320,27 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Append to an existing file (optional operation).
- * Same as append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
- * IO_FILE_BUFFER_SIZE_DEFAULT), null)
+ * Same as
+ * {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+ * IO_FILE_BUFFER_SIZE_DEFAULT), null)}
* @param f the existing file to be appended.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public FSDataOutputStream append(Path f) throws IOException {
return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT), null);
}
+
/**
* Append to an existing file (optional operation).
* Same as append(f, bufferSize, null).
* @param f the existing file to be appended.
* @param bufferSize the size of the buffer to be used.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
return append(f, bufferSize, null);
@@ -1202,7 +1351,9 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param f the existing file to be appended.
* @param bufferSize the size of the buffer to be used.
* @param progress for reporting progress if it is not null.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public abstract FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException;
@@ -1211,34 +1362,40 @@ public abstract class FileSystem extends Configured implements Closeable {
* Concat existing files together.
* @param trg the path to the target destination.
* @param psrcs the paths to the sources to use for the concatenation.
- * @throws IOException
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public void concat(final Path trg, final Path [] psrcs) throws IOException {
- throw new UnsupportedOperationException("Not implemented by the " +
+ throw new UnsupportedOperationException("Not implemented by the " +
getClass().getSimpleName() + " FileSystem implementation");
}
/**
- * Get replication.
- *
- * @deprecated Use getFileStatus() instead
+ * Get the replication factor.
+ *
+ * @deprecated Use {@link #getFileStatus(Path)} instead
* @param src file name
* @return file replication
- * @throws IOException
- */
+ * @throws FileNotFoundException if the path does not resolve.
+ * @throws IOException an IO failure
+ */
@Deprecated
public short getReplication(Path src) throws IOException {
return getFileStatus(src).getReplication();
}
/**
- * Set replication for an existing file.
- *
+ * Set the replication for an existing file.
+ * If a filesystem does not support replication, it will always
+ * return true: the check for a file existing may be bypassed.
+ * This is the default behavior.
* @param src file name
* @param replication new replication
* @throws IOException
- * @return true if successful;
- * false if file does not exist or is a directory
+ * @return true if successful, or the feature in unsupported;
+ * false if replication is supported but the file does not exist,
+ * or is a directory
*/
public boolean setReplication(Path src, short replication)
throws IOException {
@@ -1246,8 +1403,7 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Renames Path src to Path dst. Can take place on local fs
- * or remote DFS.
+ * Renames Path src to Path dst.
* @param src path to be renamed
* @param dst new path after rename
* @throws IOException on failure
@@ -1258,9 +1414,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Renames Path src to Path dst
* <ul>
- * <li>Fails if src is a file and dst is a directory.
- * <li>Fails if src is a directory and dst is a file.
- * <li>Fails if the parent of dst does not exist or is a file.
+ * <li>Fails if src is a file and dst is a directory.</li>
+ * <li>Fails if src is a directory and dst is a file.</li>
+ * <li>Fails if the parent of dst does not exist or is a file.</li>
* </ul>
* <p>
* If OVERWRITE option is not passed as an argument, rename fails
@@ -1274,12 +1430,17 @@ public abstract class FileSystem extends Configured implements Closeable {
* implementation. Please refer to the file system documentation for
* details. This default implementation is non atomic.
* <p>
- * This method is deprecated since it is a temporary method added to
- * support the transition from FileSystem to FileContext for user
+ * This method is deprecated since it is a temporary method added to
+ * support the transition from FileSystem to FileContext for user
* applications.
- *
+ *
* @param src path to be renamed
* @param dst new path after rename
+ * @throws FileNotFoundException src path does not exist, or the parent
+ * path of dst does not exist.
+ * @throws FileAlreadyExistsException dest path exists and is a file
+ * @throws ParentNotDirectoryException if the parent path of dest is not
+ * a directory
* @throws IOException on failure
*/
@Deprecated
@@ -1344,10 +1505,10 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Truncate the file in the indicated path to the indicated size.
* <ul>
- * <li>Fails if path is a directory.
- * <li>Fails if path does not exist.
- * <li>Fails if path is not closed.
- * <li>Fails if new size is greater than current size.
+ * <li>Fails if path is a directory.</li>
+ * <li>Fails if path does not exist.</li>
+ * <li>Fails if path is not closed.</li>
+ * <li>Fails if new size is greater than current size.</li>
* </ul>
* @param f The path to the file to be truncated
* @param newLength The size the file is to be truncated to
@@ -1358,44 +1519,61 @@ public abstract class FileSystem extends Configured implements Closeable {
* <code>false</code> if a background process of adjusting the length of
* the last block has been started, and clients should wait for it to
* complete before proceeding with further file updates.
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
*/
public boolean truncate(Path f, long newLength) throws IOException {
throw new UnsupportedOperationException("Not implemented by the " +
getClass().getSimpleName() + " FileSystem implementation");
}
-
+
/**
- * Delete a file
+ * Delete a file/directory.
* @deprecated Use {@link #delete(Path, boolean)} instead.
*/
@Deprecated
public boolean delete(Path f) throws IOException {
return delete(f, true);
}
-
+
/** Delete a file.
*
* @param f the path to delete.
- * @param recursive if path is a directory and set to
+ * @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
- * case of a file the recursive can be set to either true or false.
- * @return true if delete is successful else false.
- * @throws IOException
+ * case of a file the recursive can be set to either true or false.
+ * @return true if delete is successful else false.
+ * @throws IOException IO failure
*/
public abstract boolean delete(Path f, boolean recursive) throws IOException;
/**
- * Mark a path to be deleted when FileSystem is closed.
- * When the JVM shuts down,
- * all FileSystem objects will be closed automatically.
- * Then,
- * the marked path will be deleted as a result of closing the FileSystem.
+ * Mark a path to be deleted when its FileSystem is closed.
+ * When the JVM shuts down cleanly, all cached FileSystem objects will be
+ * closed automatically \u2014these the marked paths will be deleted as a result.
+ *
+ * If a FileSystem instance is not cached, i.e. has been created with
+ * {@link #createFileSystem(URI, Configuration)}, then the paths will
+ * be deleted in when {@link #close()} is called on that instance.
*
- * The path has to exist in the file system.
- *
+ * The path must exist in the filesystem at the time of the method call;
+ * it does not have to exist at the time of JVM shutdown.
+ *
+ * Notes
+ * <ol>
+ * <li>Clean shutdown of the JVM cannot be guaranteed.</li>
+ * <li>The time to shut down a FileSystem will depends on the number of
+ * files to delete. For filesystems where the cost of checking
+ * for the existence of a file/directory and the actual delete operation
+ * (for example: object stores) is high, the time to shutdown the JVM can be
+ * significantly extended by over-use of this feature.</li>
+ * <li>Connectivity problems with a remote filesystem may delay shutdown
+ * further, and may cause the files to not be deleted.</li>
+ * </ol>
* @param f the path to delete.
* @return true if deleteOnExit is successful, otherwise false.
- * @throws IOException
+ * @throws IOException IO failure
*/
public boolean deleteOnExit(Path f) throws IOException {
if (!exists(f)) {
@@ -1406,10 +1584,11 @@ public abstract class FileSystem extends Configured implements Closeable {
}
return true;
}
-
+
/**
- * Cancel the deletion of the path when the FileSystem is closed
+ * Cancel the scheduled deletion of the path when the FileSystem is closed.
* @param f the path to cancel deletion
+ * @return true if the path was found in the delete-on-exit list.
*/
public boolean cancelDeleteOnExit(Path f) {
synchronized (deleteOnExit) {
@@ -1418,8 +1597,12 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Delete all files that were marked as delete-on-exit. This recursively
- * deletes all files in the specified paths.
+ * Delete all paths that were marked as delete-on-exit. This recursively
+ * deletes all files and directories in the specified paths.
+ *
+ * The time to process this operation is {@code O(paths)}, with the actual
+ * time dependent on the time for existence and deletion operations to
+ * complete, successfully or not.
*/
protected void processDeleteOnExit() {
synchronized (deleteOnExit) {
@@ -1431,15 +1614,17 @@ public abstract class FileSystem extends Configured implements Closeable {
}
}
catch (IOException e) {
- LOG.info("Ignoring failure to deleteOnExit for path " + path);
+ LOGGER.info("Ignoring failure to deleteOnExit for path {}", path);
}
iter.remove();
}
}
}
-
- /** Check if exists.
- * @param f source file
+
+ /** Check if a path exists.
+ * @param f source path
+ * @return true if the path exists
+ * @throws IOException IO failure
*/
public boolean exists(Path f) throws IOException {
try {
@@ -1450,9 +1635,10 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/** True iff the named path is a directory.
- * Note: Avoid using this method. Instead reuse the FileStatus
+ * Note: Avoid using this method. Instead reuse the FileStatus
* returned by getFileStatus() or listStatus() methods.
* @param f path to check
+ * @throws IOException IO failure
*/
public boolean isDirectory(Path f) throws IOException {
try {
@@ -1463,9 +1649,10 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/** True iff the named path is a regular file.
- * Note: Avoid using this method. Instead reuse the FileStatus
- * returned by getFileStatus() or listStatus() methods.
+ * Note: Avoid using this method. Instead reuse the FileStatus
+ * returned by {@link #getFileStatus(Path)} or listStatus() methods.
* @param f path to check
+ * @throws IOException IO failure
*/
public boolean isFile(Path f) throws IOException {
try {
@@ -1474,17 +1661,24 @@ public abstract class FileSystem extends Configured implements Closeable {
return false; // f does not exist
}
}
-
- /** The number of bytes in a file. */
- /** @deprecated Use getFileStatus() instead */
+
+ /**
+ * The number of bytes in a file.
+ * @return the number of bytes; 0 for a directory
+ * @deprecated Use {@link #getFileStatus(Path)} instead.
+ * @throws FileNotFoundException if the path does not resolve
+ * @throws IOException IO failure
+ */
@Deprecated
public long getLength(Path f) throws IOException {
return getFileStatus(f).getLen();
}
-
+
/** Return the {@link ContentSummary} of a given {@link Path}.
- * @param f path to use
- */
+ * @param f path to use
+ * @throws FileNotFoundException if the path does not resolve
+ * @throws IOException IO failure
+ */
public ContentSummary getContentSummary(Path f) throws IOException {
FileStatus status = getFileStatus(f);
if (status.isFile()) {
@@ -1511,18 +1705,23 @@ public abstract class FileSystem extends Configured implements Closeable {
/** Return the {@link QuotaUsage} of a given {@link Path}.
* @param f path to use
+ * @return the quota usage
+ * @throws IOException IO failure
*/
public QuotaUsage getQuotaUsage(Path f) throws IOException {
return getContentSummary(f);
}
- final private static PathFilter DEFAULT_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path file) {
- return true;
- }
- };
-
+ /**
+ * The default filter accepts all paths.
+ */
+ private static final PathFilter DEFAULT_FILTER = new PathFilter() {
+ @Override
+ public boolean accept(Path file) {
+ return true;
+ }
+ };
+
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
@@ -1536,7 +1735,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* @throws FileNotFoundException when the path does not exist
* @throws IOException see specific implementation
*/
- public abstract FileStatus[] listStatus(Path f) throws IOException;
+ public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException,
+ IOException;
/**
* Represents a batch of directory entries when iteratively listing a
@@ -1599,9 +1799,11 @@ public abstract class FileSystem extends Configured implements Closeable {
return new DirectoryEntries(listing, null, false);
}
- /*
+ /**
* Filter files/directories in the given path using the user-supplied path
* filter. Results are added to the given array <code>results</code>.
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
private void listStatus(ArrayList<FileStatus> results, Path f,
PathFilter filter) throws FileNotFoundException, IOException {
@@ -1615,15 +1817,17 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
+ * List corrupted file blocks.
* @return an iterator over the corrupt files under the given path
* (may contain duplicates if a file has more than one corrupt block)
- * @throws IOException
+ * @throws UnsupportedOperationException if the operation is unsupported
+ * (default).
+ * @throws IOException IO failure
*/
public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
throw new UnsupportedOperationException(getClass().getCanonicalName() +
- " does not support" +
- " listCorruptFileBlocks");
+ " does not support listCorruptFileBlocks");
}
/**
@@ -1632,19 +1836,19 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* Does not guarantee to return the List of files/directories status in a
* sorted order.
- *
+ *
* @param f
* a path name
* @param filter
* the user-supplied path filter
* @return an array of FileStatus objects for the files under the given path
* after applying the filter
- * @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
- public FileStatus[] listStatus(Path f, PathFilter filter)
+ public FileStatus[] listStatus(Path f, PathFilter filter)
throws FileNotFoundException, IOException {
- ArrayList<FileStatus> results = new ArrayList<FileStatus>();
+ ArrayList<FileStatus> results = new ArrayList<>();
listStatus(results, f, filter);
return results.toArray(new FileStatus[results.size()]);
}
@@ -1655,13 +1859,13 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* Does not guarantee to return the List of files/directories status in a
* sorted order.
- *
+ *
* @param files
* a list of paths
* @return a list of statuses for the files under the given paths after
* applying the filter default Path filter
- * @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
public FileStatus[] listStatus(Path[] files)
throws FileNotFoundException, IOException {
@@ -1674,15 +1878,15 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* Does not guarantee to return the List of files/directories status in a
* sorted order.
- *
+ *
* @param files
* a list of paths
* @param filter
* the user-supplied path filter
* @return a list of statuses for the files under the given paths after
* applying the filter
- * @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException see specific implementation
*/
public FileStatus[] listStatus(Path[] files, PathFilter filter)
throws FileNotFoundException, IOException {
@@ -1696,7 +1900,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* <p>Return all the files that match filePattern and are not checksum
* files. Results are sorted by their names.
- *
+ *
* <p>
* A filename pattern is composed of <i>regular</i> characters and
* <i>special pattern matching</i> characters, which are:
@@ -1736,7 +1940,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* <p>
* <dt> <tt> {ab,cd} </tt>
* <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
- *
+ *
* <p>
* <dt> <tt> {ab,c{de,fh}} </tt>
* <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
@@ -1748,17 +1952,17 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param pathPattern a regular expression specifying a pth pattern
* @return an array of paths that match the path pattern
- * @throws IOException
+ * @throws IOException IO failure
*/
public FileStatus[] globStatus(Path pathPattern) throws IOException {
return new Globber(this, pathPattern, DEFAULT_FILTER).glob();
}
-
+
/**
- * Return an array of FileStatus objects whose path names match
+ * Return an array of {@link FileStatus} objects whose path names match
* {@code pathPattern} and is accepted by the user-supplied path filter.
* Results are sorted by their path names.
- *
+ *
* @param pathPattern a regular expression specifying the path pattern
* @param filter a user-supplied path filter
* @return null if {@code pathPattern} has no glob and the path does not exist
@@ -1771,17 +1975,17 @@ public abstract class FileSystem extends Configured implements Closeable {
throws IOException {
return new Globber(this, pathPattern, filter).glob();
}
-
+
/**
* List the statuses of the files/directories in the given path if the path is
- * a directory.
+ * a directory.
* Return the file's status and block locations If the path is a file.
- *
+ *
* If a returned status is a file, it contains the file's block locations.
- *
+ *
* @param f is the path
*
- * @return an iterator that traverses statuses of the files/directories
+ * @return an iterator that traverses statuses of the files/directories
* in the given path
*
* @throws FileNotFoundException If <code>f</code> does not exist
@@ -1793,12 +1997,12 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Listing a directory
+ * List a directory.
* The returned results include its block location if it is a file
* The results are filtered by the given path filter
* @param f a path
* @param filter a path filter
- * @return an iterator that traverses statuses of the files/directories
+ * @return an iterator that traverses statuses of the files/directories
* in the given path
* @throws FileNotFoundException if <code>f</code> does not exist
* @throws IOException if any I/O error occurred
@@ -1876,14 +2080,17 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Returns a remote iterator so that followup calls are made on demand
- * while consuming the entries. Each file system implementation should
+ * while consuming the entries. Each FileSystem implementation should
* override this method and provide a more efficient implementation, if
- * possible.
+ * possible.
+ *
* Does not guarantee to return the iterator that traverses statuses
* of the files in a sorted order.
*
* @param p target path
* @return remote iterator
+ * @throws FileNotFoundException if <code>p</code> does not exist
+ * @throws IOException if any I/O error occurred
*/
public RemoteIterator<FileStatus> listStatusIterator(final Path p)
throws FileNotFoundException, IOException {
@@ -1894,30 +2101,29 @@ public abstract class FileSystem extends Configured implements Closeable {
* List the statuses and block locations of the files in the given path.
* Does not guarantee to return the iterator that traverses statuses
* of the files in a sorted order.
- *
- * If the path is a directory,
+ * <pre>
+ * If the path is a directory,
* if recursive is false, returns files in the directory;
* if recursive is true, return files in the subtree rooted at the path.
* If the path is a file, return the file's status and block locations.
- *
+ * </pre>
* @param f is the path
* @param recursive if the subdirectories need to be traversed recursively
*
* @return an iterator that traverses statuses of the files
*
* @throws FileNotFoundException when the path does not exist;
- * IOException see specific implementation
+ * @throws IOException see specific implementation
*/
public RemoteIterator<LocatedFileStatus> listFiles(
final Path f, final boolean recursive)
throws FileNotFoundException, IOException {
return new RemoteIterator<LocatedFileStatus>() {
- private Stack<RemoteIterator<LocatedFileStatus>> itors =
- new Stack<RemoteIterator<LocatedFileStatus>>();
+ private Stack<RemoteIterator<LocatedFileStatus>> itors = new Stack<>();
private RemoteIterator<LocatedFileStatus> curItor =
listLocatedStatus(f);
private LocatedFileStatus curFile;
-
+
@Override
public boolean hasNext() throws IOException {
while (curFile == null) {
@@ -1955,14 +2161,14 @@ public abstract class FileSystem extends Configured implements Closeable {
LocatedFileStatus result = curFile;
curFile = null;
return result;
- }
+ }
throw new java.util.NoSuchElementException("No more entry in " + f);
}
};
}
-
- /** Return the current user's home directory in this filesystem.
- * The default implementation returns "/user/$USER/".
+
+ /** Return the current user's home directory in this FileSystem.
+ * The default implementation returns {@code "/user/$USER/"}.
*/
public Path getHomeDirectory() {
return this.makeQualified(
@@ -1971,29 +2177,28 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
- * Set the current working directory for the given file system. All relative
+ * Set the current working directory for the given FileSystem. All relative
* paths will be resolved relative to it.
- *
+ *
* @param new_dir Path of new working directory
*/
public abstract void setWorkingDirectory(Path new_dir);
-
+
/**
- * Get the current working directory for the given file system
+ * Get the current working directory for the given FileSystem
* @return the directory pathname
*/
public abstract Path getWorkingDirectory();
-
-
- /**
- * Note: with the new FilesContext class, getWorkingDirectory()
- * will be removed.
- * The working directory is implemented in FilesContext.
- *
- * Some file systems like LocalFileSystem have an initial workingDir
+
+ /**
+ * Note: with the new FileContext class, getWorkingDirectory()
+ * will be removed.
+ * The working directory is implemented in FileContext.
+ *
+ * Some FileSystems like LocalFileSystem have an initial workingDir
* that we use as the starting workingDir. For other file systems
* like HDFS there is no built in notion of an initial workingDir.
- *
+ *
* @return if there is built in notion of workingDir then it
* is returned; else a null is returned.
*/
@@ -2003,6 +2208,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Call {@link #mkdirs(Path, FsPermission)} with default permission.
+ * @param f path
+ * @return true if the directory was created
+ * @throws IOException IO failure
*/
public boolean mkdirs(Path f) throws IOException {
return mkdirs(f, FsPermission.getDirDefault());
@@ -2010,19 +2218,21 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Make the given file and all non-existent parents into
- * directories. Has the semantics of Unix 'mkdir -p'.
+ * directories. Has roughly the semantics of Unix @{code mkdir -p}.
* Existence of the directory hierarchy is not an error.
* @param f path to create
* @param permission to apply to f
+ * @throws IOException IO failure
*/
public abstract boolean mkdirs(Path f, FsPermission permission
) throws IOException;
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to filesystem at
* the given dst name and the source is kept intact afterwards
* @param src path
* @param dst path
+ * @throws IOException IO failure
*/
public void copyFromLocalFile(Path src, Path dst)
throws IOException {
@@ -2030,10 +2240,11 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * The src files is on the local disk. Add it to FS at
+ * The src files is on the local disk. Add it to filesystem at
* the given dst name, removing the source afterwards.
- * @param srcs path
+ * @param srcs source paths
* @param dst path
+ * @throws IOException IO failure
*/
public void moveFromLocalFile(Path[] srcs, Path dst)
throws IOException {
@@ -2041,10 +2252,11 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to the filesystem at
* the given dst name, removing the source afterwards.
- * @param src path
+ * @param src local path
* @param dst path
+ * @throws IOException IO failure
*/
public void moveFromLocalFile(Path src, Path dst)
throws IOException {
@@ -2052,7 +2264,7 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to the filesystem at
* the given dst name.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
@@ -2063,80 +2275,83 @@ public abstract class FileSystem extends Configured implements Closeable {
throws IOException {
copyFromLocalFile(delSrc, true, src, dst);
}
-
+
/**
- * The src files are on the local disk. Add it to FS at
+ * The src files are on the local disk. Add it to the filesystem at
* the given dst name.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param srcs array of paths which are source
* @param dst path
+ * @throws IOException IO failure
*/
- public void copyFromLocalFile(boolean delSrc, boolean overwrite,
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite,
Path[] srcs, Path dst)
throws IOException {
Configuration conf = getConf();
FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
}
-
+
/**
- * The src file is on the local disk. Add it to FS at
+ * The src file is on the local disk. Add it to the filesystem at
* the given dst name.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param src path
* @param dst path
+ * @throws IOException IO failure
*/
- public void copyFromLocalFile(boolean delSrc, boolean overwrite,
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite,
Path src, Path dst)
throws IOException {
Configuration conf = getConf();
FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
}
-
+
/**
- * The src file is under FS, and the dst is on the local disk.
- * Copy it from FS control to the local dst name.
- * @param src path
- * @param dst path
+ * Copy it a file from the remote filesystem to the local one.
+ * @param src path src file in the remote filesystem
+ * @param dst path local destination
+ * @throws IOException IO failure
*/
public void copyToLocalFile(Path src, Path dst) throws IOException {
copyToLocalFile(false, src, dst);
}
-
+
/**
- * The src file is under FS, and the dst is on the local disk.
- * Copy it from FS control to the local dst name.
- * Remove the source afterwards
- * @param src path
- * @param dst path
+ * Copy a file to the local filesystem, then delete it from the
+ * remote filesystem (if successfully copied).
+ * @param src path src file in the remote filesystem
+ * @param dst path local destination
+ * @throws IOException IO failure
*/
public void moveToLocalFile(Path src, Path dst) throws IOException {
copyToLocalFile(true, src, dst);
}
/**
- * The src file is under FS, and the dst is on the local disk.
- * Copy it from FS control to the local dst name.
+ * Copy it a file from a remote filesystem to the local one.
* delSrc indicates if the src will be removed or not.
* @param delSrc whether to delete the src
- * @param src path
- * @param dst path
- */
+ * @param src path src file in the remote filesystem
+ * @param dst path local destination
+ * @throws IOException IO failure
+ */
public void copyToLocalFile(boolean delSrc, Path src, Path dst)
throws IOException {
copyToLocalFile(delSrc, src, dst, false);
}
-
- /**
- * The src file is under FS, and the dst is on the local disk. Copy it from FS
- * control to the local dst name. delSrc indicates if the src will be removed
+
+ /**
+ * The src file is under this filesystem, and the dst is on the local disk.
+ * Copy it from the remote filesystem to the local dst name.
+ * delSrc indicates if the src will be removed
* or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
- * as local file system or not. RawLocalFileSystem is non crc file system.So,
- * It will not create any crc files at local.
- *
+ * as the local file system or not. RawLocalFileSystem is non checksumming,
+ * So, It will not create any crc files at local.
+ *
* @param delSrc
* whether to delete the src
* @param src
@@ -2145,9 +2360,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* path
* @param useRawLocalFileSystem
* whether to use RawLocalFileSystem as local file system or not.
- *
- * @throws IOException
- * - if any IO error
+ *
+ * @throws IOException for any IO error
*/
public void copyToLocalFile(boolean delSrc, Path src, Path dst,
boolean useRawLocalFileSystem) throws IOException {
@@ -2162,12 +2376,14 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Returns a local File that the user can write output to. The caller
- * provides both the eventual FS target name and the local working
- * file. If the FS is local, we write directly into the target. If
- * the FS is remote, we write into the tmp local area.
+ * Returns a local file that the user can write output to. The caller
+ * provides both the eventual target name in this FileSystem
+ * and the local working file path.
+ * If this FileSystem is local, we write directly into the target. If
+ * the FileSystem is not local, we write into the tmp local area.
* @param fsOutputFile path of output file
* @param tmpLocalFile path of local tmp file
+ * @throws IOException IO failure
*/
public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
throws IOException {
@@ -2175,12 +2391,14 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * Called when we're all done writing to the target. A local FS will
- * do nothing, because we've written to exactly the right place. A remote
- * FS will copy the contents of tmpLocalFile to the correct target at
+ * Called when we're all done writing to the target.
+ * A local FS will do nothing, because we've written to exactly the
+ * right place.
+ * A remote FS will copy the contents of tmpLocalFile to the correct target at
* fsOutputFile.
* @param fsOutputFile path of output file
* @param tmpLocalFile path to local tmp file
+ * @throws IOException IO failure
*/
public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
throws IOException {
@@ -2188,8 +2406,14 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
- * No more filesystem operations are needed. Will
- * release any held locks.
+ * Close this FileSystem instance.
+ * Will release any held locks, delete all files queued for deletion
+ * through calls to {@link #deleteOnExit(Path)}, and remove this FS instance
+ * from the cache, if cached.
+ *
+ * After this operation, the outcome of any method call on this FileSystem
+ * instance, or any input/output stream created by it is <i>undefined</i>.
+ * @throws IOException IO failure
*/
@Override
public void close() throws IOException {
@@ -2198,13 +2422,19 @@ public abstract class FileSystem extends Configured implements Closeable {
CACHE.remove(this.key, this);
}
- /** Return the total size of all files in the filesystem. */
+ /**
+ * Return the total size of all files in the filesystem.
+ * @throws IOException IO failure
+ */
public long getUsed() throws IOException {
Path path = new Path("/");
return getUsed(path);
}
- /** Return the total size of all files from a specified path. */
+ /**
+ * Return the total size of all files from a specified path.
+ * @throws IOException IO failure
+ */
public long getUsed(Path path) throws IOException {
return getContentSummary(path).getLength();
}
@@ -2213,8 +2443,10 @@ public abstract class FileSystem extends Configured implements Closeable {
* Get the block size for a particular file.
* @param f the filename
* @return the number of bytes in a block
+ * @deprecated Use {@link #getFileStatus(Path)} instead
+ * @throws FileNotFoundException if the path is not present
+ * @throws IOException IO failure
*/
- /** @deprecated Use getFileStatus() instead */
@Deprecated
public long getBlockSize(Path f) throws IOException {
return getFileStatus(f).getBlockSize();
@@ -2222,7 +2454,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Return the number of bytes that large input files should be optimally
- * be split into to minimize i/o time.
+ * be split into to minimize I/O time.
* @deprecated use {@link #getDefaultBlockSize(Path)} instead
*/
@Deprecated
@@ -2230,9 +2462,10 @@ public abstract class FileSystem extends Configured implements Closeable {
// default to 32MB: large enough to minimize the impact of seeks
return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
}
-
- /** Return the number of bytes that large input files should be optimally
- * be split into to minimize i/o time. The given path will be used to
+
+ /**
+ * Return the number of bytes that large input files should be optimally
+ * be split into to minimize I/O time. The given path will be used to
* locate the actual filesystem. The full path does not have to exist.
* @param f path of
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: HADOOP-13605. Clean up FileSystem javadocs,
logging; improve diagnostics on FS load. Contributed by Steve Loughran
Posted by li...@apache.org.
HADOOP-13605. Clean up FileSystem javadocs, logging; improve diagnostics on FS load. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/860d49aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/860d49aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/860d49aa
Branch: refs/heads/trunk
Commit: 860d49aa6ae7790d970d7f2322fed890b0e5cda2
Parents: de48949
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Nov 23 16:32:42 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Nov 23 16:42:27 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 1449 +++++++++++-------
.../src/site/markdown/filesystem/filesystem.md | 2 +-
.../org/apache/hadoop/fs/TestDefaultUri.java | 40 +-
.../apache/hadoop/fs/TestFileSystemCaching.java | 8 +-
4 files changed, 920 insertions(+), 579 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org