You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2013/08/12 23:26:02 UTC
svn commit: r1513258 [3/9] - in
/hadoop/common/branches/YARN-321/hadoop-common-project: ./
hadoop-annotations/ hadoop-auth-examples/
hadoop-auth-examples/src/main/webapp/
hadoop-auth-examples/src/main/webapp/annonymous/
hadoop-auth-examples/src/main/we...
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1505610,1507165,1507259,1509070
Merged /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java:r1503799-1513205
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Mon Aug 12 21:25:49 2013
@@ -1084,7 +1084,7 @@ public class Configuration implements It
*/
public boolean getBoolean(String name, boolean defaultValue) {
String valueString = getTrimmed(name);
- if (null == valueString || "".equals(valueString)) {
+ if (null == valueString || valueString.isEmpty()) {
return defaultValue;
}
@@ -1238,7 +1238,7 @@ public class Configuration implements It
*/
public Pattern getPattern(String name, Pattern defaultValue) {
String valString = get(name);
- if (null == valString || "".equals(valString)) {
+ if (null == valString || valString.isEmpty()) {
return defaultValue;
}
try {
@@ -2173,12 +2173,12 @@ public class Configuration implements It
/**
* Write out the non-default properties in this configuration to the given
- * {@link OutputStream}.
+ * {@link OutputStream} using UTF-8 encoding.
*
* @param out the output stream to write to.
*/
public void writeXml(OutputStream out) throws IOException {
- writeXml(new OutputStreamWriter(out));
+ writeXml(new OutputStreamWriter(out, "UTF-8"));
}
/**
@@ -2220,7 +2220,7 @@ public class Configuration implements It
doc.appendChild(conf);
conf.appendChild(doc.createTextNode("\n"));
handleDeprecation(); //ensure properties is set and deprecation is handled
- for (Enumeration e = properties.keys(); e.hasMoreElements();) {
+ for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
String name = (String)e.nextElement();
Object object = properties.get(name);
String value = null;
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java Mon Aug 12 21:25:49 2013
@@ -153,9 +153,9 @@ public class ReconfigurationServlet exte
StringEscapeUtils.unescapeHtml(req.getParameter(rawParam));
if (value != null) {
if (value.equals(newConf.getRaw(param)) || value.equals("default") ||
- value.equals("null") || value.equals("")) {
+ value.equals("null") || value.isEmpty()) {
if ((value.equals("default") || value.equals("null") ||
- value.equals("")) &&
+ value.isEmpty()) &&
oldConf.getRaw(param) != null) {
out.println("<p>Changed \"" +
StringEscapeUtils.escapeHtml(param) + "\" from \"" +
@@ -163,7 +163,7 @@ public class ReconfigurationServlet exte
"\" to default</p>");
reconf.reconfigureProperty(param, null);
} else if (!value.equals("default") && !value.equals("null") &&
- !value.equals("") &&
+ !value.isEmpty() &&
(oldConf.getRaw(param) == null ||
!oldConf.getRaw(param).equals(value))) {
// change from default or value to different value
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Mon Aug 12 21:25:49 2013
@@ -96,7 +96,7 @@ public class CommonConfigurationKeys ext
public static final int IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT =
256 * 1024;
- /** Internal buffer size for Snappy compressor/decompressors */
+ /** Internal buffer size for Lz4 compressor/decompressors */
public static final String IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY =
"io.compression.codec.lz4.buffersize";
@@ -104,6 +104,14 @@ public class CommonConfigurationKeys ext
public static final int IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT =
256 * 1024;
+ /** Use lz4hc(slow but with high compression ratio) for lz4 compression */
+ public static final String IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY =
+ "io.compression.codec.lz4.use.lz4hc";
+
+ /** Default value for IO_COMPRESSION_CODEC_USELZ4HC_KEY */
+ public static final boolean IO_COMPRESSION_CODEC_LZ4_USELZ4HC_DEFAULT =
+ false;
+
/**
* Service Authorization
*/
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Mon Aug 12 21:25:49 2013
@@ -58,6 +58,11 @@ public class CommonConfigurationKeysPubl
public static final String FS_DU_INTERVAL_KEY = "fs.du.interval";
/** Default value for FS_DU_INTERVAL_KEY */
public static final long FS_DU_INTERVAL_DEFAULT = 600000;
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY =
+ "fs.client.resolve.remote.symlinks";
+ /** Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY */
+ public static final boolean FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT = true;
//Defaults are not specified for following keys
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java Mon Aug 12 21:25:49 2013
@@ -28,7 +28,8 @@ import org.apache.hadoop.classification.
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FSDataInputStream extends DataInputStream
- implements Seekable, PositionedReadable, Closeable, ByteBufferReadable, HasFileDescriptor {
+ implements Seekable, PositionedReadable, Closeable,
+ ByteBufferReadable, HasFileDescriptor, CanSetDropBehind, CanSetReadahead {
public FSDataInputStream(InputStream in)
throws IOException {
@@ -143,4 +144,27 @@ public class FSDataInputStream extends D
return null;
}
}
+
+ @Override
+ public void setReadahead(Long readahead)
+ throws IOException, UnsupportedOperationException {
+ try {
+ ((CanSetReadahead)in).setReadahead(readahead);
+ } catch (ClassCastException e) {
+ throw new UnsupportedOperationException(
+ "this stream does not support setting the readahead " +
+ "caching strategy.");
+ }
+ }
+
+ @Override
+ public void setDropBehind(Boolean dropBehind)
+ throws IOException, UnsupportedOperationException {
+ try {
+ ((CanSetDropBehind)in).setDropBehind(dropBehind);
+ } catch (ClassCastException e) {
+ throw new UnsupportedOperationException("this stream does not " +
+ "support setting the drop-behind caching setting.");
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java Mon Aug 12 21:25:49 2013
@@ -18,6 +18,10 @@
package org.apache.hadoop.fs;
import java.io.*;
+import java.io.DataOutputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -26,8 +30,9 @@ import org.apache.hadoop.classification.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class FSDataOutputStream extends DataOutputStream implements Syncable {
- private OutputStream wrappedStream;
+public class FSDataOutputStream extends DataOutputStream
+ implements Syncable, CanSetDropBehind {
+ private final OutputStream wrappedStream;
private static class PositionCache extends FilterOutputStream {
private FileSystem.Statistics statistics;
@@ -133,4 +138,14 @@ public class FSDataOutputStream extends
wrappedStream.flush();
}
}
+
+ @Override
+ public void setDropBehind(Boolean dropBehind) throws IOException {
+ try {
+ ((CanSetDropBehind)wrappedStream).setDropBehind(dropBehind);
+ } catch (ClassCastException e) {
+ throw new UnsupportedOperationException("the wrapped stream does " +
+ "not support setting the drop-behind caching setting.");
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java Mon Aug 12 21:25:49 2013
@@ -90,6 +90,11 @@ public abstract class FSLinkResolver<T>
in = next(fs, p);
isLink = false;
} catch (UnresolvedLinkException e) {
+ if (!fc.resolveSymlinks) {
+ throw new IOException("Path " + path + " contains a symlink"
+ + " and symlink resolution is disabled ("
+ + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + ").", e);
+ }
if (count++ > FsConstants.MAX_PATH_LINKS) {
throw new IOException("Possible cyclic loop while " +
"following symbolic link " + path);
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Mon Aug 12 21:25:49 2013
@@ -225,6 +225,7 @@ public final class FileContext {
private FsPermission umask;
private final Configuration conf;
private final UserGroupInformation ugi;
+ final boolean resolveSymlinks;
private FileContext(final AbstractFileSystem defFs,
final FsPermission theUmask, final Configuration aConf) {
@@ -250,9 +251,12 @@ public final class FileContext {
if (workingDir == null) {
workingDir = defaultFS.getHomeDirectory();
}
+ resolveSymlinks = conf.getBoolean(
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
util = new Util(); // for the inner class
}
-
+
/*
* Remove relative part - return "absolute":
* If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
@@ -264,7 +268,7 @@ public final class FileContext {
* Hence this method is not called makeAbsolute() and
* has been deliberately declared private.
*/
- private Path fixRelativePart(Path p) {
+ Path fixRelativePart(Path p) {
if (p.isUriPathAbsolute()) {
return p;
} else {
@@ -1911,7 +1915,7 @@ public final class FileContext {
public FileStatus[] globStatus(Path pathPattern)
throws AccessControlException, UnsupportedFileSystemException,
IOException {
- return globStatus(pathPattern, DEFAULT_FILTER);
+ return new Globber(FileContext.this, pathPattern, DEFAULT_FILTER).glob();
}
/**
@@ -1940,154 +1944,7 @@ public final class FileContext {
public FileStatus[] globStatus(final Path pathPattern,
final PathFilter filter) throws AccessControlException,
UnsupportedFileSystemException, IOException {
- URI uri = getFSofPath(fixRelativePart(pathPattern)).getUri();
-
- String filename = pathPattern.toUri().getPath();
-
- List<String> filePatterns = GlobExpander.expand(filename);
- if (filePatterns.size() == 1) {
- Path absPathPattern = fixRelativePart(pathPattern);
- return globStatusInternal(uri, new Path(absPathPattern.toUri()
- .getPath()), filter);
- } else {
- List<FileStatus> results = new ArrayList<FileStatus>();
- for (String iFilePattern : filePatterns) {
- Path iAbsFilePattern = fixRelativePart(new Path(iFilePattern));
- FileStatus[] files = globStatusInternal(uri, iAbsFilePattern, filter);
- for (FileStatus file : files) {
- results.add(file);
- }
- }
- return results.toArray(new FileStatus[results.size()]);
- }
- }
-
- /**
- *
- * @param uri for all the inPathPattern
- * @param inPathPattern - without the scheme & authority (take from uri)
- * @param filter
- *
- * @return an array of FileStatus objects
- *
- * @throws AccessControlException If access is denied
- * @throws IOException If an I/O error occurred
- */
- private FileStatus[] globStatusInternal(final URI uri,
- final Path inPathPattern, final PathFilter filter)
- throws AccessControlException, IOException
- {
- Path[] parents = new Path[1];
- int level = 0;
-
- assert(inPathPattern.toUri().getScheme() == null &&
- inPathPattern.toUri().getAuthority() == null &&
- inPathPattern.isUriPathAbsolute());
-
-
- String filename = inPathPattern.toUri().getPath();
-
- // path has only zero component
- if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
- Path p = inPathPattern.makeQualified(uri, null);
- return getFileStatus(new Path[]{p});
- }
-
- // path has at least one component
- String[] components = filename.split(Path.SEPARATOR);
-
- // Path is absolute, first component is "/" hence first component
- // is the uri root
- parents[0] = new Path(new Path(uri), new Path("/"));
- level = 1;
-
- // glob the paths that match the parent path, ie. [0, components.length-1]
- boolean[] hasGlob = new boolean[]{false};
- Path[] relParentPaths =
- globPathsLevel(parents, components, level, hasGlob);
- FileStatus[] results;
-
- if (relParentPaths == null || relParentPaths.length == 0) {
- results = null;
- } else {
- // fix the pathes to be abs
- Path[] parentPaths = new Path [relParentPaths.length];
- for(int i=0; i<relParentPaths.length; i++) {
- parentPaths[i] = relParentPaths[i].makeQualified(uri, null);
- }
-
- // Now work on the last component of the path
- GlobFilter fp =
- new GlobFilter(components[components.length - 1], filter);
- if (fp.hasPattern()) { // last component has a pattern
- // list parent directories and then glob the results
- try {
- results = listStatus(parentPaths, fp);
- } catch (FileNotFoundException e) {
- results = null;
- }
- hasGlob[0] = true;
- } else { // last component does not have a pattern
- // get all the path names
- ArrayList<Path> filteredPaths =
- new ArrayList<Path>(parentPaths.length);
- for (int i = 0; i < parentPaths.length; i++) {
- parentPaths[i] = new Path(parentPaths[i],
- components[components.length - 1]);
- if (fp.accept(parentPaths[i])) {
- filteredPaths.add(parentPaths[i]);
- }
- }
- // get all their statuses
- results = getFileStatus(
- filteredPaths.toArray(new Path[filteredPaths.size()]));
- }
- }
-
- // Decide if the pathPattern contains a glob or not
- if (results == null) {
- if (hasGlob[0]) {
- results = new FileStatus[0];
- }
- } else {
- if (results.length == 0) {
- if (!hasGlob[0]) {
- results = null;
- }
- } else {
- Arrays.sort(results);
- }
- }
- return results;
- }
-
- /*
- * For a path of N components, return a list of paths that match the
- * components [<code>level</code>, <code>N-1</code>].
- */
- private Path[] globPathsLevel(Path[] parents, String[] filePattern,
- int level, boolean[] hasGlob) throws AccessControlException,
- FileNotFoundException, IOException {
- if (level == filePattern.length - 1) {
- return parents;
- }
- if (parents == null || parents.length == 0) {
- return null;
- }
- GlobFilter fp = new GlobFilter(filePattern[level]);
- if (fp.hasPattern()) {
- try {
- parents = FileUtil.stat2Paths(listStatus(parents, fp));
- } catch (FileNotFoundException e) {
- parents = null;
- }
- hasGlob[0] = true;
- } else {
- for (int i = 0; i < parents.length; i++) {
- parents[i] = new Path(parents[i], filePattern[level]);
- }
- }
- return globPathsLevel(parents, filePattern, level + 1, hasGlob);
+ return new Globber(FileContext.this, pathPattern, filter).glob();
}
/**
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Mon Aug 12 21:25:49 2013
@@ -120,6 +120,7 @@ public abstract class FileSystem extends
*/
private Set<Path> deleteOnExit = new TreeSet<Path>();
+ boolean resolveSymlinks;
/**
* This method adds a file system for testing so that we can find it later. It
* is only for testing.
@@ -196,6 +197,9 @@ public abstract class FileSystem extends
*/
public void initialize(URI name, Configuration conf) throws IOException {
statistics = getStatistics(name.getScheme(), getClass());
+ resolveSymlinks = conf.getBoolean(
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
}
/**
@@ -1615,7 +1619,7 @@ public abstract class FileSystem extends
* @throws IOException
*/
public FileStatus[] globStatus(Path pathPattern) throws IOException {
- return globStatus(pathPattern, DEFAULT_FILTER);
+ return new Globber(this, pathPattern, DEFAULT_FILTER).glob();
}
/**
@@ -1634,126 +1638,7 @@ public abstract class FileSystem extends
*/
public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
throws IOException {
- String filename = pathPattern.toUri().getPath();
- List<FileStatus> allMatches = null;
-
- List<String> filePatterns = GlobExpander.expand(filename);
- for (String filePattern : filePatterns) {
- Path path = new Path(filePattern.isEmpty() ? Path.CUR_DIR : filePattern);
- List<FileStatus> matches = globStatusInternal(path, filter);
- if (matches != null) {
- if (allMatches == null) {
- allMatches = matches;
- } else {
- allMatches.addAll(matches);
- }
- }
- }
-
- FileStatus[] results = null;
- if (allMatches != null) {
- results = allMatches.toArray(new FileStatus[allMatches.size()]);
- } else if (filePatterns.size() > 1) {
- // no matches with multiple expansions is a non-matching glob
- results = new FileStatus[0];
- }
- return results;
- }
-
- // sort gripes because FileStatus Comparable isn't parameterized...
- @SuppressWarnings("unchecked")
- private List<FileStatus> globStatusInternal(Path pathPattern,
- PathFilter filter) throws IOException {
- boolean patternHasGlob = false; // pathPattern has any globs
- List<FileStatus> matches = new ArrayList<FileStatus>();
-
- // determine starting point
- int level = 0;
- String baseDir = Path.CUR_DIR;
- if (pathPattern.isAbsolute()) {
- level = 1; // need to skip empty item at beginning of split list
- baseDir = Path.SEPARATOR;
- }
-
- // parse components and determine if it's a glob
- String[] components = null;
- GlobFilter[] filters = null;
- String filename = pathPattern.toUri().getPath();
- if (!filename.isEmpty() && !Path.SEPARATOR.equals(filename)) {
- components = filename.split(Path.SEPARATOR);
- filters = new GlobFilter[components.length];
- for (int i=level; i < components.length; i++) {
- filters[i] = new GlobFilter(components[i]);
- patternHasGlob |= filters[i].hasPattern();
- }
- if (!patternHasGlob) {
- baseDir = unquotePathComponent(filename);
- components = null; // short through to filter check
- }
- }
-
- // seed the parent directory path, return if it doesn't exist
- try {
- matches.add(getFileStatus(new Path(baseDir)));
- } catch (FileNotFoundException e) {
- return patternHasGlob ? matches : null;
- }
-
- // skip if there are no components other than the basedir
- if (components != null) {
- // iterate through each path component
- for (int i=level; (i < components.length) && !matches.isEmpty(); i++) {
- List<FileStatus> children = new ArrayList<FileStatus>();
- for (FileStatus match : matches) {
- // don't look for children in a file matched by a glob
- if (!match.isDirectory()) {
- continue;
- }
- try {
- if (filters[i].hasPattern()) {
- // get all children matching the filter
- FileStatus[] statuses = listStatus(match.getPath(), filters[i]);
- children.addAll(Arrays.asList(statuses));
- } else {
- // the component does not have a pattern
- String component = unquotePathComponent(components[i]);
- Path child = new Path(match.getPath(), component);
- children.add(getFileStatus(child));
- }
- } catch (FileNotFoundException e) {
- // don't care
- }
- }
- matches = children;
- }
- }
- // remove anything that didn't match the filter
- if (!matches.isEmpty()) {
- Iterator<FileStatus> iter = matches.iterator();
- while (iter.hasNext()) {
- if (!filter.accept(iter.next().getPath())) {
- iter.remove();
- }
- }
- }
- // no final paths, if there were any globs return empty list
- if (matches.isEmpty()) {
- return patternHasGlob ? matches : null;
- }
- Collections.sort(matches);
- return matches;
- }
-
- /**
- * The glob filter builds a regexp per path component. If the component
- * does not contain a shell metachar, then it falls back to appending the
- * raw string to the list of built up paths. This raw path needs to have
- * the quoting removed. Ie. convert all occurances of "\X" to "X"
- * @param name of the path component
- * @return the unquoted path component
- */
- private String unquotePathComponent(String name) {
- return name.replaceAll("\\\\(.)", "$1");
+ return new Globber(this, pathPattern, filter).glob();
}
/**
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java Mon Aug 12 21:25:49 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -72,12 +73,20 @@ public abstract class FileSystemLinkReso
int count = 0;
T in = null;
Path p = path;
- FileSystem fs = FileSystem.getFSofPath(p, filesys.getConf());
+ // Assumes path belongs to this FileSystem.
+ // Callers validate this by passing paths through FileSystem#checkPath
+ FileSystem fs = filesys;
for (boolean isLink = true; isLink;) {
try {
in = doCall(p);
isLink = false;
} catch (UnresolvedLinkException e) {
+ if (!filesys.resolveSymlinks) {
+ throw new IOException("Path " + path + " contains a symlink"
+ + " and symlink resolution is disabled ("
+ + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY
+ + ").", e);
+ }
if (count++ > FsConstants.MAX_PATH_LINKS) {
throw new IOException("Possible cyclic loop while " +
"following symbolic link " + path);
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Mon Aug 12 21:25:49 2013
@@ -142,7 +142,28 @@ public class FileUtil {
}
return deleteImpl(dir, true);
}
-
+
+ /**
+ * Returns the target of the given symlink. Returns the empty string if
+ * the given path does not refer to a symlink or there is an error
+ * accessing the symlink.
+ * @param f File representing the symbolic link.
+ * @return The target of the symbolic link, empty string on error or if not
+ * a symlink.
+ */
+ public static String readLink(File f) {
+ /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
+ * use getCanonicalPath in File to get the target of the symlink but that
+ * does not indicate if the given path refers to a symlink.
+ */
+ try {
+ return Shell.execCommand(
+ Shell.getReadlinkCommand(f.toString())).trim();
+ } catch (IOException x) {
+ return "";
+ }
+ }
+
/*
* Pure-Java implementation of "chmod +rwx f".
*/
@@ -737,15 +758,18 @@ public class FileUtil {
* On Windows, when symlink creation fails due to security
* setting, we will log a warning. The return code in this
* case is 2.
+ *
* @param target the target for symlink
* @param linkname the symlink
- * @return value returned by the command
+ * @return 0 on success
*/
public static int symLink(String target, String linkname) throws IOException{
// Run the input paths through Java's File so that they are converted to the
// native OS form
- File targetFile = new File(target);
- File linkFile = new File(linkname);
+ File targetFile = new File(
+ Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
+ File linkFile = new File(
+ Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
// If not on Java7+, copy a file instead of creating a symlink since
// Java6 has close to no support for symlinks on Windows. Specifically
@@ -757,9 +781,16 @@ public class FileUtil {
// is symlinked under userlogs and userlogs are generated afterwards).
if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
try {
- LOG.info("FileUtil#symlink: On Java6, copying file instead "
- + linkname + " -> " + target);
- org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
+ LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
+ "of creating a symlink. Copying " + target + " -> " + linkname);
+
+ if (!linkFile.getParentFile().exists()) {
+ LOG.warn("Parent directory " + linkFile.getParent() +
+ " does not exist.");
+ return 1;
+ } else {
+ org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
+ }
} catch (IOException ex) {
LOG.warn("FileUtil#symlink failed to copy the file with error: "
+ ex.getMessage());
@@ -769,10 +800,23 @@ public class FileUtil {
return 0;
}
- String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
- linkFile.getPath());
- ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
+ String[] cmd = Shell.getSymlinkCommand(
+ targetFile.toString(),
+ linkFile.toString());
+
+ ShellCommandExecutor shExec;
try {
+ if (Shell.WINDOWS &&
+ linkFile.getParentFile() != null &&
+ !new Path(target).isAbsolute()) {
+ // Relative links on Windows must be resolvable at the time of
+ // creation. To ensure this we run the shell command in the directory
+ // of the link.
+ //
+ shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
+ } else {
+ shExec = new ShellCommandExecutor(cmd);
+ }
shExec.execute();
} catch (Shell.ExitCodeException ec) {
int returnVal = ec.getExitCode();
@@ -795,7 +839,7 @@ public class FileUtil {
}
return shExec.getExitCode();
}
-
+
/**
* Change the permissions on a filename.
* @param filename the name of the file to change
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShellPermissions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShellPermissions.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShellPermissions.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShellPermissions.java Mon Aug 12 21:25:49 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.shell.Comman
import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.fs.shell.FsCommand;
import org.apache.hadoop.fs.shell.PathData;
-
+import org.apache.hadoop.util.Shell;
/**
* This class is the home for file permissions related commands.
@@ -111,7 +111,8 @@ public class FsShellPermissions extends
}
// used by chown/chgrp
- static private String allowedChars = "[-_./@a-zA-Z0-9]";
+ static private String allowedChars = Shell.WINDOWS ? "[-_./@a-zA-Z0-9 ]" :
+ "[-_./@a-zA-Z0-9]";
/**
* Used to change owner and/or group of files
@@ -126,9 +127,8 @@ public class FsShellPermissions extends
"\tcurrently supported.\n\n" +
"\tIf only owner or group is specified then only owner or\n" +
"\tgroup is modified.\n\n" +
- "\tThe owner and group names may only cosists of digits, alphabet,\n"+
- "\tand any of '-_.@/' i.e. [-_.@/a-zA-Z0-9]. The names are case\n" +
- "\tsensitive.\n\n" +
+ "\tThe owner and group names may only consist of digits, alphabet,\n"+
+ "\tand any of " + allowedChars + ". The names are case sensitive.\n\n" +
"\tWARNING: Avoid using '.' to separate user name and group though\n" +
"\tLinux allows it. If user names have dots in them and you are\n" +
"\tusing local file system, you might see surprising results since\n" +
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java Mon Aug 12 21:25:49 2013
@@ -24,11 +24,12 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeMap;
import java.util.HashMap;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -56,10 +57,12 @@ public class HarFileSystem extends Filte
private static final Log LOG = LogFactory.getLog(HarFileSystem.class);
+ public static final String METADATA_CACHE_ENTRIES_KEY = "fs.har.metadatacache.entries";
+ public static final int METADATA_CACHE_ENTRIES_DEFAULT = 10;
+
public static final int VERSION = 3;
- private static final Map<URI, HarMetaData> harMetaCache =
- new ConcurrentHashMap<URI, HarMetaData>();
+ private static Map<URI, HarMetaData> harMetaCache;
// uri representation of this Har filesystem
private URI uri;
@@ -98,7 +101,14 @@ public class HarFileSystem extends Filte
public HarFileSystem(FileSystem fs) {
super(fs);
}
-
+
+ private synchronized void initializeMetadataCache(Configuration conf) {
+ if (harMetaCache == null) {
+ int cacheSize = conf.getInt(METADATA_CACHE_ENTRIES_KEY, METADATA_CACHE_ENTRIES_DEFAULT);
+ harMetaCache = Collections.synchronizedMap(new LruCache<URI, HarMetaData>(cacheSize));
+ }
+ }
+
/**
* Initialize a Har filesystem per har archive. The
* archive home directory is the top level directory
@@ -114,6 +124,9 @@ public class HarFileSystem extends Filte
*/
@Override
public void initialize(URI name, Configuration conf) throws IOException {
+ // initialize the metadata cache, if needed
+ initializeMetadataCache(conf);
+
// decode the name
URI underLyingURI = decodeHarURI(name, conf);
// we got the right har Path- now check if this is
@@ -810,7 +823,8 @@ public class HarFileSystem extends Filte
/**
* Create an input stream that fakes all the reads/positions/seeking.
*/
- private static class HarFsInputStream extends FSInputStream {
+ private static class HarFsInputStream extends FSInputStream
+ implements CanSetDropBehind, CanSetReadahead {
private long position, start, end;
//The underlying data input stream that the
// underlying filesystem will return.
@@ -957,7 +971,18 @@ public class HarFileSystem extends Filte
public void readFully(long pos, byte[] b) throws IOException {
readFully(pos, b, 0, b.length);
}
-
+
+ @Override
+ public void setReadahead(Long readahead)
+ throws IOException, UnsupportedEncodingException {
+ underLyingStream.setReadahead(readahead);
+ }
+
+ @Override
+ public void setDropBehind(Boolean dropBehind)
+ throws IOException, UnsupportedEncodingException {
+ underLyingStream.setDropBehind(dropBehind);
+ }
}
/**
@@ -1105,4 +1130,18 @@ public class HarFileSystem extends Filte
HarMetaData getMetadata() {
return metadata;
}
+
+ private static class LruCache<K, V> extends LinkedHashMap<K, V> {
+ private final int MAX_ENTRIES;
+
+ public LruCache(int maxEntries) {
+ super(maxEntries + 1, 1.0f, true);
+ MAX_ENTRIES = maxEntries;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ return size() > MAX_ENTRIES;
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Mon Aug 12 21:25:49 2013
@@ -104,7 +104,7 @@ public class Path implements Comparable
// Add a slash to parent's path so resolution is compatible with URI's
URI parentUri = parent.uri;
String parentPath = parentUri.getPath();
- if (!(parentPath.equals("/") || parentPath.equals(""))) {
+ if (!(parentPath.equals("/") || parentPath.isEmpty())) {
try {
parentUri = new URI(parentUri.getScheme(), parentUri.getAuthority(),
parentUri.getPath()+"/", null, parentUri.getFragment());
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Mon Aug 12 21:25:49 2013
@@ -319,9 +319,35 @@ public class RawLocalFileSystem extends
@Override
public boolean rename(Path src, Path dst) throws IOException {
- if (pathToFile(src).renameTo(pathToFile(dst))) {
+ // Attempt rename using Java API.
+ File srcFile = pathToFile(src);
+ File dstFile = pathToFile(dst);
+ if (srcFile.renameTo(dstFile)) {
return true;
}
+
+ // Enforce POSIX rename behavior that a source directory replaces an existing
+ // destination if the destination is an empty directory. On most platforms,
+ // this is already handled by the Java API call above. Some platforms
+ // (notably Windows) do not provide this behavior, so the Java API call above
+ // fails. Delete destination and attempt rename again.
+ if (this.exists(dst)) {
+ FileStatus sdst = this.getFileStatus(dst);
+ if (sdst.isDirectory() && dstFile.list().length == 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting empty destination and renaming " + src + " to " +
+ dst);
+ }
+ if (this.delete(dst, false) && srcFile.renameTo(dstFile)) {
+ return true;
+ }
+ }
+ }
+
+ // The fallback behavior accomplishes the rename by a full copy.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Falling through to a copy of " + src + " to " + dst);
+ }
return FileUtil.copy(this, src, this, dst, true, getConf());
}
@@ -498,7 +524,7 @@ public class RawLocalFileSystem extends
* onwer.equals("").
*/
private boolean isPermissionLoaded() {
- return !super.getOwner().equals("");
+ return !super.getOwner().isEmpty();
}
RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
@@ -611,7 +637,7 @@ public class RawLocalFileSystem extends
FileUtil.makeShellPath(pathToFile(p), true)));
}
}
-
+
/**
* Sets the {@link Path}'s last modified time <em>only</em> to the given
* valid time.
@@ -634,14 +660,6 @@ public class RawLocalFileSystem extends
}
}
- private static String execCommand(File f, String... cmd) throws IOException {
- String[] args = new String[cmd.length + 1];
- System.arraycopy(cmd, 0, args, 0, cmd.length);
- args[cmd.length] = FileUtil.makeShellPath(f, true);
- String output = Shell.execCommand(args);
- return output;
- }
-
@Override
public boolean supportsSymlinks() {
return true;
@@ -658,31 +676,13 @@ public class RawLocalFileSystem extends
if (createParent) {
mkdirs(link.getParent());
}
- // NB: Use createSymbolicLink in java.nio.file.Path once available
- try {
- Shell.execCommand(Shell.getSymlinkCommand(
- Path.getPathWithoutSchemeAndAuthority(target).toString(),
- Path.getPathWithoutSchemeAndAuthority(makeAbsolute(link)).toString()));
- } catch (IOException x) {
- throw new IOException("Unable to create symlink: "+x.getMessage());
- }
- }
- /**
- * Returns the target of the given symlink. Returns the empty string if
- * the given path does not refer to a symlink or there is an error
- * accessing the symlink.
- */
- private String readLink(Path p) {
- /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
- * use getCanonicalPath in File to get the target of the symlink but that
- * does not indicate if the given path refers to a symlink.
- */
- try {
- final String path = p.toUri().getPath();
- return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim();
- } catch (IOException x) {
- return "";
+ // NB: Use createSymbolicLink in java.nio.file.Path once available
+ int result = FileUtil.symLink(target.toString(),
+ makeAbsolute(link).toString());
+ if (result != 0) {
+ throw new IOException("Error " + result + " creating symlink " +
+ link + " to " + target);
}
}
@@ -705,7 +705,7 @@ public class RawLocalFileSystem extends
}
private FileStatus getFileLinkStatusInternal(final Path f) throws IOException {
- String target = readLink(f);
+ String target = FileUtil.readLink(new File(f.toString()));
try {
FileStatus fs = getFileStatus(f);
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java Mon Aug 12 21:25:49 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.local;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@@ -28,12 +29,12 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.DelegateToFileSystem;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Shell;
/**
* The RawLocalFs implementation of AbstractFileSystem.
@@ -75,47 +76,29 @@ public class RawLocalFs extends Delegate
@Override
public boolean supportsSymlinks() {
return true;
- }
-
+ }
+
@Override
- public void createSymlink(Path target, Path link, boolean createParent)
+ public void createSymlink(Path target, Path link, boolean createParent)
throws IOException {
final String targetScheme = target.toUri().getScheme();
if (targetScheme != null && !"file".equals(targetScheme)) {
throw new IOException("Unable to create symlink to non-local file "+
- "system: "+target.toString());
+ "system: "+target.toString());
}
+
if (createParent) {
mkdir(link.getParent(), FsPermission.getDirDefault(), true);
}
+
// NB: Use createSymbolicLink in java.nio.file.Path once available
- try {
- Shell.execCommand(Shell.getSymlinkCommand(
- Path.getPathWithoutSchemeAndAuthority(target).toString(),
- Path.getPathWithoutSchemeAndAuthority(link).toString()));
- } catch (IOException x) {
- throw new IOException("Unable to create symlink: "+x.getMessage());
+ int result = FileUtil.symLink(target.toString(), link.toString());
+ if (result != 0) {
+ throw new IOException("Error " + result + " creating symlink " +
+ link + " to " + target);
}
}
- /**
- * Returns the target of the given symlink. Returns the empty string if
- * the given path does not refer to a symlink or there is an error
- * acessing the symlink.
- */
- private String readLink(Path p) {
- /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
- * use getCanonicalPath in File to get the target of the symlink but that
- * does not indicate if the given path refers to a symlink.
- */
- try {
- final String path = p.toUri().getPath();
- return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim();
- } catch (IOException x) {
- return "";
- }
- }
-
/**
* Return a FileStatus representing the given path. If the path refers
* to a symlink return a FileStatus representing the link rather than
@@ -123,11 +106,11 @@ public class RawLocalFs extends Delegate
*/
@Override
public FileStatus getFileLinkStatus(final Path f) throws IOException {
- String target = readLink(f);
+ String target = FileUtil.readLink(new File(f.toString()));
try {
FileStatus fs = getFileStatus(f);
// If f refers to a regular file or directory
- if ("".equals(target)) {
+ if (target.isEmpty()) {
return fs;
}
// Otherwise f refers to a symlink
@@ -149,7 +132,7 @@ public class RawLocalFs extends Delegate
* the readBasicFileAttributes method in java.nio.file.attributes
* when available.
*/
- if (!"".equals(target)) {
+ if (!target.isEmpty()) {
return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(),
"", "", new Path(target), f);
}
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Mon Aug 12 21:25:49 2013
@@ -300,7 +300,7 @@ public class NativeS3FileSystem extends
}
private static String pathToKey(Path path) {
- if (path.toUri().getScheme() != null && "".equals(path.toUri().getPath())) {
+ if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) {
// allow uris without trailing slash after bucket to refer to root,
// like s3n://mybucket
return "";
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Mon Aug 12 21:25:49 2013
@@ -18,18 +18,16 @@
package org.apache.hadoop.fs.shell;
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
/** Various commands for copy files */
@@ -44,6 +42,7 @@ class CopyCommands {
factory.addClass(CopyToLocal.class, "-copyToLocal");
factory.addClass(Get.class, "-get");
factory.addClass(Put.class, "-put");
+ factory.addClass(AppendToFile.class, "-appendToFile");
}
/** merge multiple files together */
@@ -235,4 +234,93 @@ class CopyCommands {
public static final String USAGE = Get.USAGE;
public static final String DESCRIPTION = "Identical to the -get command.";
}
+
+ /**
+ * Append the contents of one or more local files to a remote
+ * file.
+ */
+ public static class AppendToFile extends CommandWithDestination {
+ public static final String NAME = "appendToFile";
+ public static final String USAGE = "<localsrc> ... <dst>";
+ public static final String DESCRIPTION =
+ "Appends the contents of all the given local files to the\n" +
+ "given dst file. The dst file will be created if it does\n" +
+ "not exist. If <localSrc> is -, then the input is read\n" +
+ "from stdin.";
+
+ private static final int DEFAULT_IO_LENGTH = 1024 * 1024;
+ boolean readStdin = false;
+
+ // commands operating on local paths have no need for glob expansion
+ @Override
+ protected List<PathData> expandArgument(String arg) throws IOException {
+ List<PathData> items = new LinkedList<PathData>();
+ if (arg.equals("-")) {
+ readStdin = true;
+ } else {
+ try {
+ items.add(new PathData(new URI(arg), getConf()));
+ } catch (URISyntaxException e) {
+ if (Path.WINDOWS) {
+ // Unlike URI, PathData knows how to parse Windows drive-letter paths.
+ items.add(new PathData(arg, getConf()));
+ } else {
+ throw new IOException("Unexpected URISyntaxException: " + e.toString());
+ }
+ }
+ }
+ return items;
+ }
+
+ @Override
+ protected void processOptions(LinkedList<String> args)
+ throws IOException {
+
+ if (args.size() < 2) {
+ throw new IOException("missing destination argument");
+ }
+
+ getRemoteDestination(args);
+ super.processOptions(args);
+ }
+
+ @Override
+ protected void processArguments(LinkedList<PathData> args)
+ throws IOException {
+
+ if (!dst.exists) {
+ dst.fs.create(dst.path, false).close();
+ }
+
+ InputStream is = null;
+ FSDataOutputStream fos = dst.fs.append(dst.path);
+
+ try {
+ if (readStdin) {
+ if (args.size() == 0) {
+ IOUtils.copyBytes(System.in, fos, DEFAULT_IO_LENGTH);
+ } else {
+ throw new IOException(
+ "stdin (-) must be the sole input argument when present");
+ }
+ }
+
+ // Read in each input file and write to the target.
+ for (PathData source : args) {
+ is = new FileInputStream(source.toFile());
+ IOUtils.copyBytes(is, fos, DEFAULT_IO_LENGTH);
+ IOUtils.closeStream(is);
+ is = null;
+ }
+ } finally {
+ if (is != null) {
+ IOUtils.closeStream(is);
+ }
+
+ if (fos != null) {
+ IOUtils.closeStream(fos);
+ }
+ }
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java Mon Aug 12 21:25:49 2013
@@ -285,7 +285,7 @@ public class ActiveStandbyElector implem
String pathParts[] = znodeWorkingDir.split("/");
Preconditions.checkArgument(pathParts.length >= 1 &&
- "".equals(pathParts[0]),
+ pathParts[0].isEmpty(),
"Invalid path: %s", znodeWorkingDir);
StringBuilder sb = new StringBuilder();
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java Mon Aug 12 21:25:49 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.ha;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.KerberosInfo;
@@ -106,6 +107,7 @@ public interface HAServiceProtocol {
* @throws IOException
* if other errors happen
*/
+ @Idempotent
public void monitorHealth() throws HealthCheckFailedException,
AccessControlException,
IOException;
@@ -121,6 +123,7 @@ public interface HAServiceProtocol {
* @throws IOException
* if other errors happen
*/
+ @Idempotent
public void transitionToActive(StateChangeRequestInfo reqInfo)
throws ServiceFailedException,
AccessControlException,
@@ -137,6 +140,7 @@ public interface HAServiceProtocol {
* @throws IOException
* if other errors happen
*/
+ @Idempotent
public void transitionToStandby(StateChangeRequestInfo reqInfo)
throws ServiceFailedException,
AccessControlException,
@@ -152,6 +156,7 @@ public interface HAServiceProtocol {
* @throws IOException
* if other errors happen
*/
+ @Idempotent
public HAServiceStatus getServiceStatus() throws AccessControlException,
IOException;
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java Mon Aug 12 21:25:49 2013
@@ -241,7 +241,7 @@ public class SshFenceByTcpPort extends C
sshPort = DEFAULT_SSH_PORT;
// Parse optional user and ssh port
- if (arg != null && !"".equals(arg)) {
+ if (arg != null && !arg.isEmpty()) {
Matcher m = USER_PORT_RE.matcher(arg);
if (!m.matches()) {
throw new BadFencingConfigurationException(
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/lib/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java Mon Aug 12 21:25:49 2013
@@ -192,7 +192,7 @@ public class DefaultStringifier<T> imple
String[] parts = itemStr.split(SEPARATOR);
for (String part : parts) {
- if (!part.equals(""))
+ if (!part.isEmpty())
list.add(stringifier.fromString(part));
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java Mon Aug 12 21:25:49 2013
@@ -203,7 +203,7 @@ public class ReadaheadPool {
// It's also possible that we'll end up requesting readahead on some
// other FD, which may be wasted work, but won't cause a problem.
try {
- NativeIO.POSIX.posixFadviseIfPossible(fd, off, len,
+ NativeIO.POSIX.posixFadviseIfPossible(identifier, fd, off, len,
NativeIO.POSIX.POSIX_FADV_WILLNEED);
} catch (IOException ioe) {
if (canceled) {
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java Mon Aug 12 21:25:49 2013
@@ -69,6 +69,10 @@ public class Lz4Codec implements Configu
return NativeCodeLoader.isNativeCodeLoaded();
}
+ public static String getLibraryName() {
+ return Lz4Compressor.getLibraryName();
+ }
+
/**
* Create a {@link CompressionOutputStream} that will write to the given
* {@link OutputStream}.
@@ -103,7 +107,7 @@ public class Lz4Codec implements Configu
CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
- int compressionOverhead = Math.max((int)(bufferSize * 0.01), 10);
+ int compressionOverhead = bufferSize/255 + 16;
return new BlockCompressorStream(out, compressor, bufferSize,
compressionOverhead);
@@ -136,7 +140,10 @@ public class Lz4Codec implements Configu
int bufferSize = conf.getInt(
CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
- return new Lz4Compressor(bufferSize);
+ boolean useLz4HC = conf.getBoolean(
+ CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
+ CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_DEFAULT);
+ return new Lz4Compressor(bufferSize, useLz4HC);
}
/**
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java Mon Aug 12 21:25:49 2013
@@ -79,6 +79,10 @@ public class SnappyCodec implements Conf
SnappyDecompressor.isNativeCodeLoaded();
}
+ public static String getLibraryName() {
+ return SnappyCompressor.getLibraryName();
+ }
+
/**
* Create a {@link CompressionOutputStream} that will write to the given
* {@link OutputStream}.
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java Mon Aug 12 21:25:49 2013
@@ -298,4 +298,6 @@ public class Bzip2Compressor implements
private native static long getBytesRead(long strm);
private native static long getBytesWritten(long strm);
private native static void end(long strm);
+
+ public native static String getLibraryName();
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java Mon Aug 12 21:25:49 2013
@@ -78,6 +78,14 @@ public class Bzip2Factory {
return nativeBzip2Loaded;
}
+ public static String getLibraryName(Configuration conf) {
+ if (isNativeBzip2Loaded(conf)) {
+ return Bzip2Compressor.getLibraryName();
+ } else {
+ return bzip2LibraryName;
+ }
+ }
+
/**
* Return the appropriate type of the bzip2 compressor.
*
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java Mon Aug 12 21:25:49 2013
@@ -52,6 +52,7 @@ public class Lz4Compressor implements Co
private long bytesRead = 0L;
private long bytesWritten = 0L;
+ private final boolean useLz4HC;
static {
if (NativeCodeLoader.isNativeCodeLoaded()) {
@@ -72,8 +73,11 @@ public class Lz4Compressor implements Co
* Creates a new compressor.
*
* @param directBufferSize size of the direct buffer to be used.
+ * @param useLz4HC use high compression ratio version of lz4,
+ * which trades CPU for compression ratio.
*/
- public Lz4Compressor(int directBufferSize) {
+ public Lz4Compressor(int directBufferSize, boolean useLz4HC) {
+ this.useLz4HC = useLz4HC;
this.directBufferSize = directBufferSize;
uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
@@ -82,6 +86,15 @@ public class Lz4Compressor implements Co
}
/**
+ * Creates a new compressor.
+ *
+ * @param directBufferSize size of the direct buffer to be used.
+ */
+ public Lz4Compressor(int directBufferSize) {
+ this(directBufferSize, false);
+ }
+
+ /**
* Creates a new compressor with the default buffer size.
*/
public Lz4Compressor() {
@@ -227,7 +240,7 @@ public class Lz4Compressor implements Co
}
// Compress data
- n = compressBytesDirect();
+ n = useLz4HC ? compressBytesDirectHC() : compressBytesDirect();
compressedDirectBuf.limit(n);
uncompressedDirectBuf.clear(); // lz4 consumes all buffer input
@@ -296,4 +309,8 @@ public class Lz4Compressor implements Co
private native static void initIDs();
private native int compressBytesDirect();
+
+ private native int compressBytesDirectHC();
+
+ public native static String getLibraryName();
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java Mon Aug 12 21:25:49 2013
@@ -298,4 +298,6 @@ public class SnappyCompressor implements
private native static void initIDs();
private native int compressBytesDirect();
+
+ public native static String getLibraryName();
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java Mon Aug 12 21:25:49 2013
@@ -444,4 +444,6 @@ public class ZlibCompressor implements C
private native static long getBytesWritten(long strm);
private native static void reset(long strm);
private native static void end(long strm);
+
+ public native static String getLibraryName();
}
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java Mon Aug 12 21:25:49 2013
@@ -65,7 +65,11 @@ public class ZlibFactory {
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT);
}
-
+
+ public static String getLibraryName() {
+ return ZlibCompressor.getLibraryName();
+ }
+
/**
* Return the appropriate type of the zlib compressor.
*
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java Mon Aug 12 21:25:49 2013
@@ -2105,7 +2105,7 @@ public class TFile {
}
public boolean isSorted() {
- return !strComparator.equals("");
+ return !strComparator.isEmpty();
}
public String getComparatorString() {
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java Mon Aug 12 21:25:49 2013
@@ -37,6 +37,8 @@ import org.apache.hadoop.util.Shell;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* JNI wrappers for various native IO-related calls not available in Java.
* These functions should generally be used alongside a fallback to another
@@ -92,6 +94,9 @@ public class NativeIO {
private static final Log LOG = LogFactory.getLog(NativeIO.class);
+ @VisibleForTesting
+ public static CacheTracker cacheTracker = null;
+
private static boolean nativeLoaded = false;
private static boolean fadvisePossible = true;
private static boolean syncFileRangePossible = true;
@@ -102,6 +107,10 @@ public class NativeIO {
private static long cacheTimeout = -1;
+ public static interface CacheTracker {
+ public void fadvise(String identifier, long offset, long len, int flags);
+ }
+
static {
if (NativeCodeLoader.isNativeCodeLoaded()) {
try {
@@ -178,9 +187,12 @@ public class NativeIO {
*
* @throws NativeIOException if there is an error with the syscall
*/
- public static void posixFadviseIfPossible(
+ public static void posixFadviseIfPossible(String identifier,
FileDescriptor fd, long offset, long len, int flags)
throws NativeIOException {
+ if (cacheTracker != null) {
+ cacheTracker.fadvise(identifier, offset, len, flags);
+ }
if (nativeLoaded && fadvisePossible) {
try {
posix_fadvise(fd, offset, len, flags);
Propchange: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java Mon Aug 12 21:25:49 2013
@@ -51,8 +51,8 @@ public interface FailoverProxyProvider<T
/**
* Return a reference to the interface this provider's proxy objects actually
* implement. If any of the methods on this interface are annotated as being
- * {@link Idempotent}, then this fact will be passed to the
- * {@link RetryPolicy#shouldRetry(Exception, int, int, boolean)} method on
+ * {@link Idempotent} or {@link AtMostOnce}, then this fact will be passed to
+ * the {@link RetryPolicy#shouldRetry(Exception, int, int, boolean)} method on
* error, for use in determining whether or not failover should be attempted.
*
* @return the interface implemented by the proxy objects returned by
Modified: hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java Mon Aug 12 21:25:49 2013
@@ -18,23 +18,36 @@
package org.apache.hadoop.io.retry;
import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
-import org.apache.hadoop.util.ThreadUtil;
+import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.Client.ConnectionId;
+import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcConstants;
import org.apache.hadoop.ipc.RpcInvocationHandler;
+import org.apache.hadoop.util.ThreadUtil;
+
+import com.google.common.annotations.VisibleForTesting;
-class RetryInvocationHandler implements RpcInvocationHandler {
+/**
+ * This class implements RpcInvocationHandler and supports retry on the client
+ * side.
+ */
+@InterfaceAudience.Private
+public class RetryInvocationHandler<T> implements RpcInvocationHandler {
public static final Log LOG = LogFactory.getLog(RetryInvocationHandler.class);
- private final FailoverProxyProvider proxyProvider;
+ private final FailoverProxyProvider<T> proxyProvider;
/**
* The number of times the associated proxyProvider has ever been failed over.
@@ -44,14 +57,14 @@ class RetryInvocationHandler implements
private final RetryPolicy defaultPolicy;
private final Map<String,RetryPolicy> methodNameToPolicyMap;
- private Object currentProxy;
-
- public RetryInvocationHandler(FailoverProxyProvider proxyProvider,
+ private T currentProxy;
+
+ protected RetryInvocationHandler(FailoverProxyProvider<T> proxyProvider,
RetryPolicy retryPolicy) {
this(proxyProvider, retryPolicy, Collections.<String, RetryPolicy>emptyMap());
}
- public RetryInvocationHandler(FailoverProxyProvider proxyProvider,
+ RetryInvocationHandler(FailoverProxyProvider<T> proxyProvider,
RetryPolicy defaultPolicy,
Map<String, RetryPolicy> methodNameToPolicyMap) {
this.proxyProvider = proxyProvider;
@@ -70,6 +83,8 @@ class RetryInvocationHandler implements
// The number of times this method invocation has been failed over.
int invocationFailoverCount = 0;
+ final boolean isRpc = isRpcInvocation(currentProxy);
+ final int callId = isRpc? Client.nextCallId(): RpcConstants.INVALID_CALL_ID;
int retries = 0;
while (true) {
// The number of times this invocation handler has ever been failed over,
@@ -79,16 +94,25 @@ class RetryInvocationHandler implements
synchronized (proxyProvider) {
invocationAttemptFailoverCount = proxyProviderFailoverCount;
}
+
+ if (isRpc) {
+ Client.setCallIdAndRetryCount(callId, retries);
+ }
try {
Object ret = invokeMethod(method, args);
hasMadeASuccessfulCall = true;
return ret;
} catch (Exception e) {
- boolean isMethodIdempotent = proxyProvider.getInterface()
+ boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
.getMethod(method.getName(), method.getParameterTypes())
.isAnnotationPresent(Idempotent.class);
- RetryAction action = policy.shouldRetry(e, retries++, invocationFailoverCount,
- isMethodIdempotent);
+ if (!isIdempotentOrAtMostOnce) {
+ isIdempotentOrAtMostOnce = proxyProvider.getInterface()
+ .getMethod(method.getName(), method.getParameterTypes())
+ .isAnnotationPresent(AtMostOnce.class);
+ }
+ RetryAction action = policy.shouldRetry(e, retries++,
+ invocationFailoverCount, isIdempotentOrAtMostOnce);
if (action.action == RetryAction.RetryDecision.FAIL) {
if (action.reason != null) {
LOG.warn("Exception while invoking " +
@@ -156,7 +180,7 @@ class RetryInvocationHandler implements
}
}
- private Object invokeMethod(Method method, Object[] args) throws Throwable {
+ protected Object invokeMethod(Method method, Object[] args) throws Throwable {
try {
if (!method.isAccessible()) {
method.setAccessible(true);
@@ -167,6 +191,18 @@ class RetryInvocationHandler implements
}
}
+ @VisibleForTesting
+ static boolean isRpcInvocation(Object proxy) {
+ if (proxy instanceof ProtocolTranslator) {
+ proxy = ((ProtocolTranslator) proxy).getUnderlyingProxyObject();
+ }
+ if (!Proxy.isProxyClass(proxy.getClass())) {
+ return false;
+ }
+ final InvocationHandler ih = Proxy.getInvocationHandler(proxy);
+ return ih instanceof RpcInvocationHandler;
+ }
+
@Override
public void close() throws IOException {
proxyProvider.close();