You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2013/09/26 05:41:12 UTC
svn commit: r1526366 - in
/hadoop/common/branches/HDFS-2832/hadoop-common-project: hadoop-common/
hadoop-common/src/main/docs/ hadoop-common/src/main/java/
hadoop-common/src/main/java/org/apache/hadoop/fs/
hadoop-common/src/main/resources/ hadoop-commo...
Author: arp
Date: Thu Sep 26 03:41:09 2013
New Revision: 1526366
URL: http://svn.apache.org/r1526366
Log:
Merging r1525759 through r1526365 from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/resources/core-site.xml
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt Thu Sep 26 03:41:09 2013
@@ -342,9 +342,6 @@ Release 2.3.0 - UNRELEASED
HADOOP-9582. Non-existent file to "hadoop fs -conf" doesn't throw error
(Ashwin Shankar via jlowe)
- HADOOP-9761. ViewFileSystem#rename fails when using DistributedFileSystem.
- (Andrew Wang via Colin Patrick McCabe)
-
HADOOP-9817. FileSystem#globStatus and FileContext#globStatus need to work
with symlinks. (Colin Patrick McCabe via Andrew Wang)
@@ -372,6 +369,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-9791. Add a test case covering long paths for new FileUtil access
check methods (ivanmi)
+ HADOOP-9981. globStatus should minimize its listStatus and getFileStatus
+ calls. (Contributed by Colin Patrick McCabe)
+
Release 2.2.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -384,6 +384,24 @@ Release 2.2.0 - UNRELEASED
BUG FIXES
+Release 2.1.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ HADOOP-9776. HarFileSystem.listStatus() returns invalid authority if port
+ number is empty. (Shanyu Zhao via ivanmi)
+
+ HADOOP-9761. ViewFileSystem#rename fails when using DistributedFileSystem.
+ (Andrew Wang via Colin Patrick McCabe)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1525759-1526365
Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1525759-1526365
Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1525759-1526365
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java Thu Sep 26 03:41:09 2013
@@ -84,6 +84,15 @@ class Globber {
}
/**
+ * Convert a path component that contains backslash ecape sequences to a
+ * literal string. This is necessary when you want to explicitly refer to a
+ * path that contains globber metacharacters.
+ */
+ private static String unescapePathComponent(String name) {
+ return name.replaceAll("\\\\(.)", "$1");
+ }
+
+ /**
* Translate an absolute path into a list of path components.
* We merge double slashes into a single slash here.
* POSIX root path, i.e. '/', does not get an entry in the list.
@@ -166,37 +175,72 @@ class Globber {
new Path(scheme, authority, Path.SEPARATOR)));
}
- for (String component : components) {
+ for (int componentIdx = 0; componentIdx < components.size();
+ componentIdx++) {
ArrayList<FileStatus> newCandidates =
new ArrayList<FileStatus>(candidates.size());
- GlobFilter globFilter = new GlobFilter(component);
+ GlobFilter globFilter = new GlobFilter(components.get(componentIdx));
+ String component = unescapePathComponent(components.get(componentIdx));
if (globFilter.hasPattern()) {
sawWildcard = true;
}
if (candidates.isEmpty() && sawWildcard) {
+ // Optimization: if there are no more candidates left, stop examining
+ // the path components. We can only do this if we've already seen
+ // a wildcard component-- otherwise, we still need to visit all path
+ // components in case one of them is a wildcard.
break;
}
- for (FileStatus candidate : candidates) {
- FileStatus resolvedCandidate = candidate;
- if (candidate.isSymlink()) {
- // We have to resolve symlinks, because otherwise we don't know
- // whether they are directories.
- resolvedCandidate = getFileStatus(candidate.getPath());
+ if ((componentIdx < components.size() - 1) &&
+ (!globFilter.hasPattern())) {
+ // Optimization: if this is not the terminal path component, and we
+ // are not matching against a glob, assume that it exists. If it
+ // doesn't exist, we'll find out later when resolving a later glob
+ // or the terminal path component.
+ for (FileStatus candidate : candidates) {
+ candidate.setPath(new Path(candidate.getPath(), component));
}
- if (resolvedCandidate == null ||
- resolvedCandidate.isDirectory() == false) {
- continue;
- }
- FileStatus[] children = listStatus(candidate.getPath());
- for (FileStatus child : children) {
- // Set the child path based on the parent path.
- // This keeps the symlinks in our path.
- child.setPath(new Path(candidate.getPath(),
- child.getPath().getName()));
- if (globFilter.accept(child.getPath())) {
- newCandidates.add(child);
+ continue;
+ }
+ for (FileStatus candidate : candidates) {
+ if (globFilter.hasPattern()) {
+ FileStatus[] children = listStatus(candidate.getPath());
+ if (children.length == 1) {
+ // If we get back only one result, this could be either a listing
+ // of a directory with one entry, or it could reflect the fact
+ // that what we listed resolved to a file.
+ //
+ // Unfortunately, we can't just compare the returned paths to
+ // figure this out. Consider the case where you have /a/b, where
+ // b is a symlink to "..". In that case, listing /a/b will give
+ // back "/a/b" again. If we just went by returned pathname, we'd
+ // incorrectly conclude that /a/b was a file and should not match
+ // /a/*/*. So we use getFileStatus of the path we just listed to
+ // disambiguate.
+ if (!getFileStatus(candidate.getPath()).isDirectory()) {
+ continue;
+ }
}
- }
+ for (FileStatus child : children) {
+ // Set the child path based on the parent path.
+ child.setPath(new Path(candidate.getPath(),
+ child.getPath().getName()));
+ if (globFilter.accept(child.getPath())) {
+ newCandidates.add(child);
+ }
+ }
+ } else {
+ // When dealing with non-glob components, use getFileStatus
+ // instead of listStatus. This is an optimization, but it also
+ // is necessary for correctness in HDFS, since there are some
+ // special HDFS directories like .reserved and .snapshot that are
+ // not visible to listStatus, but which do exist. (See HADOOP-9877)
+ FileStatus childStatus = getFileStatus(
+ new Path(candidate.getPath(), component));
+ if (childStatus != null) {
+ newCandidates.add(childStatus);
+ }
+ }
}
candidates = newCandidates;
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java Thu Sep 26 03:41:09 2013
@@ -283,8 +283,9 @@ public class HarFileSystem extends Filte
private String getHarAuth(URI underLyingUri) {
String auth = underLyingUri.getScheme() + "-";
if (underLyingUri.getHost() != null) {
- auth += underLyingUri.getHost() + ":";
+ auth += underLyingUri.getHost();
if (underLyingUri.getPort() != -1) {
+ auth += ":";
auth += underLyingUri.getPort();
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Thu Sep 26 03:41:09 2013
@@ -1226,4 +1226,19 @@
</description>
</property>
+<property>
+ <name>nfs3.server.port</name>
+ <value>2049</value>
+ <description>
+ Specify the port number used by Hadoop NFS.
+ </description>
+</property>
+
+<property>
+ <name>nfs3.mountd.port</name>
+ <value>4242</value>
+ <description>
+ Specify the port number used by Hadoop mount daemon.
+ </description>
+</property>
</configuration>
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm Thu Sep 26 03:41:09 2013
@@ -311,7 +311,7 @@ Hadoop MapReduce Next Generation - Clust
| | | Only applicable if log-aggregation is enabled. |
*-------------------------+-------------------------+------------------------+
| <<<yarn.nodemanager.aux-services>>> | | |
-| | mapreduce.shuffle | |
+| | mapreduce_shuffle | |
| | | Shuffle service that needs to be set for Map Reduce applications. |
*-------------------------+-------------------------+------------------------+
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm Thu Sep 26 03:41:09 2013
@@ -140,7 +140,7 @@ Add the following configs to your <<<yar
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce.shuffle</value>
+ <value>mapreduce_shuffle</value>
<description>shuffle service that needs to be set for Map Reduce to run </description>
</property>
+---+
Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1525759-1526365
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java Thu Sep 26 03:41:09 2013
@@ -221,6 +221,17 @@ public class TestHarFileSystemBasics {
hfs.initialize(uri, new Configuration());
}
+ @Test
+ public void testPositiveListFilesNotEndInColon() throws Exception {
+ // re-initialize the har file system with host name
+ // make sure the qualified path name does not append ":" at the end of host name
+ final URI uri = new URI("har://file-localhost" + harPath.toString());
+ harFileSystem.initialize(uri, conf);
+ Path p1 = new Path("har://file-localhost" + harPath.toString());
+ Path p2 = harFileSystem.makeQualified(p1);
+ assertTrue(p2.toUri().toString().startsWith("har://file-localhost/"));
+ }
+
// ========== Negative:
@Test
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/resources/core-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/resources/core-site.xml?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/resources/core-site.xml (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/resources/core-site.xml Thu Sep 26 03:41:09 2013
@@ -69,4 +69,13 @@
<value>simple</value>
</property>
+<property>
+ <name>nfs3.server.port</name>
+ <value>2079</value>
+</property>
+
+<property>
+ <name>nfs3.mountd.port</name>
+ <value>4272</value>
+</property>
</configuration>
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java?rev=1526366&r1=1526365&r2=1526366&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java Thu Sep 26 03:41:09 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.nfs.nfs3;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mount.MountdBase;
import org.apache.hadoop.oncrpc.RpcProgram;
import org.apache.hadoop.oncrpc.RpcUtil;
@@ -38,6 +39,7 @@ public abstract class Nfs3Base {
public static final Log LOG = LogFactory.getLog(Nfs3Base.class);
private final MountdBase mountd;
private final RpcProgram rpcProgram;
+ private final int nfsPort;
public MountdBase getMountBase() {
return mountd;
@@ -47,9 +49,17 @@ public abstract class Nfs3Base {
return rpcProgram;
}
+ protected Nfs3Base(MountdBase mountd, RpcProgram program, Configuration conf) {
+ this.mountd = mountd;
+ this.rpcProgram = program;
+ this.nfsPort = conf.getInt("nfs3.server.port", Nfs3Constant.PORT);
+ LOG.info("NFS server port set to: "+nfsPort);
+ }
+
protected Nfs3Base(MountdBase mountd, RpcProgram program) {
this.mountd = mountd;
this.rpcProgram = program;
+ this.nfsPort = Nfs3Constant.PORT;
}
public void start(boolean register) {
@@ -61,7 +71,7 @@ public abstract class Nfs3Base {
}
private void startTCPServer() {
- SimpleTcpServer tcpServer = new SimpleTcpServer(Nfs3Constant.PORT,
+ SimpleTcpServer tcpServer = new SimpleTcpServer(nfsPort,
rpcProgram, 0) {
@Override
public ChannelPipelineFactory getPipelineFactory() {