You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2014/02/13 19:21:48 UTC
svn commit: r1567994 - in
/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common: ./
src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/ipc/
src/main/java/org/apache/hadoop/net/unix/ src/mai...
Author: cnauroth
Date: Thu Feb 13 18:21:46 2014
New Revision: 1567994
URL: http://svn.apache.org/r1567994
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Waitable.java
- copied unchanged from r1567993, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Waitable.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1567994&r1=1567993&r2=1567994&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt Thu Feb 13 18:21:46 2014
@@ -334,6 +334,9 @@ Release 2.4.0 - UNRELEASED
HADOOP-10326. M/R jobs can not access S3 if Kerberos is enabled. (bc Wong
via atm)
+ HADOOP-10338. Cannot get the FileStatus of the root inode from the new
+ Globber (cmccabe)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1567238-1567993
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1567238-1567993
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1567238-1567993
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java?rev=1567994&r1=1567993&r2=1567994&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java Thu Feb 13 18:21:46 2014
@@ -114,7 +114,8 @@ class Globber {
if (fs != null) {
scheme = fs.getUri().getScheme();
} else {
- scheme = fc.getDefaultFileSystem().getUri().getScheme();
+ scheme = fc.getFSofPath(fc.fixRelativePart(path)).
+ getUri().getScheme();
}
}
return scheme;
@@ -126,7 +127,8 @@ class Globber {
if (fs != null) {
authority = fs.getUri().getAuthority();
} else {
- authority = fc.getDefaultFileSystem().getUri().getAuthority();
+ authority = fc.getFSofPath(fc.fixRelativePart(path)).
+ getUri().getAuthority();
}
}
return authority ;
@@ -162,18 +164,26 @@ class Globber {
// Starting out at the root of the filesystem, we try to match
// filesystem entries against pattern components.
ArrayList<FileStatus> candidates = new ArrayList<FileStatus>(1);
+ // To get the "real" FileStatus of root, we'd have to do an expensive
+ // RPC to the NameNode. So we create a placeholder FileStatus which has
+ // the correct path, but defaults for the rest of the information.
+ // Later, if it turns out we actually want the FileStatus of root, we'll
+ // replace the placeholder with a real FileStatus obtained from the
+ // NameNode.
+ FileStatus rootPlaceholder;
if (Path.WINDOWS && !components.isEmpty()
&& Path.isWindowsAbsolutePath(absPattern.toUri().getPath(), true)) {
// On Windows the path could begin with a drive letter, e.g. /E:/foo.
// We will skip matching the drive letter and start from listing the
// root of the filesystem on that drive.
String driveLetter = components.remove(0);
- candidates.add(new FileStatus(0, true, 0, 0, 0, new Path(scheme,
- authority, Path.SEPARATOR + driveLetter + Path.SEPARATOR)));
+ rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme,
+ authority, Path.SEPARATOR + driveLetter + Path.SEPARATOR));
} else {
- candidates.add(new FileStatus(0, true, 0, 0, 0,
- new Path(scheme, authority, Path.SEPARATOR)));
+ rootPlaceholder = new FileStatus(0, true, 0, 0, 0,
+ new Path(scheme, authority, Path.SEPARATOR));
}
+ candidates.add(rootPlaceholder);
for (int componentIdx = 0; componentIdx < components.size();
componentIdx++) {
@@ -245,6 +255,12 @@ class Globber {
candidates = newCandidates;
}
for (FileStatus status : candidates) {
+ // Use object equality to see if this status is the root placeholder.
+ // See the explanation for rootPlaceholder above for more information.
+ if (status == rootPlaceholder) {
+ status = getFileStatus(rootPlaceholder.getPath());
+ if (status == null) continue;
+ }
// HADOOP-3497 semantics: the user-defined filter is applied at the
// end, once the full path is built up.
if (filter.accept(status.getPath())) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1567994&r1=1567993&r2=1567994&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java Thu Feb 13 18:21:46 2014
@@ -220,7 +220,7 @@ public class Client {
* @param conf Configuration
* @return the ping interval
*/
- final static int getPingInterval(Configuration conf) {
+ final public static int getPingInterval(Configuration conf) {
return conf.getInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,
CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT);
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java?rev=1567994&r1=1567993&r2=1567994&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java Thu Feb 13 18:21:46 2014
@@ -37,6 +37,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.NativeCodeLoader;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.Uninterruptibles;
@@ -48,7 +49,7 @@ import com.google.common.util.concurrent
* See {@link DomainSocket} for more information about UNIX domain sockets.
*/
@InterfaceAudience.LimitedPrivate("HDFS")
-public final class DomainSocketWatcher extends Thread implements Closeable {
+public final class DomainSocketWatcher implements Closeable {
static {
if (SystemUtils.IS_OS_WINDOWS) {
loadingFailureReason = "UNIX Domain sockets are not available on Windows.";
@@ -281,7 +282,7 @@ public final class DomainSocketWatcher e
try {
processedCond.await();
} catch (InterruptedException e) {
- this.interrupt();
+ Thread.currentThread().interrupt();
}
if (!toAdd.contains(entry)) {
break;
@@ -308,7 +309,7 @@ public final class DomainSocketWatcher e
try {
processedCond.await();
} catch (InterruptedException e) {
- this.interrupt();
+ Thread.currentThread().interrupt();
}
if (!toRemove.containsKey(sock.fd)) {
break;
@@ -381,7 +382,8 @@ public final class DomainSocketWatcher e
}
}
- private final Thread watcherThread = new Thread(new Runnable() {
+ @VisibleForTesting
+ final Thread watcherThread = new Thread(new Runnable() {
@Override
public void run() {
LOG.info(this + ": starting with interruptCheckPeriodMs = " +
@@ -443,6 +445,7 @@ public final class DomainSocketWatcher e
} catch (IOException e) {
LOG.error(toString() + " terminating on IOException", e);
} finally {
+ kick(); // allow the handler for notificationSockets[0] to read a byte
for (Entry entry : entries.values()) {
sendCallback("close", entries, fdSet, entry.getDomainSocket().fd);
}
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1567238-1567993
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java?rev=1567994&r1=1567993&r2=1567994&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java Thu Feb 13 18:21:46 2014
@@ -73,9 +73,10 @@ public class TestDomainSocketWatcher {
*/
@Test(timeout=60000)
public void testInterruption() throws Exception {
- DomainSocketWatcher watcher = new DomainSocketWatcher(10);
- watcher.interrupt();
- Uninterruptibles.joinUninterruptibly(watcher);
+ final DomainSocketWatcher watcher = new DomainSocketWatcher(10);
+ watcher.watcherThread.interrupt();
+ Uninterruptibles.joinUninterruptibly(watcher.watcherThread);
+ watcher.close();
}
@Test(timeout=300000)