You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2013/03/28 21:57:54 UTC
svn commit: r1462301 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/ src/test/java/org/apache/hadoop/fs/
src/test/java/org/apache/hadoop/test/
Author: todd
Date: Thu Mar 28 20:57:53 2013
New Revision: 1462301
URL: http://svn.apache.org/r1462301
Log:
HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs. Contributed by Todd Lipcon.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1462301&r1=1462300&r2=1462301&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Mar 28 20:57:53 2013
@@ -35,6 +35,9 @@ Release 2.0.5-beta - UNRELEASED
OPTIMIZATIONS
+ HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
+ (todd)
+
BUG FIXES
HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh)
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1462301&r1=1462300&r2=1462301&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Thu Mar 28 20:57:53 2013
@@ -21,6 +21,7 @@ import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
@@ -211,12 +212,46 @@ public abstract class FileSystem extends
public abstract URI getUri();
/**
- * Resolve the uri's hostname and add the default port if not in the uri
+ * Return a canonicalized form of this FileSystem's URI.
+ *
+ * The default implementation simply calls {@link #canonicalizeUri(URI)}
+ * on the filesystem's own URI, so subclasses typically only need to
+ * implement that method.
+ *
+ * @see #canonicalizeUri(URI)
+ */
+ protected URI getCanonicalUri() {
+ return canonicalizeUri(getUri());
+ }
+
+ /**
+ * Canonicalize the given URI.
+ *
+ * This is filesystem-dependent, but may for example consist of
+ * canonicalizing the hostname using DNS and adding the default
+ * port if not specified.
+ *
+ * The default implementation simply fills in the default port if
+ * not specified and if the filesystem has a default port.
+ *
* @return URI
* @see NetUtils#getCanonicalUri(URI, int)
*/
- protected URI getCanonicalUri() {
- return NetUtils.getCanonicalUri(getUri(), getDefaultPort());
+ protected URI canonicalizeUri(URI uri) {
+ if (uri.getPort() == -1 && getDefaultPort() > 0) {
+ // reconstruct the uri with the default port set
+ try {
+ uri = new URI(uri.getScheme(), uri.getUserInfo(),
+ uri.getHost(), getDefaultPort(),
+ uri.getPath(), uri.getQuery(), uri.getFragment());
+ } catch (URISyntaxException e) {
+ // Should never happen!
+ throw new AssertionError("Valid URI became unparseable: " +
+ uri);
+ }
+ }
+
+ return uri;
}
/**
@@ -581,7 +616,7 @@ public abstract class FileSystem extends
}
if (uri != null) {
// canonicalize uri before comparing with this fs
- uri = NetUtils.getCanonicalUri(uri, getDefaultPort());
+ uri = canonicalizeUri(uri);
thatAuthority = uri.getAuthority();
if (thisAuthority == thatAuthority || // authorities match
(thisAuthority != null &&
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1462301&r1=1462300&r2=1462301&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Thu Mar 28 20:57:53 2013
@@ -95,16 +95,18 @@ public class FilterFileSystem extends Fi
public URI getUri() {
return fs.getUri();
}
-
- /**
- * Returns a qualified URI whose scheme and authority identify this
- * FileSystem.
- */
+
+
@Override
protected URI getCanonicalUri() {
return fs.getCanonicalUri();
}
-
+
+ @Override
+ protected URI canonicalizeUri(URI uri) {
+ return fs.canonicalizeUri(uri);
+ }
+
/** Make sure that a path specifies a FileSystem. */
@Override
public Path makeQualified(Path path) {
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1462301&r1=1462300&r2=1462301&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Thu Mar 28 20:57:53 2013
@@ -26,6 +26,7 @@ import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.NetUtilsTestResolver;
import org.apache.hadoop.util.Progressable;
import org.junit.BeforeClass;
@@ -313,6 +314,11 @@ public class TestFileSystemCanonicalizat
}
@Override
+ protected URI canonicalizeUri(URI uri) {
+ return NetUtils.getCanonicalUri(uri, getDefaultPort());
+ }
+
+ @Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
throw new IOException("not supposed to be here");
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java?rev=1462301&r1=1462300&r2=1462301&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java Thu Mar 28 20:57:53 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.test;
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
+import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Random;
import java.util.Set;
@@ -266,15 +267,29 @@ public abstract class GenericTestUtils {
*/
public static class DelegateAnswer implements Answer<Object> {
private final Object delegate;
+ private final Log log;
public DelegateAnswer(Object delegate) {
+ this(null, delegate);
+ }
+
+ public DelegateAnswer(Log log, Object delegate) {
+ this.log = log;
this.delegate = delegate;
}
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
- return invocation.getMethod().invoke(
- delegate, invocation.getArguments());
+ try {
+ if (log != null) {
+ log.info("Call to " + invocation + " on " + delegate,
+ new Exception("TRACE"));
+ }
+ return invocation.getMethod().invoke(
+ delegate, invocation.getArguments());
+ } catch (InvocationTargetException ite) {
+ throw ite.getCause();
+ }
}
}