You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/12/23 23:19:13 UTC
svn commit: r1553226 - in
/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common: ./
src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/http/
src/main/java/org/apache/hadoop/io/retry/ src/ma...
Author: cnauroth
Date: Mon Dec 23 22:19:12 2013
New Revision: 1553226
URL: http://svn.apache.org/r1553226
Log:
Merge trunk to HDFS-4685.
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1553226&r1=1553225&r2=1553226&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt Mon Dec 23 22:19:12 2013
@@ -404,6 +404,9 @@ Release 2.4.0 - UNRELEASED
HADOOP-10164. Allow UGI to login with a known Subject (bobby)
+ HADOOP-10169. Remove the unnecessary synchronized in JvmMetrics class.
+ (Liang Xie via jing9)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
@@ -562,6 +565,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-10087. UserGroupInformation.getGroupNames() fails to return primary
group first when JniBasedUnixGroupsMappingWithFallback is used (cmccabe)
+ HADOOP-10175. Har files system authority should preserve userinfo.
+ (Chuan Liu via cnauroth)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1552466-1553224
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1552466-1553224
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1552466-1553224
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1553226&r1=1553225&r2=1553226&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java Mon Dec 23 22:19:12 2013
@@ -294,6 +294,10 @@ public class HarFileSystem extends FileS
private String getHarAuth(URI underLyingUri) {
String auth = underLyingUri.getScheme() + "-";
if (underLyingUri.getHost() != null) {
+ if (underLyingUri.getUserInfo() != null) {
+ auth += underLyingUri.getUserInfo();
+ auth += "@";
+ }
auth += underLyingUri.getHost();
if (underLyingUri.getPort() != -1) {
auth += ":";
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1553226&r1=1553225&r2=1553226&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Mon Dec 23 22:19:12 2013
@@ -305,12 +305,13 @@ public class HttpServer implements Filte
}
}
- if (endpoints.size() == 0) {
+ if (endpoints.size() == 0 && connector == null) {
throw new HadoopIllegalArgumentException("No endpoints specified");
}
if (hostName == null) {
- hostName = endpoints.get(0).getHost();
+ hostName = endpoints.size() == 0 ? connector.getHost() : endpoints.get(
+ 0).getHost();
}
if (this.conf == null) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java?rev=1553226&r1=1553225&r2=1553226&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java Mon Dec 23 22:19:12 2013
@@ -71,13 +71,6 @@ public class RetryPolicies {
/**
* <p>
- * Keep failing over forever
- * </p>
- */
- public static final RetryPolicy FAILOVER_FOREVER = new FailoverForever();
-
- /**
- * <p>
* Keep trying a limited number of times, waiting a fixed time between attempts,
* and then fail by re-throwing the exception.
* </p>
@@ -173,14 +166,6 @@ public class RetryPolicies {
return RetryAction.RETRY;
}
}
-
- static class FailoverForever implements RetryPolicy {
- @Override
- public RetryAction shouldRetry(Exception e, int retries, int failovers,
- boolean isIdempotentOrAtMostOnce) throws Exception {
- return RetryAction.FAILOVER_AND_RETRY;
- }
- }
/**
* Retry up to maxRetries.
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java?rev=1553226&r1=1553225&r2=1553226&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java Mon Dec 23 22:19:12 2013
@@ -24,10 +24,8 @@ import java.lang.management.MemoryUsage;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.lang.management.GarbageCollectorMXBean;
-import java.util.Map;
import java.util.List;
-
-import com.google.common.collect.Maps;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.log.metrics.EventCounter;
@@ -67,7 +65,8 @@ public class JvmMetrics implements Metri
ManagementFactory.getGarbageCollectorMXBeans();
final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
final String processName, sessionId;
- final Map<String, MetricsInfo[]> gcInfoCache = Maps.newHashMap();
+ final ConcurrentHashMap<String, MetricsInfo[]> gcInfoCache =
+ new ConcurrentHashMap<String, MetricsInfo[]>();
JvmMetrics(String processName, String sessionId) {
this.processName = processName;
@@ -123,13 +122,17 @@ public class JvmMetrics implements Metri
.addCounter(GcTimeMillis, timeMillis);
}
- private synchronized MetricsInfo[] getGcInfo(String gcName) {
+ private MetricsInfo[] getGcInfo(String gcName) {
MetricsInfo[] gcInfo = gcInfoCache.get(gcName);
if (gcInfo == null) {
gcInfo = new MetricsInfo[2];
- gcInfo[0] = Interns.info("GcCount"+ gcName, "GC Count for "+ gcName);
- gcInfo[1] = Interns.info("GcTimeMillis"+ gcName, "GC Time for "+ gcName);
- gcInfoCache.put(gcName, gcInfo);
+ gcInfo[0] = Interns.info("GcCount" + gcName, "GC Count for " + gcName);
+ gcInfo[1] = Interns
+ .info("GcTimeMillis" + gcName, "GC Time for " + gcName);
+ MetricsInfo[] previousGcInfo = gcInfoCache.putIfAbsent(gcName, gcInfo);
+ if (previousGcInfo != null) {
+ return previousGcInfo;
+ }
}
return gcInfo;
}
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1552466-1553224
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java?rev=1553226&r1=1553225&r2=1553226&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java Mon Dec 23 22:19:12 2013
@@ -258,6 +258,22 @@ public class TestHarFileSystemBasics {
0, expectedFileNames.size());
}
+ @Test
+ public void testMakeQualifiedPath() throws Exception {
+ // Construct a valid har file system path with authority that
+ // contains userinfo and port. The userinfo and port are useless
+ // in local fs uri. They are only used to verify har file system
+ // can correctly preserve the information for the underlying file system.
+ String harPathWithUserinfo = "har://file-user:passwd@localhost:80"
+ + harPath.toUri().getPath().toString();
+ Path path = new Path(harPathWithUserinfo);
+ Path qualifiedPath = path.getFileSystem(conf).makeQualified(path);
+ assertTrue(String.format(
+ "The qualified path (%s) did not match the expected path (%s).",
+ qualifiedPath.toString(), harPathWithUserinfo),
+ qualifiedPath.toString().equals(harPathWithUserinfo));
+ }
+
// ========== Negative:
@Test
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java?rev=1553226&r1=1553225&r2=1553226&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java Mon Dec 23 22:19:12 2013
@@ -66,6 +66,8 @@ import org.mockito.internal.util.reflect
import org.mortbay.jetty.Connector;
import org.mortbay.util.ajax.JSON;
+import static org.mockito.Mockito.*;
+
public class TestHttpServer extends HttpServerFunctionalTest {
static final Log LOG = LogFactory.getLog(TestHttpServer.class);
private static HttpServer server;
@@ -588,4 +590,15 @@ public class TestHttpServer extends Http
assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date"));
}
+ /**
+ * HTTPServer.Builder should proceed if a external connector is available.
+ */
+ @Test
+ public void testHttpServerBuilderWithExternalConnector() throws Exception {
+ Connector c = mock(Connector.class);
+ doReturn("localhost").when(c).getHost();
+ HttpServer s = new HttpServer.Builder().setName("test").setConnector(c)
+ .build();
+ s.stop();
+ }
}