You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cl...@apache.org on 2014/05/30 00:27:30 UTC
svn commit: r1598435 [1/2] - in
/hadoop/common/branches/fs-encryption/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/site/apt/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/
hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/ hadoop-...
Author: clamb
Date: Thu May 29 22:27:25 2014
New Revision: 1598435
URL: http://svn.apache.org/r1598435
Log:
merge from trunk r1598430
Added:
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
- copied unchanged from r1598430, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
Modified:
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1596816-1598430
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Hadoop HDFS over HTTP ${project.version} - Server Setup
- \[ {{{./index.html}Go Back}} \]
-
This page explains how to quickly setup HttpFS with Pseudo authentication
against a Hadoop cluster with Pseudo authentication.
@@ -159,5 +157,3 @@ $ keytool -genkey -alias tomcat -keyalg
<<<swebhdfs://>>> scheme. Make sure the JVM is picking up the truststore
containing the public key of the SSL certificate if using a self-signed
certificate.
-
- \[ {{{./index.html}Go Back}} \]
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Hadoop HDFS over HTTP ${project.version} - Using HTTP Tools
- \[ {{{./index.html}Go Back}} \]
-
* Security
Out of the box HttpFS supports both pseudo authentication and Kerberos HTTP
@@ -87,5 +85,3 @@ $ curl --negotiate -u foo -c ~/.httpfsau
+---+
$ curl -b ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=liststatus"
+---+
-
- \[ {{{./index.html}Go Back}} \]
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java Thu May 29 22:27:25 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.nfs.nfs3.respon
import org.apache.hadoop.oncrpc.XDR;
import org.apache.hadoop.oncrpc.security.VerifierNone;
import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
import org.jboss.netty.channel.Channel;
import com.google.common.annotations.VisibleForTesting;
@@ -136,7 +137,7 @@ class OpenFileCtx {
this.channel = channel;
this.xid = xid;
this.preOpAttr = preOpAttr;
- this.startTime = System.currentTimeMillis();
+ this.startTime = Time.monotonicNow();
}
@Override
@@ -158,11 +159,11 @@ class OpenFileCtx {
private Daemon dumpThread;
private void updateLastAccessTime() {
- lastAccessTime = System.currentTimeMillis();
+ lastAccessTime = Time.monotonicNow();
}
private boolean checkStreamTimeout(long streamTimeout) {
- return System.currentTimeMillis() - lastAccessTime > streamTimeout;
+ return Time.monotonicNow() - lastAccessTime > streamTimeout;
}
long getLastAccessTime() {
@@ -696,7 +697,7 @@ class OpenFileCtx {
+ " updating the mtime, then return success");
Nfs3FileAttributes postOpAttr = null;
try {
- dfsClient.setTimes(path, System.currentTimeMillis(), -1);
+ dfsClient.setTimes(path, Time.monotonicNow(), -1);
postOpAttr = Nfs3Utils.getFileAttr(dfsClient, path, iug);
} catch (IOException e) {
LOG.info("Got error when processing perfect overwrite, path=" + path
@@ -997,7 +998,7 @@ class OpenFileCtx {
if (LOG.isDebugEnabled()) {
LOG.debug("FileId: " + latestAttr.getFileId() + " Service time:"
- + (System.currentTimeMillis() - commit.getStartTime())
+ + (Time.monotonicNow() - commit.getStartTime())
+ "ms. Sent response for commit:" + commit);
}
entry = pendingCommits.firstEntry();
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java Thu May 29 22:27:25 2014
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -99,7 +100,7 @@ class OpenFileCtxCache {
LOG.warn("No eviction candidate. All streams have pending work.");
return null;
} else {
- long idleTime = System.currentTimeMillis()
+ long idleTime = Time.monotonicNow()
- idlest.getValue().getLastAccessTime();
if (idleTime < Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT) {
if (LOG.isDebugEnabled()) {
@@ -250,7 +251,7 @@ class OpenFileCtxCache {
// Check if it can sleep
try {
- long workedTime = System.currentTimeMillis() - lastWakeupTime;
+ long workedTime = Time.monotonicNow() - lastWakeupTime;
if (workedTime < rotation) {
if (LOG.isTraceEnabled()) {
LOG.trace("StreamMonitor can still have a sleep:"
@@ -258,7 +259,7 @@ class OpenFileCtxCache {
}
Thread.sleep(rotation - workedTime);
}
- lastWakeupTime = System.currentTimeMillis();
+ lastWakeupTime = Time.monotonicNow();
} catch (InterruptedException e) {
LOG.info("StreamMonitor got interrupted");
@@ -267,4 +268,4 @@ class OpenFileCtxCache {
}
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java Thu May 29 22:27:25 2014
@@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.client.Hdf
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.nfs.AccessPrivilege;
import org.apache.hadoop.nfs.NfsExports;
@@ -124,6 +123,7 @@ import org.apache.hadoop.oncrpc.security
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
@@ -151,13 +151,6 @@ public class RpcProgramNfs3 extends RpcP
private final NfsExports exports;
- /**
- * superUserClient should always impersonate HDFS file system owner to send
- * requests which requires supergroup privilege. This requires the same user
- * to start HDFS and NFS.
- */
- private final DFSClient superUserClient;
-
private final short replication;
private final long blockSize;
private final int bufferSize;
@@ -179,7 +172,6 @@ public class RpcProgramNfs3 extends RpcP
exports = NfsExports.getInstance(config);
writeManager = new WriteManager(iug, config);
clientCache = new DFSClientCache(config);
- superUserClient = new DFSClient(NameNode.getAddress(config), config);
replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
DFSConfigKeys.DFS_REPLICATION_DEFAULT);
blockSize = config.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
@@ -270,6 +262,17 @@ public class RpcProgramNfs3 extends RpcP
Nfs3FileAttributes attrs = null;
try {
attrs = writeManager.getFileAttr(dfsClient, handle, iug);
+ } catch (RemoteException r) {
+ LOG.warn("Exception ", r);
+ IOException io = r.unwrapRemoteException();
+ /**
+ * AuthorizationException can be thrown if the user can't be proxy'ed.
+ */
+ if (io instanceof AuthorizationException) {
+ return new GETATTR3Response(Nfs3Status.NFS3ERR_ACCES);
+ } else {
+ return new GETATTR3Response(Nfs3Status.NFS3ERR_IO);
+ }
} catch (IOException e) {
LOG.info("Can't get file attribute, fileId=" + handle.getFileId(), e);
response.setStatus(Nfs3Status.NFS3ERR_IO);
@@ -499,6 +502,17 @@ public class RpcProgramNfs3 extends RpcP
securityHandler.getUid(), securityHandler.getGid(), attrs);
return new ACCESS3Response(Nfs3Status.NFS3_OK, attrs, access);
+ } catch (RemoteException r) {
+ LOG.warn("Exception ", r);
+ IOException io = r.unwrapRemoteException();
+ /**
+ * AuthorizationException can be thrown if the user can't be proxy'ed.
+ */
+ if (io instanceof AuthorizationException) {
+ return new ACCESS3Response(Nfs3Status.NFS3ERR_ACCES);
+ } else {
+ return new ACCESS3Response(Nfs3Status.NFS3ERR_IO);
+ }
} catch (IOException e) {
LOG.warn("Exception ", e);
return new ACCESS3Response(Nfs3Status.NFS3ERR_IO);
@@ -1658,8 +1672,7 @@ public class RpcProgramNfs3 extends RpcP
}
try {
- // Use superUserClient to get file system status
- FsStatus fsStatus = superUserClient.getDiskStatus();
+ FsStatus fsStatus = dfsClient.getDiskStatus();
long totalBytes = fsStatus.getCapacity();
long freeBytes = fsStatus.getRemaining();
@@ -1680,6 +1693,17 @@ public class RpcProgramNfs3 extends RpcP
return new FSSTAT3Response(Nfs3Status.NFS3_OK, attrs, totalBytes,
freeBytes, freeBytes, maxFsObjects, maxFsObjects, maxFsObjects, 0);
+ } catch (RemoteException r) {
+ LOG.warn("Exception ", r);
+ IOException io = r.unwrapRemoteException();
+ /**
+ * AuthorizationException can be thrown if the user can't be proxy'ed.
+ */
+ if (io instanceof AuthorizationException) {
+ return new FSSTAT3Response(Nfs3Status.NFS3ERR_ACCES);
+ } else {
+ return new FSSTAT3Response(Nfs3Status.NFS3ERR_IO);
+ }
} catch (IOException e) {
LOG.warn("Exception ", e);
return new FSSTAT3Response(Nfs3Status.NFS3ERR_IO);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java Thu May 29 22:27:25 2014
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.net.InetAddress;
-import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -41,9 +40,8 @@ import org.apache.hadoop.nfs.nfs3.respon
import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response.EntryPlus3;
import org.apache.hadoop.oncrpc.XDR;
import org.apache.hadoop.oncrpc.security.SecurityHandler;
-import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
-import org.apache.hadoop.util.StringUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -67,10 +65,10 @@ public class TestReaddir {
public static void setup() throws Exception {
String currentUser = System.getProperty("user.name");
config.set(
- ProxyUsers.getProxySuperuserGroupConfKey(currentUser),
+ DefaultImpersonationProvider.getProxySuperuserGroupConfKey(currentUser),
"*");
config.set(
- ProxyUsers.getProxySuperuserIpConfKey(currentUser),
+ DefaultImpersonationProvider.getProxySuperuserIpConfKey(currentUser),
"*");
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java Thu May 29 22:27:25 2014
@@ -50,6 +50,7 @@ import org.apache.hadoop.nfs.nfs3.respon
import org.apache.hadoop.nfs.nfs3.response.READ3Response;
import org.apache.hadoop.oncrpc.XDR;
import org.apache.hadoop.oncrpc.security.SecurityHandler;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.jboss.netty.channel.Channel;
import org.junit.Assert;
@@ -288,10 +289,10 @@ public class TestWrites {
System.getProperty("user.name"));
String currentUser = System.getProperty("user.name");
config.set(
- ProxyUsers.getProxySuperuserGroupConfKey(currentUser),
+ DefaultImpersonationProvider.getProxySuperuserGroupConfKey(currentUser),
"*");
config.set(
- ProxyUsers.getProxySuperuserIpConfKey(currentUser),
+ DefaultImpersonationProvider.getProxySuperuserIpConfKey(currentUser),
"*");
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu May 29 22:27:25 2014
@@ -317,6 +317,9 @@ Trunk (Unreleased)
HDFS-6374. setXAttr should require the user to be the owner of the file
or directory (Charles Lamb via wang)
+ HDFS-6110 adding more slow action log in critical write path
+ (Liang Xie via stack)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -440,6 +443,17 @@ Release 2.5.0 - UNRELEASED
HDFS-6396. Remove support for ACL feature from INodeSymlink.
(Charles Lamb via wang)
+ HDFS-6435. Add support for specifying a static uid/gid mapping for the NFS
+ gateway. (atm via wang)
+
+ HDFS-6416. Use Time#monotonicNow in OpenFileCtx and OpenFileCtxCatch to
+ avoid system clock bugs (Abhiraj Butala via brandonli)
+
+ HDFS-6356. Fix typo in DatanodeLayoutVersion. (Tulasi G via wang)
+
+ HDFS-6447. balancer should timestamp the completion message.
+ (Juan Yu via wang).
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
@@ -583,6 +597,26 @@ Release 2.5.0 - UNRELEASED
HDFS-6423. Diskspace quota usage should be updated when appending data to
partial block. (jing9)
+ HDFS-6443. Fix MiniQJMHACluster related test failures. (Zesheng Wu via
+ Arpit Agarwal)
+
+ HDFS-6227. ShortCircuitCache#unref should purge ShortCircuitReplicas whose
+ streams have been closed by java interrupts. (Colin Patrick McCabe via jing9)
+
+ HDFS-6442. Fix TestEditLogAutoroll and TestStandbyCheckpoints failure
+ caused by port conficts. (Zesheng Wu via Arpit Agarwal)
+
+ HDFS-6448. BlockReaderLocalLegacy should set socket timeout based on
+ conf.socketTimeout (liangxie via cmccabe)
+
+ HDFS-6453. Use Time#monotonicNow to avoid system clock reset.
+ (Liang Xie via wang)
+
+ HDFS-6461. Use Time#monotonicNow to compute duration in DataNode#shutDown.
+ (James Thomas via wang)
+
+ HDFS-6462. NFS: fsstat request fails with the secure hdfs (brandonli)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -666,6 +700,9 @@ Release 2.4.1 - UNRELEASED
HDFS-6397. NN shows inconsistent value in deadnode count.
(Mohammad Kamrul Islam via kihwal)
+ HDFS-6411. nfs-hdfs-gateway mount raises I/O error and hangs when a
+ unauthorized user attempts to access it (brandonli)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1596816-1598430
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java Thu May 29 22:27:25 2014
@@ -116,7 +116,7 @@ public class Hdfs extends AbstractFileSy
@Override
public FileChecksum getFileChecksum(Path f)
throws IOException, UnresolvedLinkException {
- return dfs.getFileChecksum(getUriPath(f));
+ return dfs.getFileChecksum(getUriPath(f), Long.MAX_VALUE);
}
@Override
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java Thu May 29 22:27:25 2014
@@ -187,7 +187,7 @@ class BlockReaderLocalLegacy implements
userGroupInformation = UserGroupInformation.getCurrentUser();
}
pathinfo = getBlockPathInfo(userGroupInformation, blk, node,
- configuration, conf.hdfsTimeout, token,
+ configuration, conf.socketTimeout, token,
conf.connectToDnViaHostname);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Thu May 29 22:27:25 2014
@@ -276,6 +276,7 @@ public class DFSClient implements java.i
final int retryTimesForGetLastBlockLength;
final int retryIntervalForGetLastBlockLength;
final long datanodeRestartTimeout;
+ final long dfsclientSlowIoWarningThresholdMs;
final boolean useLegacyBlockReader;
final boolean useLegacyBlockReaderLocal;
@@ -430,6 +431,9 @@ public class DFSClient implements java.i
datanodeRestartTimeout = conf.getLong(
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000;
+ dfsclientSlowIoWarningThresholdMs = conf.getLong(
+ DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
+ DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
}
public boolean isUseLegacyBlockReaderLocal() {
@@ -1801,15 +1805,19 @@ public class DFSClient implements java.i
}
/**
- * Get the checksum of a file.
+ * Get the checksum of the whole file of a range of the file. Note that the
+ * range always starts from the beginning of the file.
* @param src The file path
+ * @param length The length of the range
* @return The checksum
* @see DistributedFileSystem#getFileChecksum(Path)
*/
- public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException {
+ public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length)
+ throws IOException {
checkOpen();
- return getFileChecksum(src, clientName, namenode, socketFactory,
- dfsClientConf.socketTimeout, getDataEncryptionKey(),
+ Preconditions.checkArgument(length >= 0);
+ return getFileChecksum(src, length, clientName, namenode,
+ socketFactory, dfsClientConf.socketTimeout, getDataEncryptionKey(),
dfsClientConf.connectToDnViaHostname);
}
@@ -1850,8 +1858,9 @@ public class DFSClient implements java.i
}
/**
- * Get the checksum of a file.
+ * Get the checksum of the whole file or a range of the file.
* @param src The file path
+ * @param length the length of the range, i.e., the range is [0, length]
* @param clientName the name of the client requesting the checksum.
* @param namenode the RPC proxy for the namenode
* @param socketFactory to create sockets to connect to DNs
@@ -1861,12 +1870,13 @@ public class DFSClient implements java.i
* @return The checksum
*/
private static MD5MD5CRC32FileChecksum getFileChecksum(String src,
- String clientName,
- ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout,
+ long length, String clientName, ClientProtocol namenode,
+ SocketFactory socketFactory, int socketTimeout,
DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
throws IOException {
- //get all block locations
- LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
+ //get block locations for the file range
+ LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0,
+ length);
if (null == blockLocations) {
throw new FileNotFoundException("File does not exist: " + src);
}
@@ -1878,10 +1888,11 @@ public class DFSClient implements java.i
boolean refetchBlocks = false;
int lastRetriedIndex = -1;
- //get block checksum for each block
- for(int i = 0; i < locatedblocks.size(); i++) {
+ // get block checksum for each block
+ long remaining = length;
+ for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) {
if (refetchBlocks) { // refetch to get fresh tokens
- blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
+ blockLocations = callGetBlockLocations(namenode, src, 0, length);
if (null == blockLocations) {
throw new FileNotFoundException("File does not exist: " + src);
}
@@ -1890,6 +1901,10 @@ public class DFSClient implements java.i
}
LocatedBlock lb = locatedblocks.get(i);
final ExtendedBlock block = lb.getBlock();
+ if (remaining < block.getNumBytes()) {
+ block.setNumBytes(remaining);
+ }
+ remaining -= block.getNumBytes();
final DatanodeInfo[] datanodes = lb.getLocations();
//try each datanode location of the block
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu May 29 22:27:25 2014
@@ -645,5 +645,12 @@ public class DFSConfigKeys extends Commo
public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned.
public static final String DFS_NFS_ALLOW_INSECURE_PORTS_KEY = "dfs.nfs.allow.insecure.ports";
public static final boolean DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT = true;
-
+
+ // Slow io warning log threshold settings for dfsclient and datanode.
+ public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
+ "dfs.client.slow.io.warning.threshold.ms";
+ public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
+ public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY =
+ "dfs.datanode.slow.io.warning.threshold.ms";
+ public static final long DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 300;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Thu May 29 22:27:25 2014
@@ -122,6 +122,7 @@ public class DFSOutputStream extends FSO
implements Syncable, CanSetDropBehind {
private static final int MAX_PACKETS = 80; // each packet 64K, total 5MB
private final DFSClient dfsClient;
+ private final long dfsclientSlowLogThresholdMs;
private Socket s;
// closed is accessed by different threads under different locks.
private volatile boolean closed = false;
@@ -788,11 +789,19 @@ public class DFSOutputStream extends FSO
// process responses from datanodes.
try {
// read an ack from the pipeline
+ long begin = Time.monotonicNow();
ack.readFields(blockReplyStream);
- if (DFSClient.LOG.isDebugEnabled()) {
+ long duration = Time.monotonicNow() - begin;
+ if (duration > dfsclientSlowLogThresholdMs
+ && ack.getSeqno() != Packet.HEART_BEAT_SEQNO) {
+ DFSClient.LOG
+ .warn("Slow ReadProcessor read fields took " + duration
+ + "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms); ack: "
+ + ack + ", targets: " + Arrays.asList(targets));
+ } else if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("DFSClient " + ack);
}
-
+
long seqno = ack.getSeqno();
// processes response status from datanodes.
for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) {
@@ -1570,6 +1579,8 @@ public class DFSOutputStream extends FSO
}
this.checksum = checksum;
+ this.dfsclientSlowLogThresholdMs =
+ dfsClient.getConf().dfsclientSlowIoWarningThresholdMs;
}
/** Construct a new output stream for creating a file. */
@@ -2001,6 +2012,7 @@ public class DFSOutputStream extends FSO
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("Waiting for ack for: " + seqno);
}
+ long begin = Time.monotonicNow();
try {
synchronized (dataQueue) {
while (!closed) {
@@ -2020,6 +2032,11 @@ public class DFSOutputStream extends FSO
checkClosed();
} catch (ClosedChannelException e) {
}
+ long duration = Time.monotonicNow() - begin;
+ if (duration > dfsclientSlowLogThresholdMs) {
+ DFSClient.LOG.warn("Slow waitForAckedSeqno took " + duration
+ + "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms)");
+ }
}
private synchronized void start() {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Thu May 29 22:27:25 2014
@@ -68,14 +68,12 @@ import org.apache.hadoop.hdfs.protocol.C
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
@@ -85,7 +83,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
@@ -1142,7 +1139,7 @@ public class DistributedFileSystem exten
@Override
public FileChecksum doCall(final Path p)
throws IOException, UnresolvedLinkException {
- return dfs.getFileChecksum(getPathName(p));
+ return dfs.getFileChecksum(getPathName(p), Long.MAX_VALUE);
}
@Override
@@ -1154,6 +1151,32 @@ public class DistributedFileSystem exten
}
@Override
+ public FileChecksum getFileChecksum(Path f, final long length)
+ throws IOException {
+ statistics.incrementReadOps(1);
+ Path absF = fixRelativePart(f);
+ return new FileSystemLinkResolver<FileChecksum>() {
+ @Override
+ public FileChecksum doCall(final Path p)
+ throws IOException, UnresolvedLinkException {
+ return dfs.getFileChecksum(getPathName(p), length);
+ }
+
+ @Override
+ public FileChecksum next(final FileSystem fs, final Path p)
+ throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ return ((DistributedFileSystem) fs).getFileChecksum(p, length);
+ } else {
+ throw new UnsupportedFileSystemException(
+ "getFileChecksum(Path, long) is not supported by "
+ + fs.getClass().getSimpleName());
+ }
+ }
+ }.resolve(this, absF);
+ }
+
+ @Override
public void setPermission(Path p, final FsPermission permission
) throws IOException {
statistics.incrementWriteOps(1);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java Thu May 29 22:27:25 2014
@@ -1552,6 +1552,7 @@ public class Balancer {
System.out.println(e + ". Exiting ...");
return ReturnStatus.INTERRUPTED.code;
} finally {
+ System.out.format("%-24s ", DateFormat.getDateTimeInstance().format(new Date()));
System.out.println("Balancing took " + time2Str(Time.now()-startTime));
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java Thu May 29 22:27:25 2014
@@ -69,7 +69,7 @@ class BlockReceiver implements Closeable
@VisibleForTesting
static long CACHE_DROP_LAG_BYTES = 8 * 1024 * 1024;
-
+ private final long datanodeSlowLogThresholdMs;
private DataInputStream in = null; // from where data are read
private DataChecksum clientChecksum; // checksum used by client
private DataChecksum diskChecksum; // checksum we write to disk
@@ -140,7 +140,7 @@ class BlockReceiver implements Closeable
this.isDatanode = clientname.length() == 0;
this.isClient = !this.isDatanode;
this.restartBudget = datanode.getDnConf().restartReplicaExpiry;
-
+ this.datanodeSlowLogThresholdMs = datanode.getDnConf().datanodeSlowIoWarningThresholdMs;
//for datanode, we have
//1: clientName.length() == 0, and
//2: stage == null or PIPELINE_SETUP_CREATE
@@ -335,6 +335,7 @@ class BlockReceiver implements Closeable
*/
void flushOrSync(boolean isSync) throws IOException {
long flushTotalNanos = 0;
+ long begin = Time.monotonicNow();
if (checksumOut != null) {
long flushStartNanos = System.nanoTime();
checksumOut.flush();
@@ -363,6 +364,12 @@ class BlockReceiver implements Closeable
datanode.metrics.incrFsyncCount();
}
}
+ long duration = Time.monotonicNow() - begin;
+ if (duration > datanodeSlowLogThresholdMs) {
+ LOG.warn("Slow flushOrSync took " + duration + "ms (threshold="
+ + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos="
+ + flushTotalNanos + "ns");
+ }
}
/**
@@ -488,8 +495,14 @@ class BlockReceiver implements Closeable
//First write the packet to the mirror:
if (mirrorOut != null && !mirrorError) {
try {
+ long begin = Time.monotonicNow();
packetReceiver.mirrorPacketTo(mirrorOut);
mirrorOut.flush();
+ long duration = Time.monotonicNow() - begin;
+ if (duration > datanodeSlowLogThresholdMs) {
+ LOG.warn("Slow BlockReceiver write packet to mirror took " + duration
+ + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
+ }
} catch (IOException e) {
handleMirrorOutError(e);
}
@@ -572,7 +585,13 @@ class BlockReceiver implements Closeable
int numBytesToDisk = (int)(offsetInBlock-onDiskLen);
// Write data to disk.
+ long begin = Time.monotonicNow();
out.write(dataBuf.array(), startByteToDisk, numBytesToDisk);
+ long duration = Time.monotonicNow() - begin;
+ if (duration > datanodeSlowLogThresholdMs) {
+ LOG.warn("Slow BlockReceiver write data to disk cost:" + duration
+ + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
+ }
// If this is a partial chunk, then verify that this is the only
// chunk in the packet. Calculate new crc for this chunk.
@@ -638,6 +657,7 @@ class BlockReceiver implements Closeable
try {
if (outFd != null &&
offsetInBlock > lastCacheManagementOffset + CACHE_DROP_LAG_BYTES) {
+ long begin = Time.monotonicNow();
//
// For SYNC_FILE_RANGE_WRITE, we want to sync from
// lastCacheManagementOffset to a position "two windows ago"
@@ -670,6 +690,11 @@ class BlockReceiver implements Closeable
NativeIO.POSIX.POSIX_FADV_DONTNEED);
}
lastCacheManagementOffset = offsetInBlock;
+ long duration = Time.monotonicNow() - begin;
+ if (duration > datanodeSlowLogThresholdMs) {
+ LOG.warn("Slow manageWriterOsCache took " + duration
+ + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
+ }
}
} catch (Throwable t) {
LOG.warn("Error managing cache for writer of block " + block, t);
@@ -1299,9 +1324,15 @@ class BlockReceiver implements Closeable
replicaInfo.setBytesAcked(offsetInBlock);
}
// send my ack back to upstream datanode
+ long begin = Time.monotonicNow();
replyAck.write(upstreamOut);
upstreamOut.flush();
- if (LOG.isDebugEnabled()) {
+ long duration = Time.monotonicNow() - begin;
+ if (duration > datanodeSlowLogThresholdMs) {
+ LOG.warn("Slow PacketResponder send ack to upstream took " + duration
+ + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + myString
+ + ", replyAck=" + replyAck);
+ } else if (LOG.isDebugEnabled()) {
LOG.debug(myString + ", replyAck=" + replyAck);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java Thu May 29 22:27:25 2014
@@ -79,6 +79,8 @@ public class DNConf {
final long deleteReportInterval;
final long initialBlockReportDelay;
final long cacheReportInterval;
+ final long dfsclientSlowIoWarningThresholdMs;
+ final long datanodeSlowIoWarningThresholdMs;
final int writePacketSize;
final String minimumNameNodeVersion;
@@ -129,7 +131,14 @@ public class DNConf {
DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT);
this.cacheReportInterval = conf.getLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY,
DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT);
-
+
+ this.dfsclientSlowIoWarningThresholdMs = conf.getLong(
+ DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
+ DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
+ this.datanodeSlowIoWarningThresholdMs = conf.getLong(
+ DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY,
+ DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
+
long initBRDelay = conf.getLong(
DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT) * 1000L;
@@ -168,7 +177,7 @@ public class DNConf {
DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY,
DFS_DATANODE_RESTART_REPLICA_EXPIRY_DEFAULT) * 1000L;
}
-
+
// We get minimumNameNodeVersion via a method so it can be mocked out in tests.
String getMinimumNameNodeVersion() {
return this.minimumNameNodeVersion;
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Thu May 29 22:27:25 2014
@@ -1221,7 +1221,7 @@ public class DataNode extends Configured
}
// Record the time of initial notification
- long timeNotified = Time.now();
+ long timeNotified = Time.monotonicNow();
if (localDataXceiverServer != null) {
((DataXceiverServer) this.localDataXceiverServer.getRunnable()).kill();
@@ -1253,8 +1253,9 @@ public class DataNode extends Configured
while (true) {
// When shutting down for restart, wait 2.5 seconds before forcing
// termination of receiver threads.
- if (!this.shutdownForUpgrade ||
- (this.shutdownForUpgrade && (Time.now() - timeNotified > 2500))) {
+ if (!this.shutdownForUpgrade ||
+ (this.shutdownForUpgrade && (Time.monotonicNow() - timeNotified
+ > 2500))) {
this.threadGroup.interrupt();
}
LOG.info("Waiting for threadgroup to exit, active threads is " +
@@ -2581,7 +2582,7 @@ public class DataNode extends Configured
return;
}
synchronized(checkDiskErrorMutex) {
- lastDiskErrorCheck = System.currentTimeMillis();
+ lastDiskErrorCheck = Time.monotonicNow();
}
}
try {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java Thu May 29 22:27:25 2014
@@ -62,7 +62,7 @@ public class DataNodeLayoutVersion {
* </ul>
*/
public static enum Feature implements LayoutFeature {
- FIRST_LAYOUT(-55, -53, "First datenode layout", false);
+ FIRST_LAYOUT(-55, -53, "First datanode layout", false);
private final FeatureInfo info;
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Thu May 29 22:27:25 2014
@@ -42,6 +42,7 @@ import java.net.Socket;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.nio.channels.ClosedChannelException;
+import java.security.MessageDigest;
import java.util.Arrays;
import org.apache.commons.logging.Log;
@@ -83,6 +84,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
+import com.google.common.base.Preconditions;
import com.google.common.net.InetAddresses;
import com.google.protobuf.ByteString;
@@ -802,7 +804,44 @@ class DataXceiver extends Receiver imple
IOUtils.closeStream(out);
}
}
-
+
+ private MD5Hash calcPartialBlockChecksum(ExtendedBlock block,
+ long requestLength, DataChecksum checksum, DataInputStream checksumIn)
+ throws IOException {
+ final int bytesPerCRC = checksum.getBytesPerChecksum();
+ final int csize = checksum.getChecksumSize();
+ final byte[] buffer = new byte[4*1024];
+ MessageDigest digester = MD5Hash.getDigester();
+
+ long remaining = requestLength / bytesPerCRC * csize;
+ for (int toDigest = 0; remaining > 0; remaining -= toDigest) {
+ toDigest = checksumIn.read(buffer, 0,
+ (int) Math.min(remaining, buffer.length));
+ if (toDigest < 0) {
+ break;
+ }
+ digester.update(buffer, 0, toDigest);
+ }
+
+ int partialLength = (int) (requestLength % bytesPerCRC);
+ if (partialLength > 0) {
+ byte[] buf = new byte[partialLength];
+ final InputStream blockIn = datanode.data.getBlockInputStream(block,
+ requestLength - partialLength);
+ try {
+ // Get the CRC of the partialLength.
+ IOUtils.readFully(blockIn, buf, 0, partialLength);
+ } finally {
+ IOUtils.closeStream(blockIn);
+ }
+ checksum.update(buf, 0, partialLength);
+ byte[] partialCrc = new byte[csize];
+ checksum.writeValue(partialCrc, 0, true);
+ digester.update(partialCrc);
+ }
+ return new MD5Hash(digester.digest());
+ }
+
@Override
public void blockChecksum(final ExtendedBlock block,
final Token<BlockTokenIdentifier> blockToken) throws IOException {
@@ -810,25 +849,32 @@ class DataXceiver extends Receiver imple
getOutputStream());
checkAccess(out, true, block, blockToken,
Op.BLOCK_CHECKSUM, BlockTokenSecretManager.AccessMode.READ);
- updateCurrentThreadName("Reading metadata for block " + block);
- final LengthInputStream metadataIn =
- datanode.data.getMetaDataInputStream(block);
- final DataInputStream checksumIn = new DataInputStream(new BufferedInputStream(
- metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE));
+ // client side now can specify a range of the block for checksum
+ long requestLength = block.getNumBytes();
+ Preconditions.checkArgument(requestLength >= 0);
+ long visibleLength = datanode.data.getReplicaVisibleLength(block);
+ boolean partialBlk = requestLength < visibleLength;
+ updateCurrentThreadName("Reading metadata for block " + block);
+ final LengthInputStream metadataIn = datanode.data
+ .getMetaDataInputStream(block);
+
+ final DataInputStream checksumIn = new DataInputStream(
+ new BufferedInputStream(metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE));
updateCurrentThreadName("Getting checksum for block " + block);
try {
//read metadata file
- final BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn);
- final DataChecksum checksum = header.getChecksum();
+ final BlockMetadataHeader header = BlockMetadataHeader
+ .readHeader(checksumIn);
+ final DataChecksum checksum = header.getChecksum();
+ final int csize = checksum.getChecksumSize();
final int bytesPerCRC = checksum.getBytesPerChecksum();
- final long crcPerBlock = checksum.getChecksumSize() > 0
- ? (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize())/checksum.getChecksumSize()
- : 0;
-
- //compute block checksum
- final MD5Hash md5 = MD5Hash.digest(checksumIn);
+ final long crcPerBlock = csize <= 0 ? 0 :
+ (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize()) / csize;
+ final MD5Hash md5 = partialBlk && crcPerBlock > 0 ?
+ calcPartialBlockChecksum(block, requestLength, checksum, checksumIn)
+ : MD5Hash.digest(checksumIn);
if (LOG.isDebugEnabled()) {
LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC
+ ", crcPerBlock=" + crcPerBlock + ", md5=" + md5);
@@ -841,8 +887,7 @@ class DataXceiver extends Receiver imple
.setBytesPerCrc(bytesPerCRC)
.setCrcPerBlock(crcPerBlock)
.setMd5(ByteString.copyFrom(md5.getDigest()))
- .setCrcType(PBHelper.convert(checksum.getChecksumType()))
- )
+ .setCrcType(PBHelper.convert(checksum.getChecksumType())))
.build()
.writeDelimitedTo(out);
out.flush();
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java Thu May 29 22:27:25 2014
@@ -23,6 +23,7 @@ import java.util.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
class FsVolumeList {
@@ -97,7 +98,7 @@ class FsVolumeList {
}
void getAllVolumesMap(final String bpid, final ReplicaMap volumeMap) throws IOException {
- long totalStartTime = System.currentTimeMillis();
+ long totalStartTime = Time.monotonicNow();
final List<IOException> exceptions = Collections.synchronizedList(
new ArrayList<IOException>());
List<Thread> replicaAddingThreads = new ArrayList<Thread>();
@@ -107,9 +108,9 @@ class FsVolumeList {
try {
FsDatasetImpl.LOG.info("Adding replicas to map for block pool " +
bpid + " on volume " + v + "...");
- long startTime = System.currentTimeMillis();
+ long startTime = Time.monotonicNow();
v.getVolumeMap(bpid, volumeMap);
- long timeTaken = System.currentTimeMillis() - startTime;
+ long timeTaken = Time.monotonicNow() - startTime;
FsDatasetImpl.LOG.info("Time to add replicas to map for block pool"
+ " " + bpid + " on volume " + v + ": " + timeTaken + "ms");
} catch (IOException ioe) {
@@ -132,7 +133,7 @@ class FsVolumeList {
if (!exceptions.isEmpty()) {
throw exceptions.get(0);
}
- long totalTimeTaken = System.currentTimeMillis() - totalStartTime;
+ long totalTimeTaken = Time.monotonicNow() - totalStartTime;
FsDatasetImpl.LOG.info("Total time to add all replicas to map: "
+ totalTimeTaken + "ms");
}
@@ -141,9 +142,9 @@ class FsVolumeList {
throws IOException {
FsDatasetImpl.LOG.info("Adding replicas to map for block pool " + bpid +
" on volume " + volume + "...");
- long startTime = System.currentTimeMillis();
+ long startTime = Time.monotonicNow();
volume.getVolumeMap(bpid, volumeMap);
- long timeTaken = System.currentTimeMillis() - startTime;
+ long timeTaken = Time.monotonicNow() - startTime;
FsDatasetImpl.LOG.info("Time to add replicas to map for block pool " + bpid +
" on volume " + volume + ": " + timeTaken + "ms");
}
@@ -195,7 +196,7 @@ class FsVolumeList {
void addBlockPool(final String bpid, final Configuration conf) throws IOException {
- long totalStartTime = System.currentTimeMillis();
+ long totalStartTime = Time.monotonicNow();
final List<IOException> exceptions = Collections.synchronizedList(
new ArrayList<IOException>());
@@ -206,9 +207,9 @@ class FsVolumeList {
try {
FsDatasetImpl.LOG.info("Scanning block pool " + bpid +
" on volume " + v + "...");
- long startTime = System.currentTimeMillis();
+ long startTime = Time.monotonicNow();
v.addBlockPool(bpid, conf);
- long timeTaken = System.currentTimeMillis() - startTime;
+ long timeTaken = Time.monotonicNow() - startTime;
FsDatasetImpl.LOG.info("Time taken to scan block pool " + bpid +
" on " + v + ": " + timeTaken + "ms");
} catch (IOException ioe) {
@@ -232,7 +233,7 @@ class FsVolumeList {
throw exceptions.get(0);
}
- long totalTimeTaken = System.currentTimeMillis() - totalStartTime;
+ long totalTimeTaken = Time.monotonicNow() - totalStartTime;
FsDatasetImpl.LOG.info("Total time to scan all replicas for block pool " +
bpid + ": " + totalTimeTaken + "ms");
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Thu May 29 22:27:25 2014
@@ -74,7 +74,6 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -452,7 +451,7 @@ public class DatanodeWebHdfsMethods {
MD5MD5CRC32FileChecksum checksum = null;
DFSClient dfsclient = newDfsClient(nnId, conf);
try {
- checksum = dfsclient.getFileChecksum(fullpath);
+ checksum = dfsclient.getFileChecksum(fullpath, Long.MAX_VALUE);
dfsclient.close();
dfsclient = null;
} finally {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java Thu May 29 22:27:25 2014
@@ -59,6 +59,7 @@ import org.apache.hadoop.hdfs.util.MD5Fi
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressorStream;
+import org.apache.hadoop.util.Time;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -160,13 +161,13 @@ public final class FSImageFormatProtobuf
}
void load(File file) throws IOException {
- long start = System.currentTimeMillis();
+ long start = Time.monotonicNow();
imgDigest = MD5FileUtils.computeMd5ForFile(file);
RandomAccessFile raFile = new RandomAccessFile(file, "r");
FileInputStream fin = new FileInputStream(file);
try {
loadInternal(raFile, fin);
- long end = System.currentTimeMillis();
+ long end = Time.monotonicNow();
LOG.info("Loaded FSImage in " + (end - start) / 1000 + " seconds.");
} finally {
fin.close();
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java Thu May 29 22:27:25 2014
@@ -159,7 +159,7 @@ public class TransferFsImage {
}
}
- final long milliTime = System.currentTimeMillis();
+ final long milliTime = Time.monotonicNow();
String tmpFileName = NNStorage.getTemporaryEditsFileName(
log.getStartTxId(), log.getEndTxId(), milliTime);
List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS,
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java Thu May 29 22:27:25 2014
@@ -437,11 +437,22 @@ public class ShortCircuitCache implement
void unref(ShortCircuitReplica replica) {
lock.lock();
try {
- // If the replica is stale, but we haven't purged it yet, let's do that.
- // It would be a shame to evict a non-stale replica so that we could put
- // a stale one into the cache.
- if ((!replica.purged) && replica.isStale()) {
- purge(replica);
+ // If the replica is stale or unusable, but we haven't purged it yet,
+ // let's do that. It would be a shame to evict a non-stale replica so
+ // that we could put a stale or unusable one into the cache.
+ if (!replica.purged) {
+ String purgeReason = null;
+ if (!replica.getDataStream().getChannel().isOpen()) {
+ purgeReason = "purging replica because its data channel is closed.";
+ } else if (!replica.getMetaStream().getChannel().isOpen()) {
+ purgeReason = "purging replica because its meta channel is closed.";
+ } else if (replica.isStale()) {
+ purgeReason = "purging replica because it is stale.";
+ }
+ if (purgeReason != null) {
+ LOG.debug(this + ": " + purgeReason);
+ purge(replica);
+ }
}
String addedString = "";
boolean shouldTrimEvictionMaps = false;
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1596816-1598430
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Thu May 29 22:27:25 2014
@@ -1930,4 +1930,22 @@
</description>
</property>
+<property>
+ <name>dfs.client.slow.io.warning.threshold.ms</name>
+ <value>30000</value>
+ <description>The threshold in milliseconds at which we will log a slow
+ io warning in a dfsclient. By default, this parameter is set to 30000
+ milliseconds (30 seconds).
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.slow.io.warning.threshold.ms</name>
+ <value>300</value>
+ <description>The threshold in milliseconds at which we will log a slow
+ io warning in a datanode. By default, this parameter is set to 300
+ milliseconds.
+ </description>
+</property>
+
</configuration>
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1596816-1598430
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1596816-1598430
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1596816-1598430
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Centralized Cache Management in HDFS
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=2|toDepth=4}
* {Overview}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Extended Attributes in HDFS
- \[ {{{../../index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=2|toDepth=4}
* {Overview}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm Thu May 29 22:27:25 2014
@@ -19,8 +19,6 @@
HDFS NFS Gateway
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* {Overview}
@@ -338,8 +336,21 @@ HDFS NFS Gateway
The system administrator must ensure that the user on NFS client host has the same
name and UID as that on the NFS gateway host. This is usually not a problem if
the same user management system (e.g., LDAP/NIS) is used to create and deploy users on
- HDFS nodes and NFS client node. In case the user account is created manually in different hosts, one might need to
+ HDFS nodes and NFS client node. In case the user account is created manually on different hosts, one might need to
modify UID (e.g., do "usermod -u 123 myusername") on either NFS client or NFS gateway host
in order to make it the same on both sides. More technical details of RPC AUTH_UNIX can be found
in {{{http://tools.ietf.org/html/rfc1057}RPC specification}}.
+ Optionally, the system administrator can configure a custom static mapping
+ file in the event one wishes to access the HDFS NFS Gateway from a system with
+ a completely disparate set of UIDs/GIDs. By default this file is located at
+ "/etc/nfs.map", but a custom location can be configured by setting the
+ "dfs.nfs.static.mapping.file" property to the path of the static mapping file.
+ The format of the static mapping file is similar to what is described in the
+ exports(5) manual page, but roughly it is:
+
+-------------------------
+# Mapping for clients accessing the NFS gateway
+uid 10 100 # Map the remote UID 10 the local UID 100
+gid 11 101 # Map the remote GID 11 to the local GID 101
+-------------------------
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
ViewFs Guide
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* {Introduction}
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1596816-1598430
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java Thu May 29 22:27:25 2014
@@ -28,12 +28,15 @@ import static org.hamcrest.CoreMatchers.
import java.io.File;
import java.io.IOException;
+import java.nio.channels.ClosedByInterruptException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -409,4 +412,121 @@ public class TestBlockReaderFactory {
getDomainSocketWatcher().isClosed());
cluster.shutdown();
}
+
+ /**
+ * When an InterruptedException is sent to a thread calling
+ * FileChannel#read, the FileChannel is immediately closed and the
+ * thread gets an exception. This effectively means that we might have
+ * someone asynchronously calling close() on the file descriptors we use
+ * in BlockReaderLocal. So when unreferencing a ShortCircuitReplica in
+ * ShortCircuitCache#unref, we should check if the FileChannel objects
+ * are still open. If not, we should purge the replica to avoid giving
+ * it out to any future readers.
+ *
+ * This is a regression test for HDFS-6227: Short circuit read failed
+ * due to ClosedChannelException.
+ *
+ * Note that you may still get ClosedChannelException errors if two threads
+ * are reading from the same replica and an InterruptedException is delivered
+ * to one of them.
+ */
+ @Test(timeout=120000)
+ public void testPurgingClosedReplicas() throws Exception {
+ BlockReaderTestUtil.enableBlockReaderFactoryTracing();
+ final AtomicInteger replicasCreated = new AtomicInteger(0);
+ final AtomicBoolean testFailed = new AtomicBoolean(false);
+ DFSInputStream.tcpReadsDisabledForTesting = true;
+ BlockReaderFactory.createShortCircuitReplicaInfoCallback =
+ new ShortCircuitCache.ShortCircuitReplicaCreator() {
+ @Override
+ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
+ replicasCreated.incrementAndGet();
+ return null;
+ }
+ };
+ TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
+ Configuration conf = createShortCircuitConf(
+ "testPurgingClosedReplicas", sockDir);
+ final MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 4095;
+ final int SEED = 0xFADE0;
+ final DistributedFileSystem fs =
+ (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf);
+ DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+
+ final Semaphore sem = new Semaphore(0);
+ final List<LocatedBlock> locatedBlocks =
+ cluster.getNameNode().getRpcServer().getBlockLocations(
+ TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks();
+ final LocatedBlock lblock = locatedBlocks.get(0); // first block
+ final byte[] buf = new byte[TEST_FILE_LEN];
+ Runnable readerRunnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ BlockReader blockReader = null;
+ try {
+ blockReader = BlockReaderTestUtil.
+ getBlockReader(cluster, lblock, 0, TEST_FILE_LEN);
+ sem.release();
+ try {
+ blockReader.readAll(buf, 0, TEST_FILE_LEN);
+ } finally {
+ sem.acquireUninterruptibly();
+ }
+ } catch (ClosedByInterruptException e) {
+ LOG.info("got the expected ClosedByInterruptException", e);
+ sem.release();
+ break;
+ } finally {
+ if (blockReader != null) blockReader.close();
+ }
+ LOG.info("read another " + TEST_FILE_LEN + " bytes.");
+ }
+ } catch (Throwable t) {
+ LOG.error("getBlockReader failure", t);
+ testFailed.set(true);
+ sem.release();
+ }
+ }
+ };
+ Thread thread = new Thread(readerRunnable);
+ thread.start();
+
+ // While the thread is reading, send it interrupts.
+ // These should trigger a ClosedChannelException.
+ while (thread.isAlive()) {
+ sem.acquireUninterruptibly();
+ thread.interrupt();
+ sem.release();
+ }
+ Assert.assertFalse(testFailed.get());
+
+ // We should be able to read from the file without
+ // getting a ClosedChannelException.
+ BlockReader blockReader = null;
+ try {
+ blockReader = BlockReaderTestUtil.
+ getBlockReader(cluster, lblock, 0, TEST_FILE_LEN);
+ blockReader.readFully(buf, 0, TEST_FILE_LEN);
+ } finally {
+ if (blockReader != null) blockReader.close();
+ }
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ Assert.assertTrue(Arrays.equals(buf, expected));
+
+ // Another ShortCircuitReplica object should have been created.
+ Assert.assertEquals(2, replicasCreated.get());
+
+ dfs.close();
+ cluster.shutdown();
+ sockDir.close();
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java Thu May 29 22:27:25 2014
@@ -22,8 +22,12 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import java.io.IOException;
+import java.net.BindException;
import java.net.URI;
+import java.util.Random;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -37,14 +41,13 @@ public class MiniQJMHACluster {
private MiniDFSCluster cluster;
private MiniJournalCluster journalCluster;
private final Configuration conf;
+ private static final Log LOG = LogFactory.getLog(MiniQJMHACluster.class);
public static final String NAMESERVICE = "ns1";
private static final String NN1 = "nn1";
private static final String NN2 = "nn2";
- private static final int NN1_IPC_PORT = 10000;
- private static final int NN1_INFO_PORT = 10001;
- private static final int NN2_IPC_PORT = 10002;
- private static final int NN2_INFO_PORT = 10003;
+ private static final Random RANDOM = new Random();
+ private int basePort = 10000;
public static class Builder {
private final Configuration conf;
@@ -69,51 +72,62 @@ public class MiniQJMHACluster {
}
}
- public static MiniDFSNNTopology createDefaultTopology() {
+ public static MiniDFSNNTopology createDefaultTopology(int basePort) {
return new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf(NAMESERVICE).addNN(
- new MiniDFSNNTopology.NNConf("nn1").setIpcPort(NN1_IPC_PORT)
- .setHttpPort(NN1_INFO_PORT)).addNN(
- new MiniDFSNNTopology.NNConf("nn2").setIpcPort(NN2_IPC_PORT)
- .setHttpPort(NN2_INFO_PORT)));
+ new MiniDFSNNTopology.NNConf("nn1").setIpcPort(basePort)
+ .setHttpPort(basePort + 1)).addNN(
+ new MiniDFSNNTopology.NNConf("nn2").setIpcPort(basePort + 2)
+ .setHttpPort(basePort + 3)));
}
-
+
private MiniQJMHACluster(Builder builder) throws IOException {
this.conf = builder.conf;
- // start 3 journal nodes
- journalCluster = new MiniJournalCluster.Builder(conf).format(true)
- .build();
- URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE);
-
- // start cluster with 2 NameNodes
- MiniDFSNNTopology topology = createDefaultTopology();
-
- initHAConf(journalURI, builder.conf);
-
- // First start up the NNs just to format the namespace. The MinIDFSCluster
- // has no way to just format the NameNodes without also starting them.
- cluster = builder.dfsBuilder.nnTopology(topology)
- .manageNameDfsSharedDirs(false).build();
- cluster.waitActive();
- cluster.shutdown();
-
- // initialize the journal nodes
- Configuration confNN0 = cluster.getConfiguration(0);
- NameNode.initializeSharedEdits(confNN0, true);
-
- cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt);
- cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt);
-
- // restart the cluster
- cluster.restartNameNodes();
+ int retryCount = 0;
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ // start 3 journal nodes
+ journalCluster = new MiniJournalCluster.Builder(conf).format(true)
+ .build();
+ URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE);
+
+ // start cluster with 2 NameNodes
+ MiniDFSNNTopology topology = createDefaultTopology(basePort);
+
+ initHAConf(journalURI, builder.conf);
+
+ // First start up the NNs just to format the namespace. The MinIDFSCluster
+ // has no way to just format the NameNodes without also starting them.
+ cluster = builder.dfsBuilder.nnTopology(topology)
+ .manageNameDfsSharedDirs(false).build();
+ cluster.waitActive();
+ cluster.shutdown();
+
+ // initialize the journal nodes
+ Configuration confNN0 = cluster.getConfiguration(0);
+ NameNode.initializeSharedEdits(confNN0, true);
+
+ cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt);
+ cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt);
+
+ // restart the cluster
+ cluster.restartNameNodes();
+ ++retryCount;
+ break;
+ } catch (BindException e) {
+ LOG.info("MiniQJMHACluster port conflicts, retried " +
+ retryCount + " times");
+ }
+ }
}
private Configuration initHAConf(URI journalURI, Configuration conf) {
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
journalURI.toString());
- String address1 = "127.0.0.1:" + NN1_IPC_PORT;
- String address2 = "127.0.0.1:" + NN2_IPC_PORT;
+ String address1 = "127.0.0.1:" + basePort;
+ String address2 = "127.0.0.1:" + (basePort + 2);
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
NAMESERVICE, NN1), address1);
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,