You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2013/09/20 01:42:22 UTC
svn commit: r1524865 [2/5] - in
/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project: hadoop-hdfs-nfs/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/
hadoop-hdfs-nfs/src/t...
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Sep 19 23:42:10 2013
@@ -6,6 +6,9 @@ Trunk (Unreleased)
HDFS-3034. Remove the deprecated DFSOutputStream.sync() method. (szetszwo)
+ HDFS-5079. Cleaning up NNHAStatusHeartbeat.State from
+ DatanodeProtocolProtos. (Tao Luo via shv)
+
NEW FEATURES
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
@@ -117,9 +120,6 @@ Trunk (Unreleased)
HDFS-4904. Remove JournalService. (Arpit Agarwal via cnauroth)
- HDFS-5004. Add additional JMX bean for NameNode status data
- (Trevor Lorimer via cos)
-
HDFS-4953. Enable HDFS local reads via mmap.
(Colin Patrick McCabe via wang).
@@ -259,6 +259,22 @@ Release 2.3.0 - UNRELEASED
HDFS-4817. Make HDFS advisory caching configurable on a per-file basis.
(Colin Patrick McCabe)
+ HDFS-5004. Add additional JMX bean for NameNode status data
+ (Trevor Lorimer via cos)
+
+ HDFS-5068. Convert NNThroughputBenchmark to a Tool to allow generic options.
+ (shv)
+
+ HDFS-4994. Audit log getContentSummary() calls. (Robert Parker via kihwal)
+
+ HDFS-5144. Document time unit to NameNodeMetrics. (Akira Ajisaka via
+ suresh)
+
+ HDFS-4491. Parallel testing HDFS. (Andrey Klochkov via cnauroth)
+
+ HDFS-4879. Add "blocked ArrayList" collection to avoid CMS full GCs
+ (Todd Lipcon via Colin Patrick McCabe)
+
OPTIMIZATIONS
BUG FIXES
@@ -273,6 +289,12 @@ Release 2.3.0 - UNRELEASED
HDFS-4816. transitionToActive blocks if the SBN is doing checkpoint image
transfer. (Andrew Wang)
+ HDFS-5164. deleteSnapshot should check if OperationCategory.WRITE is
+ possible before taking write lock. (Colin Patrick McCabe)
+
+ HDFS-5170. BlockPlacementPolicyDefault uses the wrong classname when
+ alerting to enable debug logging. (Andrew Wang)
+
Release 2.1.1-beta - UNRELEASED
INCOMPATIBLE CHANGES
@@ -285,6 +307,35 @@ Release 2.1.1-beta - UNRELEASED
HDFS-4763 Add script changes/utility for starting NFS gateway (brandonli)
+ HDFS-5076 Add MXBean methods to query NN's transaction information and
+ JournalNode's journal status. (jing9)
+
+ HDFS-5104 Support dotdot name in NFS LOOKUP operation (brandonli)
+
+ HDFS-5107 Fix array copy error in Readdir and Readdirplus responses
+ (brandonli)
+
+ HDFS-5110 Change FSDataOutputStream to HdfsDataOutputStream for opened
+ streams to fix type cast error. (brandonli)
+
+ HDFS-5069 Include hadoop-nfs and hadoop-hdfs-nfs into hadoop dist for
+ NFS deployment (brandonli)
+
+ HDFS-4947 Add NFS server export table to control export by hostname or
+ IP range (Jing Zhao via brandonli)
+
+ HDFS-5078 Support file append in NFSv3 gateway to enable data streaming
+ to HDFS (brandonli)
+
+ HDFS-5136 MNT EXPORT should give the full group list which can mount the
+ exports (brandonli)
+
+ HDFS-5118. Provide testing support for DFSClient to drop RPC responses.
+ (jing9)
+
+ HDFS-5085. Refactor o.a.h.nfs to support different types of
+ authentications. (jing9)
+
IMPROVEMENTS
HDFS-4513. Clarify in the WebHDFS REST API that all JSON respsonses may
@@ -302,6 +353,25 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5047. Supress logging of full stack trace of quota and lease
exceptions. (Robert Parker via kihwal)
+ HDFS-2933. Improve DataNode Web UI Index Page. (Vivek Ganesan via
+ Arpit Agarwal)
+
+ HDFS-5111. Remove duplicated error message for snapshot commands when
+ processing invalid arguments. (jing9)
+
+ HDFS-5045. Add more unit tests for retry cache to cover all AtMostOnce
+ methods. (jing9)
+
+ HDFS-3245. Add metrics and web UI for cluster version summary. (Ravi
+ Prakash via kihwal)
+
+ HDFS-5128. Allow multiple net interfaces to be used with HA namenode RPC
+ server. (kihwal)
+
+ HDFS-5150. Allow per NN SPN for internal SPNEGO. (kihwal)
+
+ HDFS-4680. Audit logging of delegation tokens for MR tracing. (Andrew Wang)
+
OPTIMIZATIONS
BUG FIXES
@@ -318,7 +388,59 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5091. Support for spnego keytab separate from the JournalNode keytab
for secure HA. (jing9)
-Release 2.1.0-beta - 2013-08-06
+ HDFS-5051. nn fails to download checkpointed image from snn in some
+ setups. (Vinay and suresh via suresh)
+
+ HDFS-4898. BlockPlacementPolicyWithNodeGroup.chooseRemoteRack() fails to
+ properly fallback to local rack. (szetszwo)
+
+ HDFS-4632. globStatus using backslash for escaping does not work on Windows.
+ (Chuan Liu via cnauroth)
+
+ HDFS-5093. TestGlobPaths should re-use the MiniDFSCluster to avoid failure
+ on Windows. (Chuan Liu via cnauroth)
+
+ HDFS-5080. BootstrapStandby not working with QJM when the existing NN is
+ active. (jing9)
+
+ HDFS-5099. Namenode#copyEditLogSegmentsToSharedDir should close
+ EditLogInputStreams upon finishing. (Chuan Liu via cnauroth)
+
+ HDFS-2994. If lease soft limit is recovered successfully
+ the append can fail. (Tao Luo via shv)
+
+ HDFS-5100. TestNamenodeRetryCache fails on Windows due to incorrect cleanup.
+ (Chuan Liu via cnauroth)
+
+ HDFS-5103. TestDirectoryScanner fails on Windows. (Chuan Liu via cnauroth)
+
+ HDFS-5102. Snapshot names should not be allowed to contain slash characters.
+ (jing9)
+
+ HDFS-5105. TestFsck fails on Windows. (Chuan Liu via arp)
+
+ HDFS-5106. TestDatanodeBlockScanner fails on Windows due to incorrect path
+ format. (Chuan Liu via cnauroth)
+
+ HDFS-4594. WebHDFS open sets Content-Length header to what is specified by
+ length parameter rather than how much data is actually returned. (cnauroth)
+
+ HDFS-5124. DelegationTokenSecretManager#retrievePassword can cause deadlock
+ in NameNode. (Daryn Sharp via jing9)
+
+ HDFS-5132. Deadlock in NameNode between SafeModeMonitor#run and
+ DatanodeManager#handleHeartbeat. (kihwal)
+
+ HDFS-5077. NPE in FSNamesystem.commitBlockSynchronization().
+ (Plamen Jeliazkov via shv)
+
+ HDFS-5140. Too many safemode monitor threads being created in the standby
+ namenode causing it to fail with out of memory error. (jing9)
+
+ HDFS-5159. Secondary NameNode fails to checkpoint if error occurs
+ downloading edits on first checkpoint. (atm)
+
+Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
@@ -3367,6 +3489,9 @@ Release 0.23.10 - UNRELEASED
HDFS-4998. TestUnderReplicatedBlocks fails intermittently (kihwal)
+ HDFS-4329. DFSShell issues with directories with spaces in name (Cristina
+ L. Abad via jeagles)
+
Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/pom.xml Thu Sep 19 23:42:10 2013
@@ -63,7 +63,6 @@ http://maven.apache.org/xsd/maven-4.0.0.
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <version>3.4.2</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -701,5 +700,44 @@ http://maven.apache.org/xsd/maven-4.0.0.
</plugins>
</build>
</profile>
+ <profile>
+ <id>parallel-tests</id>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-parallel-tests-dirs</id>
+ <phase>test-compile</phase>
+ <configuration>
+ <target>
+ <exec executable="sh">
+ <arg value="-c"/>
+ <arg value="for i in {1..${testsThreadCount}}; do mkdir -p ${test.build.data}/$i; mkdir -p ${hadoop.tmp.dir}/$i; done"/>
+ </exec>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -DminiClusterDedicatedDirs=true</argLine>
+ <systemPropertyVariables>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
Propchange: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1514105-1522706
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Thu Sep 19 23:42:10 2013
@@ -27,6 +27,9 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
@@ -44,9 +47,6 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT;
@@ -100,6 +100,7 @@ import org.apache.hadoop.fs.MD5MD5CRC32G
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.VolumeId;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -114,13 +115,13 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
@@ -145,6 +146,7 @@ import org.apache.hadoop.io.EnumSetWrita
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
@@ -488,7 +490,11 @@ public class DFSClient implements java.i
/**
* Create a new DFSClient connected to the given nameNodeUri or rpcNamenode.
- * Exactly one of nameNodeUri or rpcNamenode must be null.
+ * If HA is enabled and a positive value is set for
+ * {@link DFSConfigKeys#DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY} in the
+ * configuration, the DFSClient will use {@link LossyRetryInvocationHandler}
+ * as its RetryInvocationHandler. Otherwise one of nameNodeUri or rpcNamenode
+ * must be null.
*/
@VisibleForTesting
public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode,
@@ -512,7 +518,20 @@ public class DFSClient implements java.i
this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
- if (rpcNamenode != null) {
+ int numResponseToDrop = conf.getInt(
+ DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
+ DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
+ if (numResponseToDrop > 0) {
+ // This case is used for testing.
+ LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY
+ + " is set to " + numResponseToDrop
+ + ", this hacked client will proactively drop responses");
+ NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = NameNodeProxies
+ .createProxyWithLossyRetryHandler(conf, nameNodeUri,
+ ClientProtocol.class, numResponseToDrop);
+ this.dtService = proxyInfo.getDelegationTokenService();
+ this.namenode = proxyInfo.getProxy();
+ } else if (rpcNamenode != null) {
// This case is used for testing.
Preconditions.checkArgument(nameNodeUri == null);
this.namenode = rpcNamenode;
@@ -552,7 +571,7 @@ public class DFSClient implements java.i
new CachingStrategy(writeDropBehind, readahead);
this.mmapManager = MMAP_MANAGER_FACTORY.get(conf);
}
-
+
/**
* Return the socket addresses to use with each configured
* local interface. Local interfaces may be specified by IP
@@ -634,7 +653,8 @@ public class DFSClient implements java.i
return dfsClientConf.hdfsTimeout;
}
- String getClientName() {
+ @VisibleForTesting
+ public String getClientName() {
return clientName;
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Sep 19 23:42:10 2013
@@ -110,7 +110,9 @@ public class DFSConfigKeys extends Commo
public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address";
public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT;
public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address";
+ public static final String DFS_NAMENODE_RPC_BIND_HOST_KEY = "dfs.namenode.rpc-bind-host";
public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY = "dfs.namenode.servicerpc-address";
+ public static final String DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY = "dfs.namenode.servicerpc-bind-host";
public static final String DFS_NAMENODE_MAX_OBJECTS_KEY = "dfs.namenode.max.objects";
public static final long DFS_NAMENODE_MAX_OBJECTS_DEFAULT = 0;
public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_KEY = "dfs.namenode.safemode.extension";
@@ -278,6 +280,8 @@ public class DFSConfigKeys extends Commo
public static final String DFS_CLIENT_LOCAL_INTERFACES = "dfs.client.local.interfaces";
public static final String DFS_NAMENODE_AUDIT_LOGGERS_KEY = "dfs.namenode.audit.loggers";
public static final String DFS_NAMENODE_DEFAULT_AUDIT_LOGGER_NAME = "default";
+ public static final String DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_KEY = "dfs.namenode.audit.log.token.tracking.id";
+ public static final boolean DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT = false;
// Much code in hdfs is not yet updated to use these keys.
public static final String DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY = "dfs.client.block.write.locateFollowingBlock.retries";
@@ -516,6 +520,12 @@ public class DFSConfigKeys extends Commo
public static final long DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_DEFAULT = 600000; // 10 minutes
public static final String DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_KEY = "dfs.namenode.retrycache.heap.percent";
public static final float DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_DEFAULT = 0.03f;
+
+ // The number of NN response dropped by client proactively in each RPC call.
+ // For testing NN retry cache, we can set this property with positive value.
+ public static final String DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY = "dfs.client.test.drop.namenode.response.number";
+ public static final int DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT = 0;
+
// Hidden configuration undocumented in hdfs-site. xml
// Timeout to wait for block receiver and responder thread to stop
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Thu Sep 19 23:42:10 2013
@@ -70,6 +70,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
@@ -204,13 +205,20 @@ public class DFSUtil {
String[] components = StringUtils.split(src, '/');
for (int i = 0; i < components.length; i++) {
String element = components[i];
- if (element.equals("..") ||
- element.equals(".") ||
+ if (element.equals(".") ||
(element.indexOf(":") >= 0) ||
(element.indexOf("/") >= 0)) {
return false;
}
-
+ // ".." is allowed in path starting with /.reserved/.inodes
+ if (element.equals("..")) {
+ if (components.length > 4
+ && components[1].equals(FSDirectory.DOT_RESERVED_STRING)
+ && components[2].equals(FSDirectory.DOT_INODES_STRING)) {
+ continue;
+ }
+ return false;
+ }
// The string may start or end with a /, but not have
// "//" in the middle.
if (element.isEmpty() && i != components.length - 1 &&
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Thu Sep 19 23:42:10 2013
@@ -51,7 +51,7 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher;
-import org.apache.hadoop.hdfs.web.URLUtils;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
@@ -86,6 +86,8 @@ public class HftpFileSystem extends File
HttpURLConnection.setFollowRedirects(true);
}
+ URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY;
+
public static final Text TOKEN_KIND = new Text("HFTP delegation");
protected UserGroupInformation ugi;
@@ -331,8 +333,8 @@ public class HftpFileSystem extends File
throws IOException {
query = addDelegationTokenParam(query);
final URL url = getNamenodeURL(path, query);
- final HttpURLConnection connection =
- (HttpURLConnection)URLUtils.openConnection(url);
+ final HttpURLConnection connection;
+ connection = (HttpURLConnection)connectionFactory.openConnection(url);
connection.setRequestMethod("GET");
connection.connect();
return connection;
@@ -352,12 +354,14 @@ public class HftpFileSystem extends File
}
static class RangeHeaderUrlOpener extends ByteRangeInputStream.URLOpener {
+ URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_CONNECTION_FACTORY;
+
RangeHeaderUrlOpener(final URL url) {
super(url);
}
protected HttpURLConnection openConnection() throws IOException {
- return (HttpURLConnection)URLUtils.openConnection(url);
+ return (HttpURLConnection)connectionFactory.openConnection(url);
}
/** Use HTTP Range header for specifying offset. */
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java Thu Sep 19 23:42:10 2013
@@ -40,7 +40,6 @@ import javax.net.ssl.X509TrustManager;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.web.URLUtils;
import org.apache.hadoop.util.Time;
/**
@@ -154,7 +153,8 @@ public class HsftpFileSystem extends Hft
query = addDelegationTokenParam(query);
final URL url = new URL("https", nnUri.getHost(),
nnUri.getPort(), path + '?' + query);
- HttpsURLConnection conn = (HttpsURLConnection)URLUtils.openConnection(url);
+ HttpsURLConnection conn;
+ conn = (HttpsURLConnection)connectionFactory.openConnection(url);
// bypass hostname verification
conn.setHostnameVerifier(new DummyHostnameVerifier());
conn.setRequestMethod("GET");
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java Thu Sep 19 23:42:10 2013
@@ -17,10 +17,18 @@
*/
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
import java.io.IOException;
import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Proxy;
import java.net.InetSocketAddress;
import java.net.URI;
import java.util.HashMap;
@@ -48,6 +56,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider;
import org.apache.hadoop.io.retry.FailoverProxyProvider;
+import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
@@ -144,6 +153,61 @@ public class NameNodeProxies {
return new ProxyAndInfo<T>(proxy, dtService);
}
}
+
+ /**
+ * Generate a dummy namenode proxy instance that utilizes our hacked
+ * {@link LossyRetryInvocationHandler}. Proxy instance generated using this
+ * method will proactively drop RPC responses. Currently this method only
+ * support HA setup. IllegalStateException will be thrown if the given
+ * configuration is not for HA.
+ *
+ * @param config the configuration containing the required IPC
+ * properties, client failover configurations, etc.
+ * @param nameNodeUri the URI pointing either to a specific NameNode
+ * or to a logical nameservice.
+ * @param xface the IPC interface which should be created
+ * @param numResponseToDrop The number of responses to drop for each RPC call
+ * @return an object containing both the proxy and the associated
+ * delegation token service it corresponds to
+ * @throws IOException if there is an error creating the proxy
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> ProxyAndInfo<T> createProxyWithLossyRetryHandler(
+ Configuration config, URI nameNodeUri, Class<T> xface,
+ int numResponseToDrop) throws IOException {
+ Preconditions.checkArgument(numResponseToDrop > 0);
+ Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
+ getFailoverProxyProviderClass(config, nameNodeUri, xface);
+ if (failoverProxyProviderClass != null) { // HA case
+ FailoverProxyProvider<T> failoverProxyProvider =
+ createFailoverProxyProvider(config, failoverProxyProviderClass,
+ xface, nameNodeUri);
+ int delay = config.getInt(
+ DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
+ DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
+ int maxCap = config.getInt(
+ DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
+ DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
+ int maxFailoverAttempts = config.getInt(
+ DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
+ DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
+ InvocationHandler dummyHandler = new LossyRetryInvocationHandler<T>(
+ numResponseToDrop, failoverProxyProvider,
+ RetryPolicies.failoverOnNetworkException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL,
+ Math.max(numResponseToDrop + 1, maxFailoverAttempts), delay,
+ maxCap));
+
+ T proxy = (T) Proxy.newProxyInstance(
+ failoverProxyProvider.getInterface().getClassLoader(),
+ new Class[] { xface }, dummyHandler);
+ Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
+ return new ProxyAndInfo<T>(proxy, dtService);
+ } else {
+ throw new IllegalStateException("Currently creating proxy using " +
+ "LossyRetryInvocationHandler requires NN HA setup");
+ }
+ }
/**
* Creates an explicitly non-HA-enabled proxy object. Most of the time you
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Thu Sep 19 23:42:10 2013
@@ -49,6 +49,7 @@ public class DatanodeInfo extends Datano
private long lastUpdate;
private int xceiverCount;
private String location = NetworkTopology.DEFAULT_RACK;
+ private String softwareVersion;
// Datanode administrative states
public enum AdminStates {
@@ -458,4 +459,12 @@ public class DatanodeInfo extends Datano
// by DatanodeID
return (this == obj) || super.equals(obj);
}
+
+ public String getSoftwareVersion() {
+ return softwareVersion;
+ }
+
+ public void setSoftwareVersion(String softwareVersion) {
+ this.softwareVersion = softwareVersion;
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Thu Sep 19 23:42:10 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -1330,10 +1331,10 @@ public class PBHelper {
NNHAStatusHeartbeatProto.newBuilder();
switch (hb.getState()) {
case ACTIVE:
- builder.setState(NNHAStatusHeartbeatProto.State.ACTIVE);
+ builder.setState(HAServiceProtocolProtos.HAServiceStateProto.ACTIVE);
break;
case STANDBY:
- builder.setState(NNHAStatusHeartbeatProto.State.STANDBY);
+ builder.setState(HAServiceProtocolProtos.HAServiceStateProto.STANDBY);
break;
default:
throw new IllegalArgumentException("Unexpected NNHAStatusHeartbeat.State:" +
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java Thu Sep 19 23:42:10 2013
@@ -109,7 +109,7 @@ interface AsyncLogger {
* Fetch the list of edit logs available on the remote node.
*/
public ListenableFuture<RemoteEditLogManifest> getEditLogManifest(
- long fromTxnId, boolean forReading);
+ long fromTxnId, boolean forReading, boolean inProgressOk);
/**
* Prepare recovery. See the HDFS-3077 design document for details.
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java Thu Sep 19 23:42:10 2013
@@ -262,14 +262,14 @@ class AsyncLoggerSet {
return QuorumCall.create(calls);
}
- public QuorumCall<AsyncLogger, RemoteEditLogManifest>
- getEditLogManifest(long fromTxnId, boolean forReading) {
+ public QuorumCall<AsyncLogger, RemoteEditLogManifest> getEditLogManifest(
+ long fromTxnId, boolean forReading, boolean inProgressOk) {
Map<AsyncLogger,
ListenableFuture<RemoteEditLogManifest>> calls
= Maps.newHashMap();
for (AsyncLogger logger : loggers) {
ListenableFuture<RemoteEditLogManifest> future =
- logger.getEditLogManifest(fromTxnId, forReading);
+ logger.getEditLogManifest(fromTxnId, forReading, inProgressOk);
calls.put(logger, future);
}
return QuorumCall.create(calls);
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java Thu Sep 19 23:42:10 2013
@@ -519,12 +519,13 @@ public class IPCLoggerChannel implements
@Override
public ListenableFuture<RemoteEditLogManifest> getEditLogManifest(
- final long fromTxnId, final boolean forReading) {
+ final long fromTxnId, final boolean forReading,
+ final boolean inProgressOk) {
return executor.submit(new Callable<RemoteEditLogManifest>() {
@Override
public RemoteEditLogManifest call() throws IOException {
GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest(
- journalId, fromTxnId, forReading);
+ journalId, fromTxnId, forReading, inProgressOk);
// Update the http port, since we need this to build URLs to any of the
// returned logs.
httpPort = ret.getHttpPort();
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java Thu Sep 19 23:42:10 2013
@@ -456,7 +456,7 @@ public class QuorumJournalManager implem
long fromTxnId, boolean inProgressOk, boolean forReading) throws IOException {
QuorumCall<AsyncLogger, RemoteEditLogManifest> q =
- loggers.getEditLogManifest(fromTxnId, forReading);
+ loggers.getEditLogManifest(fromTxnId, forReading, inProgressOk);
Map<AsyncLogger, RemoteEditLogManifest> resps =
loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
"selectInputStreams");
@@ -480,8 +480,7 @@ public class QuorumJournalManager implem
allStreams.add(elis);
}
}
- JournalSet.chainAndMakeRedundantStreams(
- streams, allStreams, fromTxnId, inProgressOk);
+ JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}
@Override
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java Thu Sep 19 23:42:10 2013
@@ -125,10 +125,13 @@ public interface QJournalProtocol {
* @param sinceTxId the first transaction which the client cares about
* @param forReading whether or not the caller intends to read from the edit
* logs
+ * @param inProgressOk whether or not to check the in-progress edit log
+ * segment
* @return a list of edit log segments since the given transaction ID.
*/
- public GetEditLogManifestResponseProto getEditLogManifest(
- String jid, long sinceTxId, boolean forReading) throws IOException;
+ public GetEditLogManifestResponseProto getEditLogManifest(String jid,
+ long sinceTxId, boolean forReading, boolean inProgressOk)
+ throws IOException;
/**
* Begin the recovery process for a given segment. See the HDFS-3077
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java Thu Sep 19 23:42:10 2013
@@ -203,7 +203,8 @@ public class QJournalProtocolServerSideT
return impl.getEditLogManifest(
request.getJid().getIdentifier(),
request.getSinceTxId(),
- request.getForReading());
+ request.getForReading(),
+ request.getInProgressOk());
} catch (IOException e) {
throw new ServiceException(e);
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java Thu Sep 19 23:42:10 2013
@@ -228,13 +228,15 @@ public class QJournalProtocolTranslatorP
@Override
public GetEditLogManifestResponseProto getEditLogManifest(String jid,
- long sinceTxId, boolean forReading) throws IOException {
+ long sinceTxId, boolean forReading, boolean inProgressOk)
+ throws IOException {
try {
return rpcProxy.getEditLogManifest(NULL_CONTROLLER,
GetEditLogManifestRequestProto.newBuilder()
.setJid(convertJournalId(jid))
.setSinceTxId(sinceTxId)
.setForReading(forReading)
+ .setInProgressOk(inProgressOk)
.build());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java Thu Sep 19 23:42:10 2013
@@ -25,10 +25,9 @@ import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.net.URL;
import java.security.PrivilegedExceptionAction;
+import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,8 +35,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.qjournal.protocol.JournalNotFormattedException;
-import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
import org.apache.hadoop.hdfs.qjournal.protocol.JournalOutOfSyncException;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PersistedRecoveryPaxosData;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
@@ -50,6 +49,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.JournalManager;
import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.util.AtomicFileOutputStream;
import org.apache.hadoop.hdfs.util.BestEffortLongFile;
@@ -630,14 +630,31 @@ class Journal implements Closeable {
* @see QJournalProtocol#getEditLogManifest(String, long)
*/
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
- boolean forReading) throws IOException {
+ boolean forReading, boolean inProgressOk) throws IOException {
// No need to checkRequest() here - anyone may ask for the list
// of segments.
checkFormatted();
- RemoteEditLogManifest manifest = new RemoteEditLogManifest(
- fjm.getRemoteEditLogs(sinceTxId, forReading));
- return manifest;
+ // if this is for reading, ignore the in-progress editlog segment
+ inProgressOk = forReading ? false : inProgressOk;
+ List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, forReading,
+ inProgressOk);
+
+ if (inProgressOk) {
+ RemoteEditLog log = null;
+ for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
+ log = iter.next();
+ if (log.isInProgress()) {
+ iter.remove();
+ break;
+ }
+ }
+ if (log != null && log.isInProgress()) {
+ logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId()));
+ }
+ }
+
+ return new RemoteEditLogManifest(logs);
}
/**
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java Thu Sep 19 23:42:10 2013
@@ -18,8 +18,10 @@
package org.apache.hadoop.hdfs.qjournal.server;
import java.io.File;
+import java.io.FileFilter;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -34,11 +36,13 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.mortbay.util.ajax.JSON;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
@@ -51,7 +55,7 @@ import com.google.common.collect.Maps;
* in the quorum protocol.
*/
@InterfaceAudience.Private
-public class JournalNode implements Tool, Configurable {
+public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
public static final Log LOG = LogFactory.getLog(JournalNode.class);
private Configuration conf;
private JournalNodeRpcServer rpcServer;
@@ -128,6 +132,8 @@ public class JournalNode implements Tool
SecurityUtil.login(conf, DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY,
DFSConfigKeys.DFS_JOURNALNODE_USER_NAME_KEY, socAddr.getHostName());
+ registerJNMXBean();
+
httpServer = new JournalNodeHttpServer(conf, this);
httpServer.start();
@@ -208,6 +214,50 @@ public class JournalNode implements Tool
return new File(new File(dir), jid);
}
+ @Override // JournalNodeMXBean
+ public String getJournalsStatus() {
+ // jid:{Formatted:True/False}
+ Map<String, Map<String, String>> status =
+ new HashMap<String, Map<String, String>>();
+ synchronized (this) {
+ for (Map.Entry<String, Journal> entry : journalsById.entrySet()) {
+ Map<String, String> jMap = new HashMap<String, String>();
+ jMap.put("Formatted", Boolean.toString(entry.getValue().isFormatted()));
+ status.put(entry.getKey(), jMap);
+ }
+ }
+
+ // It is possible that some journals have been formatted before, while the
+ // corresponding journals are not in journalsById yet (because of restarting
+ // JN, e.g.). For simplicity, let's just assume a journal is formatted if
+ // there is a directory for it. We can also call analyzeStorage method for
+ // these directories if necessary.
+ // Also note that we do not need to check localDir here since
+ // validateAndCreateJournalDir has been called before we register the
+ // MXBean.
+ File[] journalDirs = localDir.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File file) {
+ return file.isDirectory();
+ }
+ });
+ for (File journalDir : journalDirs) {
+ String jid = journalDir.getName();
+ if (!status.containsKey(jid)) {
+ Map<String, String> jMap = new HashMap<String, String>();
+ jMap.put("Formatted", "true");
+ status.put(jid, jMap);
+ }
+ }
+ return JSON.toString(status);
+ }
+
+ /**
+ * Register JournalNodeMXBean
+ */
+ private void registerJNMXBean() {
+ MBeans.register("JournalNode", "JournalNodeInfo", this);
+ }
private class ErrorReporter implements StorageErrorReporter {
@Override
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java Thu Sep 19 23:42:10 2013
@@ -36,8 +36,8 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
/**
* Encapsulates the HTTP server started by the Journal Service.
@@ -69,16 +69,15 @@ public class JournalNodeHttpServer {
bindAddr.getHostName()));
int tmpInfoPort = bindAddr.getPort();
- httpServer = new HttpServer("journal", bindAddr.getHostName(),
- tmpInfoPort, tmpInfoPort == 0, conf, new AccessControlList(conf
- .get(DFS_ADMIN, " "))) {
- {
- if (UserGroupInformation.isSecurityEnabled()) {
- initSpnego(conf, DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY,
- DFSUtil.getSpnegoKeytabKey(conf, DFS_JOURNALNODE_KEYTAB_FILE_KEY));
- }
- }
- };
+ httpServer = new HttpServer.Builder().setName("journal")
+ .setBindAddress(bindAddr.getHostName()).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf).setACL(
+ new AccessControlList(conf.get(DFS_ADMIN, " ")))
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(
+ DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(DFSUtil.getSpnegoKeytabKey(conf,
+ DFS_JOURNALNODE_KEYTAB_FILE_KEY)).build();
httpServer.setAttribute(JN_ATTRIBUTE_KEY, localJournalNode);
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
httpServer.addInternalServlet("getJournal", "/getJournal",
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java Thu Sep 19 23:42:10 2013
@@ -175,10 +175,11 @@ class JournalNodeRpcServer implements QJ
@Override
public GetEditLogManifestResponseProto getEditLogManifest(String jid,
- long sinceTxId, boolean forReading) throws IOException {
+ long sinceTxId, boolean forReading, boolean inProgressOk)
+ throws IOException {
RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid)
- .getEditLogManifest(sinceTxId, forReading);
+ .getEditLogManifest(sinceTxId, forReading, inProgressOk);
return GetEditLogManifestResponseProto.newBuilder()
.setManifest(PBHelper.convert(manifest))
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java Thu Sep 19 23:42:10 2013
@@ -58,6 +58,15 @@ public class DelegationTokenSecretManage
.getLog(DelegationTokenSecretManager.class);
private final FSNamesystem namesystem;
+
+ public DelegationTokenSecretManager(long delegationKeyUpdateInterval,
+ long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
+ long delegationTokenRemoverScanInterval, FSNamesystem namesystem) {
+ this(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
+ delegationTokenRenewInterval, delegationTokenRemoverScanInterval, false,
+ namesystem);
+ }
+
/**
* Create a secret manager
* @param delegationKeyUpdateInterval the number of seconds for rolling new
@@ -67,13 +76,16 @@ public class DelegationTokenSecretManage
* @param delegationTokenRenewInterval how often the tokens must be renewed
* @param delegationTokenRemoverScanInterval how often the tokens are scanned
* for expired tokens
+ * @param storeTokenTrackingId whether to store the token's tracking id
*/
public DelegationTokenSecretManager(long delegationKeyUpdateInterval,
long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
- long delegationTokenRemoverScanInterval, FSNamesystem namesystem) {
+ long delegationTokenRemoverScanInterval, boolean storeTokenTrackingId,
+ FSNamesystem namesystem) {
super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
this.namesystem = namesystem;
+ this.storeTokenTrackingId = storeTokenTrackingId;
}
@Override //SecretManager
@@ -81,17 +93,28 @@ public class DelegationTokenSecretManage
return new DelegationTokenIdentifier();
}
- @Override //SecretManager
- public void checkAvailableForRead() throws StandbyException {
- namesystem.checkOperation(OperationCategory.READ);
- namesystem.readLock();
+ @Override
+ public byte[] retrievePassword(
+ DelegationTokenIdentifier identifier) throws InvalidToken {
try {
+ // this check introduces inconsistency in the authentication to a
+ // HA standby NN. non-token auths are allowed into the namespace which
+ // decides whether to throw a StandbyException. tokens are a bit
+ // different in that a standby may be behind and thus not yet know
+ // of all tokens issued by the active NN. the following check does
+ // not allow ANY token auth, however it should allow known tokens in
namesystem.checkOperation(OperationCategory.READ);
- } finally {
- namesystem.readUnlock();
+ } catch (StandbyException se) {
+ // FIXME: this is a hack to get around changing method signatures by
+ // tunneling a non-InvalidToken exception as the cause which the
+ // RPC server will unwrap before returning to the client
+ InvalidToken wrappedStandby = new InvalidToken("StandbyException");
+ wrappedStandby.initCause(se);
+ throw wrappedStandby;
}
+ return super.retrievePassword(identifier);
}
-
+
/**
* Returns expiry time of a token given its identifier.
*
@@ -173,7 +196,7 @@ public class DelegationTokenSecretManage
}
if (currentTokens.get(identifier) == null) {
currentTokens.put(identifier, new DelegationTokenInformation(expiryTime,
- password));
+ password, getTrackingIdIfEnabled(identifier)));
} else {
throw new IOException(
"Same delegation token being added twice; invalid entry in fsimage or editlogs");
@@ -212,7 +235,7 @@ public class DelegationTokenSecretManage
byte[] password = createPassword(identifier.getBytes(), allKeys
.get(keyId).getKey());
currentTokens.put(identifier, new DelegationTokenInformation(expiryTime,
- password));
+ password, getTrackingIdIfEnabled(identifier)));
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Thu Sep 19 23:42:10 2013
@@ -55,7 +55,7 @@ public class BlockPlacementPolicyDefault
private static final String enableDebugLogging =
"For more information, please enable DEBUG log level on "
- + LOG.getClass().getName();
+ + BlockPlacementPolicy.class.getName();
protected boolean considerLoad;
private boolean preferLocalNode = true;
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java Thu Sep 19 23:42:10 2013
@@ -169,16 +169,17 @@ public class BlockPlacementPolicyWithNod
long blocksize, int maxReplicasPerRack, List<DatanodeDescriptor> results,
boolean avoidStaleNodes) throws NotEnoughReplicasException {
int oldNumOfReplicas = results.size();
- // randomly choose one node from remote racks
+
+ final String rackLocation = NetworkTopology.getFirstHalf(
+ localMachine.getNetworkLocation());
try {
- chooseRandom(
- numOfReplicas,
- "~" + NetworkTopology.getFirstHalf(localMachine.getNetworkLocation()),
- excludedNodes, blocksize, maxReplicasPerRack, results,
- avoidStaleNodes);
+ // randomly choose from remote racks
+ chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize,
+ maxReplicasPerRack, results, avoidStaleNodes);
} catch (NotEnoughReplicasException e) {
+ // fall back to the local rack
chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas),
- localMachine.getNetworkLocation(), excludedNodes, blocksize,
+ rackLocation, excludedNodes, blocksize,
maxReplicasPerRack, results, avoidStaleNodes);
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Thu Sep 19 23:42:10 2013
@@ -26,6 +26,7 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableMap;
@@ -64,7 +65,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException;
import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.util.CyclicIteration;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.CachedDNSToSwitchMapping;
import org.apache.hadoop.net.DNSToSwitchMapping;
@@ -165,6 +165,14 @@ public class DatanodeManager {
* according to the NetworkTopology.
*/
private boolean hasClusterEverBeenMultiRack = false;
+
+ /**
+ * The number of datanodes for each software version. This list should change
+ * during rolling upgrades.
+ * Software version -> Number of datanodes with this version
+ */
+ private HashMap<String, Integer> datanodesSoftwareVersions =
+ new HashMap<String, Integer>(4, 0.75f);
DatanodeManager(final BlockManager blockManager, final Namesystem namesystem,
final Configuration conf) throws IOException {
@@ -456,6 +464,7 @@ public class DatanodeManager {
heartbeatManager.removeDatanode(nodeInfo);
blockManager.removeBlocksAssociatedTo(nodeInfo);
networktopology.remove(nodeInfo);
+ decrementVersionCount(nodeInfo.getSoftwareVersion());
if (LOG.isDebugEnabled()) {
LOG.debug("remove datanode " + nodeInfo);
@@ -538,6 +547,61 @@ public class DatanodeManager {
}
}
+ private void incrementVersionCount(String version) {
+ if (version == null) {
+ return;
+ }
+ synchronized(datanodeMap) {
+ Integer count = this.datanodesSoftwareVersions.get(version);
+ count = count == null ? 1 : count + 1;
+ this.datanodesSoftwareVersions.put(version, count);
+ }
+ }
+
+ private void decrementVersionCount(String version) {
+ if (version == null) {
+ return;
+ }
+ synchronized(datanodeMap) {
+ Integer count = this.datanodesSoftwareVersions.get(version);
+ if(count != null) {
+ if(count > 1) {
+ this.datanodesSoftwareVersions.put(version, count-1);
+ } else {
+ this.datanodesSoftwareVersions.remove(version);
+ }
+ }
+ }
+ }
+
+ private boolean shouldCountVersion(DatanodeDescriptor node) {
+ return node.getSoftwareVersion() != null && node.isAlive &&
+ !isDatanodeDead(node);
+ }
+
+ private void countSoftwareVersions() {
+ synchronized(datanodeMap) {
+ HashMap<String, Integer> versionCount = new HashMap<String, Integer>();
+ for(DatanodeDescriptor dn: datanodeMap.values()) {
+ // Check isAlive too because right after removeDatanode(),
+ // isDatanodeDead() is still true
+ if(shouldCountVersion(dn))
+ {
+ Integer num = versionCount.get(dn.getSoftwareVersion());
+ num = num == null ? 1 : num+1;
+ versionCount.put(dn.getSoftwareVersion(), num);
+ }
+ }
+ this.datanodesSoftwareVersions = versionCount;
+ }
+ }
+
+ public HashMap<String, Integer> getDatanodesSoftwareVersions() {
+ synchronized(datanodeMap) {
+ return new HashMap<String, Integer> (this.datanodesSoftwareVersions);
+ }
+ }
+
/* Resolve a node's network location */
private String resolveNetworkLocation (DatanodeID node) {
List<String> names = new ArrayList<String>(1);
@@ -755,21 +819,28 @@ public class DatanodeManager {
try {
// update cluster map
getNetworkTopology().remove(nodeS);
+ if(shouldCountVersion(nodeS)) {
+ decrementVersionCount(nodeS.getSoftwareVersion());
+ }
nodeS.updateRegInfo(nodeReg);
+
+ nodeS.setSoftwareVersion(nodeReg.getSoftwareVersion());
nodeS.setDisallowed(false); // Node is in the include list
-
+
// resolve network location
nodeS.setNetworkLocation(resolveNetworkLocation(nodeS));
getNetworkTopology().add(nodeS);
// also treat the registration message as a heartbeat
heartbeatManager.register(nodeS);
+ incrementVersionCount(nodeS.getSoftwareVersion());
checkDecommissioning(nodeS);
success = true;
} finally {
if (!success) {
removeDatanode(nodeS);
wipeDatanode(nodeS);
+ countSoftwareVersions();
}
}
return;
@@ -793,6 +864,7 @@ public class DatanodeManager {
try {
nodeDescr.setNetworkLocation(resolveNetworkLocation(nodeDescr));
networktopology.add(nodeDescr);
+ nodeDescr.setSoftwareVersion(nodeReg.getSoftwareVersion());
// register new datanode
addDatanode(nodeDescr);
@@ -803,10 +875,12 @@ public class DatanodeManager {
// because its is done when the descriptor is created
heartbeatManager.addDatanode(nodeDescr);
success = true;
+ incrementVersionCount(nodeReg.getSoftwareVersion());
} finally {
if (!success) {
removeDatanode(nodeDescr);
wipeDatanode(nodeDescr);
+ countSoftwareVersions();
}
}
} catch (InvalidTopologyException e) {
@@ -828,6 +902,7 @@ public class DatanodeManager {
namesystem.writeLock();
try {
refreshDatanodes();
+ countSoftwareVersions();
} finally {
namesystem.writeUnlock();
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Thu Sep 19 23:42:10 2013
@@ -387,11 +387,15 @@ public class DataNode extends Configured
String infoHost = infoSocAddr.getHostName();
int tmpInfoPort = infoSocAddr.getPort();
this.infoServer = (secureResources == null)
- ? new HttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0,
- conf, new AccessControlList(conf.get(DFS_ADMIN, " ")))
- : new HttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0,
- conf, new AccessControlList(conf.get(DFS_ADMIN, " ")),
- secureResources.getListener());
+ ? new HttpServer.Builder().setName("datanode")
+ .setBindAddress(infoHost).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf)
+ .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))).build()
+ : new HttpServer.Builder().setName("datanode")
+ .setBindAddress(infoHost).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf)
+ .setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
+ .setConnector(secureResources.getListener()).build();
LOG.info("Opened info server at " + infoHost + ":" + tmpInfoPort);
if (conf.getBoolean(DFS_HTTPS_ENABLE_KEY, false)) {
boolean needClientAuth = conf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY,
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java Thu Sep 19 23:42:10 2013
@@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.server.da
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
@@ -27,6 +29,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
+import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.jsp.JspWriter;
@@ -36,6 +39,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -43,6 +47,9 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.http.HtmlQuoting;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
@@ -50,6 +57,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.VersionInfo;
@InterfaceAudience.Private
public class DatanodeJspHelper {
@@ -712,4 +720,24 @@ public class DatanodeJspHelper {
final String nnAddr = request.getParameter(JspHelper.NAMENODE_ADDRESS);
return getDFSClient(ugi, nnAddr, conf);
}
+
+ /** Return a table containing version information. */
+ public static String getVersionTable(ServletContext context) {
+ StringBuilder sb = new StringBuilder();
+ final DataNode dataNode = (DataNode) context.getAttribute("datanode");
+ sb.append("<div class='dfstable'><table>");
+ sb.append("<tr><td class='col1'>Version:</td><td>");
+ sb.append(VersionInfo.getVersion() + ", " + VersionInfo.getRevision());
+ sb.append("</td></tr>\n" + "\n <tr><td class='col1'>Compiled:</td><td>"
+ + VersionInfo.getDate());
+ sb.append(" by " + VersionInfo.getUser() + " from "
+ + VersionInfo.getBranch());
+ if (dataNode != null) {
+ sb.append("</td></tr>\n <tr><td class='col1'>Cluster ID:</td><td>"
+ + dataNode.getClusterId());
+ }
+ sb.append("</td></tr>\n</table></div>");
+ return sb.toString();
+ }
+
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Thu Sep 19 23:42:10 2013
@@ -410,8 +410,9 @@ public class DatanodeWebHdfsMethods {
throw ioe;
}
- final long n = length.getValue() != null? length.getValue()
- : in.getVisibleLength() - offset.getValue();
+ final long n = length.getValue() != null ?
+ Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
+ in.getVisibleLength() - offset.getValue();
return Response.ok(new OpenEntity(in, n, dfsclient)).type(
MediaType.APPLICATION_OCTET_STREAM).build();
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java Thu Sep 19 23:42:10 2013
@@ -359,6 +359,7 @@ class ClusterJspHelper {
nn.httpAddress = httpAddress;
getLiveNodeCount(getProperty(props, "LiveNodes").getValueAsText(), nn);
getDeadNodeCount(getProperty(props, "DeadNodes").getValueAsText(), nn);
+ nn.softwareVersion = getProperty(props, "SoftwareVersion").getTextValue();
return nn;
}
@@ -596,6 +597,7 @@ class ClusterJspHelper {
toXmlItemBlockWithLink(doc, nn.deadDatanodeCount + " (" +
nn.deadDecomCount + ")", nn.httpAddress+"/dfsnodelist.jsp?whatNodes=DEAD"
, "Dead Datanode (Decommissioned)");
+ toXmlItemBlock(doc, "Software Version", nn.softwareVersion);
doc.endTag(); // node
}
doc.endTag(); // namenodes
@@ -624,6 +626,7 @@ class ClusterJspHelper {
int deadDatanodeCount = 0;
int deadDecomCount = 0;
String httpAddress = null;
+ String softwareVersion = "";
}
/**
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu Sep 19 23:42:10 2013
@@ -69,6 +69,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
import org.apache.hadoop.hdfs.util.ByteArray;
+import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.annotations.VisibleForTesting;
@@ -949,7 +950,7 @@ public class FSDirectory implements Clos
if (removedDst != null) {
undoRemoveDst = false;
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
filesDeleted = removedDst.cleanSubtree(null,
dstIIP.getLatestSnapshot(), collectedBlocks, removedINodes, true)
.get(Quota.NAMESPACE);
@@ -1409,7 +1410,7 @@ public class FSDirectory implements Clos
QuotaExceededException, SnapshotAccessControlException {
assert hasWriteLock();
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
final INodesInPath inodesInPath = rootDir.getINodesInPath4Write(
normalizePath(src), false);
@@ -2139,6 +2140,10 @@ public class FSDirectory implements Clos
/** Verify if the snapshot name is legal. */
void verifySnapshotName(String snapshotName, String path)
throws PathComponentTooLongException {
+ if (snapshotName.contains(Path.SEPARATOR)) {
+ throw new HadoopIllegalArgumentException(
+ "Snapshot name cannot contain \"" + Path.SEPARATOR + "\"");
+ }
final byte[] bytes = DFSUtil.string2Bytes(snapshotName);
verifyINodeName(bytes);
verifyMaxComponentLength(bytes, path, 0);
@@ -2772,6 +2777,19 @@ public class FSDirectory implements Clos
throw new FileNotFoundException(
"File for given inode path does not exist: " + src);
}
+
+ // Handle single ".." for NFS lookup support.
+ if ((pathComponents.length > 4)
+ && DFSUtil.bytes2String(pathComponents[4]).equals("..")) {
+ INode parent = inode.getParent();
+ if (parent == null || parent.getId() == INodeId.ROOT_INODE_ID) {
+ // inode is root, or its parent is root.
+ return Path.SEPARATOR;
+ } else {
+ return parent.getFullPathName();
+ }
+ }
+
StringBuilder path = id == INodeId.ROOT_INODE_ID ? new StringBuilder()
: new StringBuilder(inode.getFullPathName());
for (int i = 4; i < pathComponents.length; i++) {
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Thu Sep 19 23:42:10 2013
@@ -1274,6 +1274,7 @@ public class FSEditLog implements LogsPu
}
}
+ @Override
public void selectInputStreams(Collection<EditLogInputStream> streams,
long fromTxId, boolean inProgressOk, boolean forReading) {
journalSet.selectInputStreams(streams, fromTxId, inProgressOk, forReading);
@@ -1284,18 +1285,27 @@ public class FSEditLog implements LogsPu
return selectInputStreams(fromTxId, toAtLeastTxId, null, true);
}
+ /** Select a list of input streams to load */
+ public Collection<EditLogInputStream> selectInputStreams(
+ long fromTxId, long toAtLeastTxId, MetaRecoveryContext recovery,
+ boolean inProgressOk) throws IOException {
+ return selectInputStreams(fromTxId, toAtLeastTxId, recovery, inProgressOk,
+ true);
+ }
+
/**
- * Select a list of input streams to load.
+ * Select a list of input streams.
*
* @param fromTxId first transaction in the selected streams
* @param toAtLeast the selected streams must contain this transaction
* @param inProgessOk set to true if in-progress streams are OK
+ * @param forReading whether or not to use the streams to load the edit log
*/
public synchronized Collection<EditLogInputStream> selectInputStreams(
long fromTxId, long toAtLeastTxId, MetaRecoveryContext recovery,
- boolean inProgressOk) throws IOException {
+ boolean inProgressOk, boolean forReading) throws IOException {
List<EditLogInputStream> streams = new ArrayList<EditLogInputStream>();
- selectInputStreams(streams, fromTxId, inProgressOk, true);
+ selectInputStreams(streams, fromTxId, inProgressOk, forReading);
try {
checkForGaps(streams, fromTxId, toAtLeastTxId, inProgressOk);
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Thu Sep 19 23:42:10 2013
@@ -22,7 +22,6 @@ import static org.apache.hadoop.util.Tim
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumMap;
import java.util.List;
@@ -75,6 +74,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
+import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.hdfs.util.Holder;
import com.google.common.base.Joiner;
@@ -582,7 +582,7 @@ public class FSEditLogLoader {
case OP_DELETE_SNAPSHOT: {
DeleteSnapshotOp deleteSnapshotOp = (DeleteSnapshotOp) op;
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
fsNamesys.getSnapshotManager().deleteSnapshot(
deleteSnapshotOp.snapshotRoot, deleteSnapshotOp.snapshotName,
collectedBlocks, removedINodes);