You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by um...@apache.org on 2014/05/12 14:44:05 UTC
svn commit: r1593948 [1/4] - in
/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-nfs/ hadoop-hdfs/ hadoop-hdfs/src/main/java/
hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/
hadoop-hdfs/src/main/java/org/apache/h...
Author: umamahesh
Date: Mon May 12 12:43:59 2014
New Revision: 1593948
URL: http://svn.apache.org/r1593948
Log:
Merge from trunk to HDFS-2006 branch
Added:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/proto-web.xml
- copied unchanged from r1593927, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/proto-web.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
- copied unchanged from r1593927, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
- copied unchanged from r1593927, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
Removed:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/browseBlock.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/browseDirectory.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dataNodeHome.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/tail.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/block_info_xml.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_replicas_xml.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/decommission.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/decommission.xsl
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsclusterhealth.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsclusterhealth.xsl
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsclusterhealth_utils.xsl
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/nn_browsedfscontent.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/journalstatus.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/proto-datanode-web.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/proto-hdfs-web.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/proto-journal-web.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/proto-secondary-web.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.jsp
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterJspHelper.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorruptFilesJsp.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeJspHelper.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAWebUI.java
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAbandonBlock.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRenewer.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepDecreasing.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestExtendedBlock.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLI.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1588992-1593927
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml Mon May 12 12:43:59 2014
@@ -108,26 +108,10 @@
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
@@ -137,10 +121,6 @@
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-api-2.1</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
@@ -151,10 +131,6 @@
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -171,26 +147,10 @@
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
@@ -200,10 +160,6 @@
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-api-2.1</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
@@ -214,10 +170,6 @@
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml Mon May 12 12:43:59 2014
@@ -135,11 +135,6 @@ http://maven.apache.org/xsd/maven-4.0.0.
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>compile</scope>
@@ -180,11 +175,6 @@ http://maven.apache.org/xsd/maven-4.0.0.
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>xmlenc</groupId>
<artifactId>xmlenc</artifactId>
<scope>compile</scope>
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon May 12 12:43:59 2014
@@ -127,6 +127,8 @@ Trunk (Unreleased)
HDFS-6246. Remove 'dfs.support.append' flag from trunk code. (umamahesh)
+ HDFS-6252. Phase out the old web UI in HDFS. (wheat9)
+
OPTIMIZATIONS
BUG FIXES
@@ -265,6 +267,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6281. Provide option to use the NFS Gateway without having to use the
Hadoop portmapper. (atm)
+ HDFS-5168. Add cross node dependency support to BlockPlacementPolicy.
+ (Nikola Vujic via szetszwo)
+
IMPROVEMENTS
HDFS-6007. Update documentation about short-circuit local reads (iwasakims
@@ -329,6 +334,20 @@ Release 2.5.0 - UNRELEASED
HDFS-6210. Support GETACLSTATUS operation in WebImageViewer.
(Akira Ajisaka via wheat9)
+ HDFS-6269. NameNode Audit Log should differentiate between webHDFS open and
+ HDFS open. (Eric Payne via jeagles)
+
+ HDFS-6304. Consolidate the logic of path resolution in FSDirectory.
+ (wheat9)
+
+ HDFS-6295. Add "decommissioning" state and node state filtering to
+ dfsadmin. (wang)
+
+ HDFS-6294. Use INode IDs to avoid conflicts when a file open for write is
+ renamed. (cmccabe)
+
+ HDFS-6328. Clean up dead code in FSDirectory. (wheat9)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
@@ -412,6 +431,21 @@ Release 2.5.0 - UNRELEASED
HDFS-6218. Audit log should use true client IP for proxied webhdfs
operations. (daryn via kihwal)
+ HDFS-6288. DFSInputStream Pread doesn't update ReadStatistics.
+ (Juan Yu via wang)
+
+ HDFS-6289. HA failover can fail if there are pending DN messages for DNs
+ which no longer exist. (atm)
+
+ HDFS-6337. Setfacl testcase is failing due to dash character in username
+ in TestAclCLI (umamahesh)
+
+ HDFS-5381. ExtendedBlock#hashCode should use both blockId and block pool ID
+ (Benoy Antony via Colin Patrick McCabe)
+
+ HDFS-6240. WebImageViewer returns 404 if LISTSTATUS to an empty directory.
+ (Akira Ajisaka via wheat9)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -467,6 +501,17 @@ Release 2.4.1 - UNRELEASED
HDFS-6245. datanode fails to start with a bad disk even when failed
volumes is set. (Arpit Agarwal)
+ HDFS-2882. DN continues to start up, even if block pool fails to initialize
+ (vinayakumarb)
+
+ HDFS-6340. DN can't finalize upgrade. (Rahul Singhal via Arpit Agarwal)
+
+ HDFS-6329. WebHdfs does not work if HA is enabled on NN but logical URI is
+ not configured. (kihwal)
+
+ HDFS-6313. WebHdfs may use the wrong NN when configured for multiple HA NNs
+ (kihwal)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml Mon May 12 12:43:59 2014
@@ -122,11 +122,6 @@ http://maven.apache.org/xsd/maven-4.0.0.
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>compile</scope>
@@ -167,11 +162,6 @@ http://maven.apache.org/xsd/maven-4.0.0.
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>xmlenc</groupId>
<artifactId>xmlenc</artifactId>
<scope>compile</scope>
@@ -202,123 +192,6 @@ http://maven.apache.org/xsd/maven-4.0.0.
</configuration>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo.jspc</groupId>
- <artifactId>jspc-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>hdfs</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- <configuration>
- <compile>false</compile>
- <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory>
- <webFragmentFile>${project.build.directory}/hdfs-jsp-servlet-definitions.xml</webFragmentFile>
- <packageName>org.apache.hadoop.hdfs.server.namenode</packageName>
- <sources>
- <directory>${basedir}/src/main/webapps/hdfs</directory>
- <includes>
- <include>*.jsp</include>
- </includes>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>secondary</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- <configuration>
- <compile>false</compile>
- <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory>
- <webFragmentFile>${project.build.directory}/secondary-jsp-servlet-definitions.xml</webFragmentFile>
- <packageName>org.apache.hadoop.hdfs.server.namenode</packageName>
- <sources>
- <directory>${basedir}/src/main/webapps/secondary</directory>
- <includes>
- <include>*.jsp</include>
- </includes>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>journal</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- <configuration>
- <compile>false</compile>
- <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory>
- <webFragmentFile>${project.build.directory}/journal-jsp-servlet-definitions.xml</webFragmentFile>
- <packageName>org.apache.hadoop.hdfs.server.journalservice</packageName>
- <sources>
- <directory>${basedir}/src/main/webapps/journal</directory>
- <includes>
- <include>*.jsp</include>
- </includes>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>datanode</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- <configuration>
- <compile>false</compile>
- <workingDirectory>${project.build.directory}/generated-sources/java</workingDirectory>
- <webFragmentFile>${project.build.directory}/datanode-jsp-servlet-definitions.xml</webFragmentFile>
- <packageName>org.apache.hadoop.hdfs.server.datanode</packageName>
- <sources>
- <directory>${basedir}/src/main/webapps/datanode</directory>
- <includes>
- <include>*.jsp</include>
- </includes>
- </sources>
- </configuration>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.codehaus.mojo.jspc</groupId>
- <artifactId>jspc-compiler-tomcat5</artifactId>
- <version>2.0-alpha-3</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>1.4.1</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl104-over-slf4j</artifactId>
- <version>1.4.1</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-jsp-generated-sources-directory</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>${project.build.directory}/generated-sources/java</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<configuration>
@@ -326,18 +199,6 @@ http://maven.apache.org/xsd/maven-4.0.0.
</configuration>
<executions>
<execution>
- <id>create-jsp-generated-sources-directory</id>
- <phase>initialize</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <mkdir dir="${project.build.directory}/generated-sources/java" />
- </target>
- </configuration>
- </execution>
- <execution>
<id>create-web-xmls</id>
<phase>compile</phase>
<goals>
@@ -345,32 +206,21 @@ http://maven.apache.org/xsd/maven-4.0.0.
</goals>
<configuration>
<target>
- <loadfile property="hdfs.servlet.definitions" srcFile="${project.build.directory}/hdfs-jsp-servlet-definitions.xml"/>
- <loadfile property="secondary.servlet.definitions" srcFile="${project.build.directory}/secondary-jsp-servlet-definitions.xml"/>
- <loadfile property="datanode.servlet.definitions" srcFile="${project.build.directory}/datanode-jsp-servlet-definitions.xml"/>
- <loadfile property="journal.servlet.definitions" srcFile="${project.build.directory}/journal-jsp-servlet-definitions.xml"/>
- <echoproperties destfile="${project.build.directory}/webxml.properties">
- <propertyset>
- <propertyref regex=".*.servlet.definitions"/>
- </propertyset>
- </echoproperties>
- <filter filtersfile="${project.build.directory}/webxml.properties"/>
- <copy file="${basedir}/src/main/webapps/proto-hdfs-web.xml"
+ <copy file="${basedir}/src/main/webapps/proto-web.xml"
tofile="${project.build.directory}/webapps/hdfs/WEB-INF/web.xml"
filtering="true"/>
- <copy file="${basedir}/src/main/webapps/proto-secondary-web.xml"
+ <copy file="${basedir}/src/main/webapps/proto-web.xml"
tofile="${project.build.directory}/webapps/secondary/WEB-INF/web.xml"
filtering="true"/>
- <copy file="${basedir}/src/main/webapps/proto-datanode-web.xml"
+ <copy file="${basedir}/src/main/webapps/proto-web.xml"
tofile="${project.build.directory}/webapps/datanode/WEB-INF/web.xml"
filtering="true"/>
- <copy file="${basedir}/src/main/webapps/proto-journal-web.xml"
+ <copy file="${basedir}/src/main/webapps/proto-web.xml"
tofile="${project.build.directory}/webapps/journal/WEB-INF/web.xml"
filtering="true"/>
<copy toDir="${project.build.directory}/webapps">
<fileset dir="${basedir}/src/main/webapps">
- <exclude name="**/*.jsp"/>
- <exclude name="**/proto-*-web.xml"/>
+ <exclude name="**/proto-web.xml"/>
</fileset>
</copy>
</target>
@@ -391,6 +241,7 @@ http://maven.apache.org/xsd/maven-4.0.0.
<copy todir="${project.build.directory}/test-classes/webapps">
<fileset dir="${project.build.directory}/webapps">
<exclude name="proto-*-web.xml"/>
+ <exclude name="**/proto-web.xml"/>
</fileset>
</copy>
</target>
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1588992-1593927
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Mon May 12 12:43:59 2014
@@ -504,8 +504,8 @@ public class DFSClient implements java.i
* that are currently being written by this client.
* Note that a file can only be written by a single client.
*/
- private final Map<String, DFSOutputStream> filesBeingWritten
- = new HashMap<String, DFSOutputStream>();
+ private final Map<Long, DFSOutputStream> filesBeingWritten
+ = new HashMap<Long, DFSOutputStream>();
/**
* Same as this(NameNode.getAddress(conf), conf);
@@ -734,14 +734,14 @@ public class DFSClient implements java.i
}
/** Get a lease and start automatic renewal */
- private void beginFileLease(final String src, final DFSOutputStream out)
+ private void beginFileLease(final long inodeId, final DFSOutputStream out)
throws IOException {
- getLeaseRenewer().put(src, out, this);
+ getLeaseRenewer().put(inodeId, out, this);
}
/** Stop renewal of lease for the file. */
- void endFileLease(final String src) throws IOException {
- getLeaseRenewer().closeFile(src, this);
+ void endFileLease(final long inodeId) throws IOException {
+ getLeaseRenewer().closeFile(inodeId, this);
}
@@ -749,9 +749,9 @@ public class DFSClient implements java.i
* enforced to consistently update its local dfsclients array and
* client's filesBeingWritten map.
*/
- void putFileBeingWritten(final String src, final DFSOutputStream out) {
+ void putFileBeingWritten(final long inodeId, final DFSOutputStream out) {
synchronized(filesBeingWritten) {
- filesBeingWritten.put(src, out);
+ filesBeingWritten.put(inodeId, out);
// update the last lease renewal time only when there was no
// writes. once there is one write stream open, the lease renewer
// thread keeps it updated well with in anyone's expiration time.
@@ -762,9 +762,9 @@ public class DFSClient implements java.i
}
/** Remove a file. Only called from LeaseRenewer. */
- void removeFileBeingWritten(final String src) {
+ void removeFileBeingWritten(final long inodeId) {
synchronized(filesBeingWritten) {
- filesBeingWritten.remove(src);
+ filesBeingWritten.remove(inodeId);
if (filesBeingWritten.isEmpty()) {
lastLeaseRenewal = 0;
}
@@ -849,14 +849,14 @@ public class DFSClient implements java.i
/** Close/abort all files being written. */
private void closeAllFilesBeingWritten(final boolean abort) {
for(;;) {
- final String src;
+ final long inodeId;
final DFSOutputStream out;
synchronized(filesBeingWritten) {
if (filesBeingWritten.isEmpty()) {
return;
}
- src = filesBeingWritten.keySet().iterator().next();
- out = filesBeingWritten.remove(src);
+ inodeId = filesBeingWritten.keySet().iterator().next();
+ out = filesBeingWritten.remove(inodeId);
}
if (out != null) {
try {
@@ -866,8 +866,8 @@ public class DFSClient implements java.i
out.close();
}
} catch(IOException ie) {
- LOG.error("Failed to " + (abort? "abort": "close") + " file " + src,
- ie);
+ LOG.error("Failed to " + (abort? "abort": "close") +
+ " inode " + inodeId, ie);
}
}
}
@@ -1465,7 +1465,7 @@ public class DFSClient implements java.i
final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
src, masked, flag, createParent, replication, blockSize, progress,
buffersize, dfsClientConf.createChecksum(checksumOpt), favoredNodeStrs);
- beginFileLease(src, result);
+ beginFileLease(result.getFileId(), result);
return result;
}
@@ -1513,7 +1513,7 @@ public class DFSClient implements java.i
flag, createParent, replication, blockSize, progress, buffersize,
checksum);
}
- beginFileLease(src, result);
+ beginFileLease(result.getFileId(), result);
return result;
}
@@ -1601,7 +1601,7 @@ public class DFSClient implements java.i
+ src + " on client " + clientName);
}
final DFSOutputStream result = callAppend(stat, src, buffersize, progress);
- beginFileLease(src, result);
+ beginFileLease(result.getFileId(), result);
return result;
}
@@ -2438,8 +2438,8 @@ public class DFSClient implements java.i
}
@VisibleForTesting
- ExtendedBlock getPreviousBlock(String file) {
- return filesBeingWritten.get(file).getBlock();
+ ExtendedBlock getPreviousBlock(long fileId) {
+ return filesBeingWritten.get(fileId).getBlock();
}
/**
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Mon May 12 12:43:59 2014
@@ -1038,6 +1038,8 @@ implements ByteBufferReadable, CanSetDro
setConfiguration(dfsClient.getConfiguration()).
build();
int nread = reader.readAll(buf, offset, len);
+ updateReadStatistics(readStatistics, nread, reader);
+
if (nread != len) {
throw new IOException("truncated return from reader.read(): " +
"excpected " + len + ", got " + nread);
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Mon May 12 12:43:59 2014
@@ -1012,7 +1012,7 @@ public class DFSOutputStream extends FSO
//get a new datanode
final DatanodeInfo[] original = nodes;
final LocatedBlock lb = dfsClient.namenode.getAdditionalDatanode(
- src, block, nodes, storageIDs,
+ src, fileId, block, nodes, storageIDs,
failed.toArray(new DatanodeInfo[failed.size()]),
1, dfsClient.clientName);
setPipeline(lb);
@@ -1268,7 +1268,8 @@ public class DFSOutputStream extends FSO
if (!success) {
DFSClient.LOG.info("Abandoning " + block);
- dfsClient.namenode.abandonBlock(block, src, dfsClient.clientName);
+ dfsClient.namenode.abandonBlock(block, fileId, src,
+ dfsClient.clientName);
block = null;
DFSClient.LOG.info("Excluding datanode " + nodes[errorIndex]);
excludedNodes.put(nodes[errorIndex], nodes[errorIndex]);
@@ -1914,7 +1915,8 @@ public class DFSOutputStream extends FSO
// namenode.
if (persistBlocks.getAndSet(false) || updateLength) {
try {
- dfsClient.namenode.fsync(src, dfsClient.clientName, lastBlockLength);
+ dfsClient.namenode.fsync(src, fileId,
+ dfsClient.clientName, lastBlockLength);
} catch (IOException ioe) {
DFSClient.LOG.warn("Unable to persist blocks in hflush for " + src, ioe);
// If we got an error here, it might be because some other thread called
@@ -2035,7 +2037,7 @@ public class DFSOutputStream extends FSO
streamer.setLastException(new IOException("Lease timeout of "
+ (dfsClient.getHdfsTimeout()/1000) + " seconds expired."));
closeThreads(true);
- dfsClient.endFileLease(src);
+ dfsClient.endFileLease(fileId);
}
// shutdown datastreamer and responseprocessor threads.
@@ -2090,7 +2092,7 @@ public class DFSOutputStream extends FSO
ExtendedBlock lastBlock = streamer.getBlock();
closeThreads(false);
completeFile(lastBlock);
- dfsClient.endFileLease(src);
+ dfsClient.endFileLease(fileId);
} catch (ClosedChannelException e) {
} finally {
closed = true;
@@ -2184,7 +2186,7 @@ public class DFSOutputStream extends FSO
}
@VisibleForTesting
- long getFileId() {
+ public long getFileId() {
return fileId;
}
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java Mon May 12 12:43:59 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.security.authorize.Service;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
+import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
/**
* {@link PolicyProvider} for HDFS protocols.
@@ -64,7 +65,10 @@ public class HDFSPolicyProvider extends
RefreshUserMappingsProtocol.class),
new Service(
CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_GET_USER_MAPPINGS,
- GetUserMappingsProtocol.class)
+ GetUserMappingsProtocol.class),
+ new Service(
+ CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_CALLQUEUE,
+ RefreshCallQueueProtocol.class)
};
@Override
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java Mon May 12 12:43:59 2014
@@ -281,7 +281,7 @@ class LeaseRenewer {
&& Time.now() - emptyTime > gracePeriod;
}
- synchronized void put(final String src, final DFSOutputStream out,
+ synchronized void put(final long inodeId, final DFSOutputStream out,
final DFSClient dfsc) {
if (dfsc.isClientRunning()) {
if (!isRunning() || isRenewerExpired()) {
@@ -319,7 +319,7 @@ class LeaseRenewer {
});
daemon.start();
}
- dfsc.putFileBeingWritten(src, out);
+ dfsc.putFileBeingWritten(inodeId, out);
emptyTime = Long.MAX_VALUE;
}
}
@@ -330,8 +330,8 @@ class LeaseRenewer {
}
/** Close a file. */
- void closeFile(final String src, final DFSClient dfsc) {
- dfsc.removeFileBeingWritten(src);
+ void closeFile(final long inodeId, final DFSClient dfsc) {
+ dfsc.removeFileBeingWritten(inodeId);
synchronized(this) {
if (dfsc.isFilesBeingWrittenEmpty()) {
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Mon May 12 12:43:59 2014
@@ -292,13 +292,20 @@ public interface ClientProtocol {
* file.
* Any partial writes to the block will be discarded.
*
+ * @param b Block to abandon
+ * @param fileId The id of the file where the block resides. Older clients
+ * will pass GRANDFATHER_INODE_ID here.
+ * @param src The path of the file where the block resides.
+ * @param holder Lease holder.
+ *
* @throws AccessControlException If access is denied
* @throws FileNotFoundException file <code>src</code> is not found
* @throws UnresolvedLinkException If <code>src</code> contains a symlink
* @throws IOException If an I/O error occurred
*/
@Idempotent
- public void abandonBlock(ExtendedBlock b, String src, String holder)
+ public void abandonBlock(ExtendedBlock b, long fileId,
+ String src, String holder)
throws AccessControlException, FileNotFoundException,
UnresolvedLinkException, IOException;
@@ -346,6 +353,7 @@ public interface ClientProtocol {
* Get a datanode for an existing pipeline.
*
* @param src the file being written
+ * @param fileId the ID of the file being written
* @param blk the block being written
* @param existings the existing nodes in the pipeline
* @param excludes the excluded nodes
@@ -361,8 +369,10 @@ public interface ClientProtocol {
* @throws IOException If an I/O error occurred
*/
@Idempotent
- public LocatedBlock getAdditionalDatanode(final String src, final ExtendedBlock blk,
- final DatanodeInfo[] existings, final String[] existingStorageIDs,
+ public LocatedBlock getAdditionalDatanode(final String src,
+ final long fileId, final ExtendedBlock blk,
+ final DatanodeInfo[] existings,
+ final String[] existingStorageIDs,
final DatanodeInfo[] excludes,
final int numAdditionalNodes, final String clientName
) throws AccessControlException, FileNotFoundException,
@@ -898,6 +908,8 @@ public interface ClientProtocol {
* Write all metadata for this file into persistent storage.
* The file must be currently open for writing.
* @param src The string representation of the path
+ * @param inodeId The inode ID, or GRANDFATHER_INODE_ID if the client is
+ * too old to support fsync with inode IDs.
* @param client The string representation of the client
* @param lastBlockLength The length of the last block (under construction)
* to be reported to NameNode
@@ -907,7 +919,8 @@ public interface ClientProtocol {
* @throws IOException If an I/O error occurred
*/
@Idempotent
- public void fsync(String src, String client, long lastBlockLength)
+ public void fsync(String src, long inodeId, String client,
+ long lastBlockLength)
throws AccessControlException, FileNotFoundException,
UnresolvedLinkException, IOException;
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Mon May 12 12:43:59 2014
@@ -29,6 +29,8 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.Time;
import java.util.Date;
+import java.util.LinkedList;
+import java.util.List;
import static org.apache.hadoop.hdfs.DFSUtil.percent2String;
@@ -50,6 +52,8 @@ public class DatanodeInfo extends Datano
private int xceiverCount;
private String location = NetworkTopology.DEFAULT_RACK;
private String softwareVersion;
+ private List<String> dependentHostNames = new LinkedList<String>();
+
// Datanode administrative states
public enum AdminStates {
@@ -274,6 +278,21 @@ public class DatanodeInfo extends Datano
public synchronized void setNetworkLocation(String location) {
this.location = NodeBase.normalize(location);
}
+
+ /** Add a hostname to a list of network dependencies */
+ public void addDependentHostName(String hostname) {
+ dependentHostNames.add(hostname);
+ }
+
+ /** List of Network dependencies */
+ public List<String> getDependentHostNames() {
+ return dependentHostNames;
+ }
+
+ /** Sets the network dependencies */
+ public void setDependentHostNames(List<String> dependencyList) {
+ dependentHostNames = dependencyList;
+ }
/** A formatted string for reporting the status of the DataNode. */
public String getDatanodeReport() {
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java Mon May 12 12:43:59 2014
@@ -112,7 +112,8 @@ public class ExtendedBlock {
@Override // Object
public int hashCode() {
- return block.hashCode();
+ int result = 31 + poolId.hashCode();
+ return (31 * result + block.hashCode());
}
@Override // Object
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java Mon May 12 12:43:59 2014
@@ -104,7 +104,7 @@ public class HdfsConstants {
// type of the datanode report
public static enum DatanodeReportType {
- ALL, LIVE, DEAD
+ ALL, LIVE, DEAD, DECOMMISSIONING
}
// An invalid transaction ID that will never be seen in a real namesystem.
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java Mon May 12 12:43:59 2014
@@ -434,8 +434,8 @@ public class ClientNamenodeProtocolServe
public AbandonBlockResponseProto abandonBlock(RpcController controller,
AbandonBlockRequestProto req) throws ServiceException {
try {
- server.abandonBlock(PBHelper.convert(req.getB()), req.getSrc(),
- req.getHolder());
+ server.abandonBlock(PBHelper.convert(req.getB()), req.getFileId(),
+ req.getSrc(), req.getHolder());
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -473,7 +473,7 @@ public class ClientNamenodeProtocolServe
List<String> existingStorageIDsList = req.getExistingStorageUuidsList();
List<DatanodeInfoProto> excludesList = req.getExcludesList();
LocatedBlock result = server.getAdditionalDatanode(req.getSrc(),
- PBHelper.convert(req.getBlk()),
+ req.getFileId(), PBHelper.convert(req.getBlk()),
PBHelper.convert(existingList.toArray(
new DatanodeInfoProto[existingList.size()])),
existingStorageIDsList.toArray(
@@ -831,7 +831,8 @@ public class ClientNamenodeProtocolServe
public FsyncResponseProto fsync(RpcController controller,
FsyncRequestProto req) throws ServiceException {
try {
- server.fsync(req.getSrc(), req.getClient(), req.getLastBlockLength());
+ server.fsync(req.getSrc(), req.getFileId(),
+ req.getClient(), req.getLastBlockLength());
return VOID_FSYNC_RESPONSE;
} catch (IOException e) {
throw new ServiceException(e);
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java Mon May 12 12:43:59 2014
@@ -334,11 +334,12 @@ public class ClientNamenodeProtocolTrans
}
@Override
- public void abandonBlock(ExtendedBlock b, String src, String holder)
- throws AccessControlException, FileNotFoundException,
- UnresolvedLinkException, IOException {
+ public void abandonBlock(ExtendedBlock b, long fileId, String src,
+ String holder) throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
AbandonBlockRequestProto req = AbandonBlockRequestProto.newBuilder()
- .setB(PBHelper.convert(b)).setSrc(src).setHolder(holder).build();
+ .setB(PBHelper.convert(b)).setSrc(src).setHolder(holder)
+ .setFileId(fileId).build();
try {
rpcProxy.abandonBlock(null, req);
} catch (ServiceException e) {
@@ -370,8 +371,8 @@ public class ClientNamenodeProtocolTrans
}
@Override
- public LocatedBlock getAdditionalDatanode(String src, ExtendedBlock blk,
- DatanodeInfo[] existings, String[] existingStorageIDs,
+ public LocatedBlock getAdditionalDatanode(String src, long fileId,
+ ExtendedBlock blk, DatanodeInfo[] existings, String[] existingStorageIDs,
DatanodeInfo[] excludes,
int numAdditionalNodes, String clientName) throws AccessControlException,
FileNotFoundException, SafeModeException, UnresolvedLinkException,
@@ -379,6 +380,7 @@ public class ClientNamenodeProtocolTrans
GetAdditionalDatanodeRequestProto req = GetAdditionalDatanodeRequestProto
.newBuilder()
.setSrc(src)
+ .setFileId(fileId)
.setBlk(PBHelper.convert(blk))
.addAllExistings(PBHelper.convert(existings))
.addAllExistingStorageUuids(Arrays.asList(existingStorageIDs))
@@ -755,11 +757,13 @@ public class ClientNamenodeProtocolTrans
}
@Override
- public void fsync(String src, String client, long lastBlockLength)
+ public void fsync(String src, long fileId, String client,
+ long lastBlockLength)
throws AccessControlException, FileNotFoundException,
UnresolvedLinkException, IOException {
FsyncRequestProto req = FsyncRequestProto.newBuilder().setSrc(src)
- .setClient(client).setLastBlockLength(lastBlockLength).build();
+ .setClient(client).setLastBlockLength(lastBlockLength)
+ .setFileId(fileId).build();
try {
rpcProxy.fsync(null, req);
} catch (ServiceException e) {
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Mon May 12 12:43:59 2014
@@ -1422,6 +1422,7 @@ public class PBHelper {
case ALL: return DatanodeReportTypeProto.ALL;
case LIVE: return DatanodeReportTypeProto.LIVE;
case DEAD: return DatanodeReportTypeProto.DEAD;
+ case DECOMMISSIONING: return DatanodeReportTypeProto.DECOMMISSIONING;
default:
throw new IllegalArgumentException("Unexpected data type report:" + t);
}
@@ -1433,6 +1434,7 @@ public class PBHelper {
case ALL: return DatanodeReportType.ALL;
case LIVE: return DatanodeReportType.LIVE;
case DEAD: return DatanodeReportType.DEAD;
+ case DECOMMISSIONING: return DatanodeReportType.DECOMMISSIONING;
default:
throw new IllegalArgumentException("Unexpected data type report:" + t);
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java Mon May 12 12:43:59 2014
@@ -842,7 +842,7 @@ public class Balancer {
*/
private static void checkReplicationPolicyCompatibility(Configuration conf
) throws UnsupportedActionException {
- if (!(BlockPlacementPolicy.getInstance(conf, null, null) instanceof
+ if (!(BlockPlacementPolicy.getInstance(conf, null, null, null) instanceof
BlockPlacementPolicyDefault)) {
throw new UnsupportedActionException(
"Balancer without BlockPlacementPolicyDefault");
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Mon May 12 12:43:59 2014
@@ -267,7 +267,8 @@ public class BlockManager {
blocksMap = new BlocksMap(
LightWeightGSet.computeCapacity(2.0, "BlocksMap"));
blockplacement = BlockPlacementPolicy.getInstance(
- conf, stats, datanodeManager.getNetworkTopology());
+ conf, stats, datanodeManager.getNetworkTopology(),
+ datanodeManager.getHost2DatanodeMap());
pendingReplications = new PendingReplicationBlocks(conf.getInt(
DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,
DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_DEFAULT) * 1000L);
@@ -1007,6 +1008,8 @@ public class BlockManager {
while(it.hasNext()) {
removeStoredBlock(it.next(), node);
}
+ // Remove all pending DN messages referencing this DN.
+ pendingDNMessages.removeAllMessagesForDatanode(node);
node.resetBlocks();
invalidateBlocks.remove(node.getDatanodeUuid());
@@ -1081,7 +1084,8 @@ public class BlockManager {
DatanodeDescriptor node = getDatanodeManager().getDatanode(dn);
if (node == null) {
throw new IOException("Cannot mark " + b
- + " as corrupt because datanode " + dn + " does not exist");
+ + " as corrupt because datanode " + dn + " (" + dn.getDatanodeUuid()
+ + ") does not exist");
}
BlockCollection bc = b.corrupted.getBlockCollection();
@@ -1981,6 +1985,9 @@ public class BlockManager {
// If the block is an out-of-date generation stamp or state,
// but we're the standby, we shouldn't treat it as corrupt,
// but instead just queue it for later processing.
+ // TODO: Pretty confident this should be s/storedBlock/block below,
+ // since we should be postponing the info of the reported block, not
+ // the stored block. See HDFS-6289 for more context.
queueReportedBlock(dn, storageID, storedBlock, reportedState,
QUEUE_REASON_CORRUPT_STATE);
} else {
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java Mon May 12 12:43:59 2014
@@ -139,7 +139,8 @@ public abstract class BlockPlacementPoli
* @param clusterMap cluster topology
*/
abstract protected void initialize(Configuration conf, FSClusterStats stats,
- NetworkTopology clusterMap);
+ NetworkTopology clusterMap,
+ Host2NodesMap host2datanodeMap);
/**
* Get an instance of the configured Block Placement Policy based on the
@@ -153,14 +154,15 @@ public abstract class BlockPlacementPoli
*/
public static BlockPlacementPolicy getInstance(Configuration conf,
FSClusterStats stats,
- NetworkTopology clusterMap) {
+ NetworkTopology clusterMap,
+ Host2NodesMap host2datanodeMap) {
final Class<? extends BlockPlacementPolicy> replicatorClass = conf.getClass(
DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
BlockPlacementPolicy.class);
final BlockPlacementPolicy replicator = ReflectionUtils.newInstance(
replicatorClass, conf);
- replicator.initialize(conf, stats, clusterMap);
+ replicator.initialize(conf, stats, clusterMap, host2datanodeMap);
return replicator;
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Mon May 12 12:43:59 2014
@@ -70,6 +70,7 @@ public class BlockPlacementPolicyDefault
protected boolean considerLoad;
private boolean preferLocalNode = true;
protected NetworkTopology clusterMap;
+ protected Host2NodesMap host2datanodeMap;
private FSClusterStats stats;
protected long heartbeatInterval; // interval for DataNode heartbeats
private long staleInterval; // interval used to identify stale DataNodes
@@ -80,8 +81,9 @@ public class BlockPlacementPolicyDefault
protected int tolerateHeartbeatMultiplier;
protected BlockPlacementPolicyDefault(Configuration conf, FSClusterStats stats,
- NetworkTopology clusterMap) {
- initialize(conf, stats, clusterMap);
+ NetworkTopology clusterMap,
+ Host2NodesMap host2datanodeMap) {
+ initialize(conf, stats, clusterMap, host2datanodeMap);
}
protected BlockPlacementPolicyDefault() {
@@ -89,11 +91,13 @@ public class BlockPlacementPolicyDefault
@Override
public void initialize(Configuration conf, FSClusterStats stats,
- NetworkTopology clusterMap) {
+ NetworkTopology clusterMap,
+ Host2NodesMap host2datanodeMap) {
this.considerLoad = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, true);
this.stats = stats;
this.clusterMap = clusterMap;
+ this.host2datanodeMap = host2datanodeMap;
this.heartbeatInterval = conf.getLong(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000;
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java Mon May 12 12:43:59 2014
@@ -47,8 +47,8 @@ import org.apache.hadoop.net.NodeBase;
public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefault {
protected BlockPlacementPolicyWithNodeGroup(Configuration conf, FSClusterStats stats,
- NetworkTopology clusterMap) {
- initialize(conf, stats, clusterMap);
+ NetworkTopology clusterMap, DatanodeManager datanodeManager) {
+ initialize(conf, stats, clusterMap, host2datanodeMap);
}
protected BlockPlacementPolicyWithNodeGroup() {
@@ -56,8 +56,9 @@ public class BlockPlacementPolicyWithNod
@Override
public void initialize(Configuration conf, FSClusterStats stats,
- NetworkTopology clusterMap) {
- super.initialize(conf, stats, clusterMap);
+ NetworkTopology clusterMap,
+ Host2NodesMap host2datanodeMap) {
+ super.initialize(conf, stats, clusterMap, host2datanodeMap);
}
/** choose local node of localMachine as the target.
@@ -241,6 +242,36 @@ public class BlockPlacementPolicyWithNod
countOfExcludedNodes++;
}
}
+
+ countOfExcludedNodes += addDependentNodesToExcludedNodes(
+ chosenNode, excludedNodes);
+ return countOfExcludedNodes;
+ }
+
+ /**
+ * Add all nodes from a dependent nodes list to excludedNodes.
+ * @return number of new excluded nodes
+ */
+ private int addDependentNodesToExcludedNodes(DatanodeDescriptor chosenNode,
+ Set<Node> excludedNodes) {
+ if (this.host2datanodeMap == null) {
+ return 0;
+ }
+ int countOfExcludedNodes = 0;
+ for(String hostname : chosenNode.getDependentHostNames()) {
+ DatanodeDescriptor node =
+ this.host2datanodeMap.getDataNodeByHostName(hostname);
+ if(node!=null) {
+ if (excludedNodes.add(node)) {
+ countOfExcludedNodes++;
+ }
+ } else {
+ LOG.warn("Not able to find datanode " + hostname
+ + " which has dependency with datanode "
+ + chosenNode.getHostName());
+ }
+ }
+
return countOfExcludedNodes;
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Mon May 12 12:43:59 2014
@@ -373,6 +373,11 @@ public class DatanodeManager {
return host2DatanodeMap.getDatanodeByXferAddr(host, xferPort);
}
+ /** @return the Host2NodesMap */
+ public Host2NodesMap getHost2DatanodeMap() {
+ return this.host2DatanodeMap;
+ }
+
/**
* Given datanode address or host name, returns the DatanodeDescriptor for the
* same, or if it doesn't find the datanode, it looks for a machine local and
@@ -678,6 +683,52 @@ public class DatanodeManager {
}
/**
+ * Resolve a node's dependencies in the network. If the DNS to switch
+ * mapping fails then this method returns empty list of dependencies
+ * @param node to get dependencies for
+ * @return List of dependent host names
+ */
+ private List<String> getNetworkDependenciesWithDefault(DatanodeInfo node) {
+ List<String> dependencies;
+ try {
+ dependencies = getNetworkDependencies(node);
+ } catch (UnresolvedTopologyException e) {
+ LOG.error("Unresolved dependency mapping for host " +
+ node.getHostName() +". Continuing with an empty dependency list");
+ dependencies = Collections.emptyList();
+ }
+ return dependencies;
+ }
+
+ /**
+ * Resolves a node's dependencies in the network. If the DNS to switch
+ * mapping fails to get dependencies, then this method throws
+ * UnresolvedTopologyException.
+ * @param node to get dependencies for
+ * @return List of dependent host names
+ * @throws UnresolvedTopologyException if the DNS to switch mapping fails
+ */
+ private List<String> getNetworkDependencies(DatanodeInfo node)
+ throws UnresolvedTopologyException {
+ List<String> dependencies = Collections.emptyList();
+
+ if (dnsToSwitchMapping instanceof DNSToSwitchMappingWithDependency) {
+ //Get dependencies
+ dependencies =
+ ((DNSToSwitchMappingWithDependency)dnsToSwitchMapping).getDependency(
+ node.getHostName());
+ if(dependencies == null) {
+ LOG.error("The dependency call returned null for host " +
+ node.getHostName());
+ throw new UnresolvedTopologyException("The dependency call returned " +
+ "null for host " + node.getHostName());
+ }
+ }
+
+ return dependencies;
+ }
+
+ /**
* Remove an already decommissioned data node who is neither in include nor
* exclude hosts lists from the the list of live or dead nodes. This is used
* to not display an already decommssioned data node to the operators.
@@ -869,12 +920,14 @@ public class DatanodeManager {
nodeS.setDisallowed(false); // Node is in the include list
// resolve network location
- if(this.rejectUnresolvedTopologyDN)
- {
- nodeS.setNetworkLocation(resolveNetworkLocation(nodeS));
+ if(this.rejectUnresolvedTopologyDN) {
+ nodeS.setNetworkLocation(resolveNetworkLocation(nodeS));
+ nodeS.setDependentHostNames(getNetworkDependencies(nodeS));
} else {
nodeS.setNetworkLocation(
resolveNetworkLocationWithFallBackToDefaultLocation(nodeS));
+ nodeS.setDependentHostNames(
+ getNetworkDependenciesWithDefault(nodeS));
}
getNetworkTopology().add(nodeS);
@@ -900,9 +953,12 @@ public class DatanodeManager {
// resolve network location
if(this.rejectUnresolvedTopologyDN) {
nodeDescr.setNetworkLocation(resolveNetworkLocation(nodeDescr));
+ nodeDescr.setDependentHostNames(getNetworkDependencies(nodeDescr));
} else {
nodeDescr.setNetworkLocation(
resolveNetworkLocationWithFallBackToDefaultLocation(nodeDescr));
+ nodeDescr.setDependentHostNames(
+ getNetworkDependenciesWithDefault(nodeDescr));
}
networktopology.add(nodeDescr);
nodeDescr.setSoftwareVersion(nodeReg.getSoftwareVersion());
@@ -1183,10 +1239,15 @@ public class DatanodeManager {
/** For generating datanode reports */
public List<DatanodeDescriptor> getDatanodeListForReport(
final DatanodeReportType type) {
- boolean listLiveNodes = type == DatanodeReportType.ALL ||
- type == DatanodeReportType.LIVE;
- boolean listDeadNodes = type == DatanodeReportType.ALL ||
- type == DatanodeReportType.DEAD;
+ final boolean listLiveNodes =
+ type == DatanodeReportType.ALL ||
+ type == DatanodeReportType.LIVE;
+ final boolean listDeadNodes =
+ type == DatanodeReportType.ALL ||
+ type == DatanodeReportType.DEAD;
+ final boolean listDecommissioningNodes =
+ type == DatanodeReportType.ALL ||
+ type == DatanodeReportType.DECOMMISSIONING;
ArrayList<DatanodeDescriptor> nodes;
final HostFileManager.HostSet foundNodes = new HostFileManager.HostSet();
@@ -1197,7 +1258,10 @@ public class DatanodeManager {
nodes = new ArrayList<DatanodeDescriptor>(datanodeMap.size());
for (DatanodeDescriptor dn : datanodeMap.values()) {
final boolean isDead = isDatanodeDead(dn);
- if ((listLiveNodes && !isDead) || (listDeadNodes && isDead)) {
+ final boolean isDecommissioning = dn.isDecommissionInProgress();
+ if ((listLiveNodes && !isDead) ||
+ (listDeadNodes && isDead) ||
+ (listDecommissioningNodes && isDecommissioning)) {
nodes.add(dn);
}
foundNodes.add(HostFileManager.resolvedAddressFromDatanodeID(dn));
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java Mon May 12 12:43:59 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
@InterfaceAudience.Private
@InterfaceStability.Evolving
class Host2NodesMap {
+ private HashMap<String, String> mapHost = new HashMap<String, String>();
private final HashMap<String, DatanodeDescriptor[]> map
= new HashMap<String, DatanodeDescriptor[]>();
private final ReadWriteLock hostmapLock = new ReentrantReadWriteLock();
@@ -69,6 +70,10 @@ class Host2NodesMap {
}
String ipAddr = node.getIpAddr();
+ String hostname = node.getHostName();
+
+ mapHost.put(hostname, ipAddr);
+
DatanodeDescriptor[] nodes = map.get(ipAddr);
DatanodeDescriptor[] newNodes;
if (nodes==null) {
@@ -95,6 +100,7 @@ class Host2NodesMap {
}
String ipAddr = node.getIpAddr();
+ String hostname = node.getHostName();
hostmapLock.writeLock().lock();
try {
@@ -105,6 +111,8 @@ class Host2NodesMap {
if (nodes.length==1) {
if (nodes[0]==node) {
map.remove(ipAddr);
+ //remove hostname key since last datanode is removed
+ mapHost.remove(hostname);
return true;
} else {
return false;
@@ -188,12 +196,40 @@ class Host2NodesMap {
}
}
+
+
+ /** get a data node by its hostname. This should be used if only one
+ * datanode service is running on a hostname. If multiple datanodes
+ * are running on a hostname then use methods getDataNodeByXferAddr and
+ * getDataNodeByHostNameAndPort.
+ * @return DatanodeDescriptor if found; otherwise null.
+ */
+ DatanodeDescriptor getDataNodeByHostName(String hostname) {
+ if(hostname == null) {
+ return null;
+ }
+
+ hostmapLock.readLock().lock();
+ try {
+ String ipAddr = mapHost.get(hostname);
+ if(ipAddr == null) {
+ return null;
+ } else {
+ return getDatanodeByHost(ipAddr);
+ }
+ } finally {
+ hostmapLock.readLock().unlock();
+ }
+ }
+
@Override
public String toString() {
final StringBuilder b = new StringBuilder(getClass().getSimpleName())
.append("[");
- for(Map.Entry<String, DatanodeDescriptor[]> e : map.entrySet()) {
- b.append("\n " + e.getKey() + " => " + Arrays.asList(e.getValue()));
+ for(Map.Entry<String, String> host: mapHost.entrySet()) {
+ DatanodeDescriptor[] e = map.get(host.getValue());
+ b.append("\n " + host.getKey() + " => "+host.getValue() + " => "
+ + Arrays.asList(e));
}
return b.append("\n]").toString();
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java Mon May 12 12:43:59 2014
@@ -76,6 +76,27 @@ class PendingDataNodeMessages {
}
}
+ /**
+ * Remove all pending DN messages which reference the given DN.
+ * @param dn the datanode whose messages we should remove.
+ */
+ void removeAllMessagesForDatanode(DatanodeDescriptor dn) {
+ for (Map.Entry<Block, Queue<ReportedBlockInfo>> entry :
+ queueByBlockId.entrySet()) {
+ Queue<ReportedBlockInfo> newQueue = Lists.newLinkedList();
+ Queue<ReportedBlockInfo> oldQueue = entry.getValue();
+ while (!oldQueue.isEmpty()) {
+ ReportedBlockInfo rbi = oldQueue.remove();
+ if (!rbi.getNode().equals(dn)) {
+ newQueue.add(rbi);
+ } else {
+ count--;
+ }
+ }
+ queueByBlockId.put(entry.getKey(), newQueue);
+ }
+ }
+
void enqueueReportedBlock(DatanodeDescriptor dn, String storageID, Block block,
ReplicaState reportedState) {
block = new Block(block);