You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/06/06 02:18:04 UTC
svn commit: r1346682 [1/9] - in
/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project: ./ hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs/ hadoop-hdfs/dev-support/ hadoop-hdfs/src/contrib/bkjournal/
ha...
Author: szetszwo
Date: Wed Jun 6 00:17:38 2012
New Revision: 1346682
URL: http://svn.apache.org/viewvc?rev=1346682&view=rev
Log:
Merge r1337003 through r1346681 from trunk.
Added:
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StreamLimiter.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StreamLimiter.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Canceler.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Canceler.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/
- copied from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/hadoop.css
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStateTransitionFailure.java
- copied unchanged from r1346681, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStateTransitionFailure.java
Removed:
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/WriteLock.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/system/
Modified:
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_truncate.c
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/GetUserMappingsProtocolPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/MutableBlockCollection.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsLoader.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightLinkedSet.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AppendTestUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestNNHealthCheck.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestLightWeightHashSet.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-policy.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3042/hadoop-hdfs-project/hadoop-hdfs:r1306184-1342109
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1337003-1346681
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml Wed Jun 6 00:17:38 2012
@@ -14,7 +14,10 @@
-->
-<project>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
@@ -474,7 +477,7 @@
<!-- Using Unix script to preserve file permissions -->
<echo file="${project.build.directory}/tomcat-untar.sh">
- which cygpath 2> /dev/null
+ which cygpath 2> /dev/null
if [ $? = 1 ]; then
BUILD_DIR="${project.build.directory}"
else
@@ -517,7 +520,7 @@
<!-- Using Unix script to preserve symlinks -->
<echo file="${project.build.directory}/dist-maketar.sh">
- which cygpath 2> /dev/null
+ which cygpath 2> /dev/null
if [ $? = 1 ]; then
BUILD_DIR="${project.build.directory}"
else
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java Wed Jun 6 00:17:38 2012
@@ -288,10 +288,10 @@ public class FSOperations {
@Override
public Void execute(FileSystem fs) throws IOException {
if (replication == -1) {
- replication = fs.getDefaultReplication();
+ replication = fs.getDefaultReplication(path);
}
if (blockSize == -1) {
- blockSize = fs.getDefaultBlockSize();
+ blockSize = fs.getDefaultBlockSize(path);
}
FsPermission fsPermission = getPermission(permission);
int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java Wed Jun 6 00:17:38 2012
@@ -49,6 +49,7 @@ import org.apache.hadoop.lib.service.Pro
import org.apache.hadoop.lib.servlet.FileSystemReleaseFilter;
import org.apache.hadoop.lib.servlet.HostnameFilter;
import org.apache.hadoop.lib.wsrs.InputStreamEntity;
+import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -145,9 +146,15 @@ public class HttpFSServer {
String effectiveUser = user.getName();
if (doAs != null && !doAs.equals(user.getName())) {
ProxyUser proxyUser = HttpFSServerWebApp.get().get(ProxyUser.class);
- proxyUser.validate(user.getName(), HostnameFilter.get(), doAs);
+ String proxyUserName;
+ if (user instanceof AuthenticationToken) {
+ proxyUserName = ((AuthenticationToken)user).getUserName();
+ } else {
+ proxyUserName = user.getName();
+ }
+ proxyUser.validate(proxyUserName, HostnameFilter.get(), doAs);
effectiveUser = doAs;
- AUDIT_LOG.info("Proxy user [{}] DoAs user [{}]", user.getName(), doAs);
+ AUDIT_LOG.info("Proxy user [{}] DoAs user [{}]", proxyUserName, doAs);
}
return effectiveUser;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Jun 6 00:17:38 2012
@@ -13,6 +13,11 @@ Trunk (unreleased changes)
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
+ HDFS-744. Support hsync in HDFS. (Lars Hofhansl via szetszwo)
+
+ HDFS-3042. Automatic failover support for NameNode HA (todd)
+ (see dedicated section below for subtask breakdown)
+
IMPROVEMENTS
HDFS-1620. Rename HdfsConstants -> HdfsServerConstants, FSConstants ->
@@ -77,6 +82,14 @@ Trunk (unreleased changes)
HDFS-3293. Add toString(), equals(..) and hashCode() to JournalInfo.
(Hari Mankude via szetszwo)
+ HDFS-3197. Incorrect class comments in a few tests. (Andy Isaacson via eli)
+
+ HDFS-2391. Newly set BalancerBandwidth value is not displayed anywhere.
+ (harsh)
+
+ HDFS-3476. Correct the default used in TestDFSClientRetries.busyTest()
+ after HDFS-3462 (harsh)
+
OPTIMIZATIONS
HDFS-2834. Add a ByteBuffer-based read API to DFSInputStream.
@@ -109,9 +122,6 @@ Trunk (unreleased changes)
HDFS-2776. Missing interface annotation on JournalSet.
(Brandon Li via jitendra)
- HDFS-2759. Pre-allocate HDFS edit log files after writing version number.
- (atm)
-
HDFS-2908. Add apache license header for StorageReport.java. (Brandon Li
via jitendra)
@@ -141,8 +151,163 @@ Trunk (unreleased changes)
HDFS-3243. TestParallelRead timing out on jenkins. (Henry Robinson via todd)
HDFS-3265. PowerPc Build error. (Kumar Ravi via mattf)
+
+ HDFS-2312. FSNamesystem javadoc incorrectly says its for DNs. (harsh)
+
+ HDFS-3163. TestHDFSCLI.testAll fails if the user name is not all lowercase.
+ (Brandon Li via atm)
+
+ HDFS-3368. Missing blocks due to bad DataNodes coming up and down. (shv)
+
+ HDFS-3462. TestDFSClientRetries.busyTest() should restore default
+ xceiver count in the config. (Madhukara Phatak via harsh)
+
+ BREAKDOWN OF HDFS-3042 SUBTASKS
+
+ HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
+
+ HDFS-3200. Scope all ZKFC configurations by nameservice (todd)
-Release 2.0.0 - UNRELEASED
+ HDFS-3223. add zkfc to hadoop-daemon.sh script (todd)
+
+ HDFS-3261. TestHASafeMode fails on HDFS-3042 branch (todd)
+
+ HDFS-3159. Document NN auto-failover setup and configuration (todd)
+
+ HDFS-3412. Fix findbugs warnings in auto-HA branch (todd)
+
+ HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd)
+
+Release 2.0.1-alpha - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG
+ logging is enabled. (atm)
+
+ HDFS-3341. Change minimum RPC versions to respective SNAPSHOTs instead of
+ final releases. (todd)
+
+ HDFS-3369. Rename {get|set|add}INode(..) methods in BlockManager and
+ BlocksMap to {get|set|add}BlockCollection(..). (John George via szetszwo)
+
+ HDFS-3134. harden edit log loader against malformed or malicious input.
+ (Colin Patrick McCabe via eli)
+
+ HDFS-3230. Cleanup DatanodeID creation in the tests. (eli)
+
+ HDFS-3401. Cleanup DatanodeDescriptor creation in the tests. (eli)
+
+ HDFS-3400. DNs should be able start with jsvc even if security is disabled.
+ (atm via eli)
+
+ HDFS-3404. Make putImage in GetImageServlet infer remote address to fetch
+ from request. (atm)
+
+ HDFS-3335. check for edit log corruption at the end of the log
+ (Colin Patrick McCabe via todd)
+
+ HDFS-3417. Rename BalancerDatanode#getName to getDisplayName to be
+ consistent with Datanode. (eli)
+
+ HDFS-3416. Cleanup DatanodeID and DatanodeRegistration
+ constructors used by testing. (eli)
+
+ HDFS-3419. Cleanup LocatedBlock. (eli)
+
+ HDFS-3440. More effectively limit stream memory consumption when reading
+ corrupt edit logs (Colin Patrick McCabe via todd)
+
+ HDFS-3438. BootstrapStandby should not require a rollEdits on active node
+ (todd)
+
+ HDFS-2885. Remove "federation" from the nameservice config options.
+ (Tsz Wo (Nicholas) Sze via eli)
+
+ HDFS-3394. Do not use generic in INodeFile.getLastBlock(): the run-time
+ ClassCastException check is useless since generic type information is only
+ available in compile-time. (szetszwo)
+
+ HDFS-3454. Balancer unconditionally logs InterruptedException at
+ INFO level on shutdown if security is enabled. (eli)
+
+ HDFS-1013. Miscellaneous improvements to HTML markup for web UIs
+ (Eugene Koontz via todd)
+
+ OPTIMIZATIONS
+
+ HDFS-2982. Startup performance suffers when there are many edit log
+ segments. (Colin Patrick McCabe via todd)
+
+ BUG FIXES
+
+ HDFS-3385. The last block of INodeFileUnderConstruction is not
+ necessarily a BlockInfoUnderConstruction, so do not cast it in
+ FSNamesystem.recoverLeaseInternal(..). (szetszwo)
+
+ HDFS-3414. Balancer does not find NameNode if rpc-address or
+ servicerpc-address are not set in client configs. (atm)
+
+ HDFS-3031. Fix complete() and getAdditionalBlock() RPCs to be idempotent
+ (todd)
+
+ HDFS-2759. Pre-allocate HDFS edit log files after writing version number.
+ (atm)
+
+ HDFS-3413. TestFailureToReadEdits timing out. (atm)
+
+ HDFS-3422. TestStandbyIsHot timeouts too aggressive (todd)
+
+ HDFS-3433. GetImageServlet should allow administrative requestors when
+ security is enabled. (atm)
+
+ HDFS-1153. dfsnodelist.jsp should handle invalid input parameters.
+ (Ravi Phulari via eli)
+
+ HDFS-3434. InvalidProtocolBufferException when visiting DN
+ browseDirectory.jsp (eli)
+
+ HDFS-2800. Fix cancellation of checkpoints in the standby node to be more
+ reliable. (todd)
+
+ HDFS-3391. Fix InvalidateBlocks to compare blocks including their
+ generation stamps. (todd)
+
+ HDFS-3444. hdfs groups command doesn't work with security enabled. (atm)
+
+ HDFS-3415. Make sure all layout versions are the same for all storage
+ directories in the Namenode. (Brandon Li via szetszwo)
+
+ HDFS-3436. In DataNode.transferReplicaForPipelineRecovery(..), it should
+ use the stored generation stamp to check if the block is valid. (Vinay
+ via szetszwo)
+
+ HDFS-3460. HttpFS proxyuser validation with Kerberos ON uses full
+ principal name. (tucu)
+
+ HDFS-3484. hdfs fsck doesn't work if NN HTTP address is set to
+ 0.0.0.0 even if NN RPC address is configured. (atm via eli)
+
+ HDFS-3486. offlineimageviewer can't read fsimage files that contain
+ persistent delegation tokens. (Colin Patrick McCabe via eli)
+
+ HDFS-3487. offlineimageviewer should give byte offset information
+ when it encounters an exception. (Colin Patrick McCabe via eli)
+
+ HDFS-3442. Incorrect count for Missing Replicas in FSCK report. (Andrew
+ Wang via atm)
+
+ HDFS-3501. Checkpointing with security enabled will stop working
+ after ticket lifetime expires. (atm via eli)
+
+ HDFS-3266. DFSTestUtil#waitCorruptReplicas doesn't sleep between checks.
+ (Madhukara Phatak via atm)
+
+Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
@@ -368,9 +533,6 @@ Release 2.0.0 - UNRELEASED
HDFS-2505. Add a test to verify getFileChecksum(..) with ViewFS. (Ravi
Prakash via szetszwo)
- HDFS-3211. Add fence(..) and replace NamenodeRegistration with JournalInfo
- and epoch in JournalProtocol. (suresh via szetszwo)
-
HDFS-3240. Drop log level of "heartbeat: ..." in BPServiceActor to DEBUG
(todd)
@@ -435,21 +597,11 @@ Release 2.0.0 - UNRELEASED
so that INodeFile and INodeFileUnderConstruction do not have to be used in
block management. (John George via szetszwo)
- HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG
- logging is enabled. (atm)
-
- HDFS-3341. Change minimum RPC versions to respective SNAPSHOTs instead of
- final releases. (todd)
-
- HDFS-3369. Rename {get|set|add}INode(..) methods in BlockManager and
- BlocksMap to {get|set|add}BlockCollection(..). (John George via szetszwo)
-
- HDFS-3134. harden edit log loader against malformed or malicious input.
- (Colin Patrick McCabe via eli)
-
- HDFS-3230. Cleanup DatanodeID creation in the tests. (eli)
+ HDFS-3211. Add fence(..) and replace NamenodeRegistration with JournalInfo
+ and epoch in JournalProtocol. (suresh via szetszwo)
- HDFS-3401. Cleanup DatanodeDescriptor creation in the tests. (eli)
+ HDFS-3418. Rename BlockWithLocationsProto datanodeIDs field to storageIDs.
+ (eli)
OPTIMIZATIONS
@@ -644,9 +796,10 @@ Release 2.0.0 - UNRELEASED
HDFS-3395. NN doesn't start with HA+security enabled and HTTP address
set to 0.0.0.0. (atm)
- HDFS-3385. The last block of INodeFileUnderConstruction is not
- necessarily a BlockInfoUnderConstruction, so do not cast it in
- FSNamesystem.recoverLeaseInternal(..). (szetszwo)
+ HDFS-3026. HA: Handle failure during HA state transition. (atm)
+
+ HDFS-860. fuse-dfs truncate behavior causes issues with scp.
+ (Brian Bockelman via eli)
BREAKDOWN OF HDFS-1623 SUBTASKS
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml Wed Jun 6 00:17:38 2012
@@ -6,6 +6,9 @@
<Package name="org.apache.hadoop.hdfs.protocol.proto" />
</Match>
<Match>
+ <Package name="org.apache.hadoop.hdfs.server.namenode.ha.proto" />
+ </Match>
+ <Match>
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<Match>
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/pom.xml Wed Jun 6 00:17:38 2012
@@ -12,7 +12,10 @@
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
-<project>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
@@ -100,6 +103,33 @@
<artifactId>ant</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.4.2</version>
+ <exclusions>
+ <exclusion>
+ <!-- otherwise seems to drag in junit 3.8.1 via jline -->
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.4.2</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -247,7 +277,7 @@
<echo file="target/compile-proto.sh">
PROTO_DIR=src/main/proto
JAVA_DIR=target/generated-sources/java
- which cygpath 2> /dev/null
+ which cygpath 2> /dev/null
if [ $? = 1 ]; then
IS_WIN=false
else
@@ -255,8 +285,8 @@
WIN_PROTO_DIR=`cygpath --windows $PROTO_DIR`
WIN_JAVA_DIR=`cygpath --windows $JAVA_DIR`
fi
- mkdir -p $JAVA_DIR 2> /dev/null
- for PROTO_FILE in `ls $PROTO_DIR/*.proto 2> /dev/null`
+ mkdir -p $JAVA_DIR 2> /dev/null
+ for PROTO_FILE in `ls $PROTO_DIR/*.proto 2> /dev/null`
do
if [ "$IS_WIN" = "true" ]; then
protoc -I$WIN_PROTO_DIR --java_out=$WIN_JAVA_DIR $PROTO_FILE
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml Wed Jun 6 00:17:38 2012
@@ -12,7 +12,10 @@
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
-<project>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
@@ -55,6 +58,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-server</artifactId>
<scope>compile</scope>
@@ -64,6 +73,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
<profile>
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java Wed Jun 6 00:17:38 2012
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.BKException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -41,6 +42,7 @@ class BookKeeperEditLogInputStream exten
private final long firstTxId;
private final long lastTxId;
private final int logVersion;
+ private final boolean inProgress;
private final LedgerHandle lh;
private final FSEditLogOp.Reader reader;
@@ -69,22 +71,28 @@ class BookKeeperEditLogInputStream exten
this.firstTxId = metadata.getFirstTxId();
this.lastTxId = metadata.getLastTxId();
this.logVersion = metadata.getVersion();
+ this.inProgress = metadata.isInProgress();
+ if (firstBookKeeperEntry < 0
+ || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
+ throw new IOException("Invalid first bk entry to read: "
+ + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
+ }
BufferedInputStream bin = new BufferedInputStream(
new LedgerInputStream(lh, firstBookKeeperEntry));
tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
DataInputStream in = new DataInputStream(tracker);
- reader = new FSEditLogOp.Reader(in, logVersion);
+ reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
@Override
- public long getFirstTxId() throws IOException {
+ public long getFirstTxId() {
return firstTxId;
}
@Override
- public long getLastTxId() throws IOException {
+ public long getLastTxId() {
return lastTxId;
}
@@ -102,8 +110,10 @@ class BookKeeperEditLogInputStream exten
public void close() throws IOException {
try {
lh.close();
- } catch (Exception e) {
+ } catch (BKException e) {
throw new IOException("Exception closing ledger", e);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted closing ledger", e);
}
}
@@ -123,10 +133,28 @@ class BookKeeperEditLogInputStream exten
lh.toString(), firstTxId, lastTxId);
}
- // TODO(HA): Test this.
@Override
public boolean isInProgress() {
- return true;
+ return inProgress;
+ }
+
+ /**
+ * Skip forward to specified transaction id.
+ * Currently we do this by just iterating forward.
+ * If this proves to be too expensive, this can be reimplemented
+ * with a binary search over bk entries
+ */
+ public void skipTo(long txId) throws IOException {
+ long numToSkip = getFirstTxId() - txId;
+
+ FSEditLogOp op = null;
+ for (long i = 0; i < numToSkip; i++) {
+ op = readOp();
+ }
+ if (op != null && op.getTransactionId() != txId-1) {
+ throw new IOException("Corrupt stream, expected txid "
+ + (txId-1) + ", got " + op.getTransactionId());
+ }
}
/**
@@ -148,11 +176,8 @@ class BookKeeperEditLogInputStream exten
throws IOException {
this.lh = lh;
readEntries = firstBookKeeperEntry;
- try {
- maxEntry = lh.getLastAddConfirmed();
- } catch (Exception e) {
- throw new IOException("Error reading last entry id", e);
- }
+
+ maxEntry = lh.getLastAddConfirmed();
}
/**
@@ -173,8 +198,10 @@ class BookKeeperEditLogInputStream exten
assert !entries.hasMoreElements();
return e.getEntryInputStream();
}
- } catch (Exception e) {
+ } catch (BKException e) {
throw new IOException("Error reading entries from bookkeeper", e);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted reading entries from bookkeeper", e);
}
return null;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java Wed Jun 6 00:17:38 2012
@@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.io.DataOutputBuffer;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* Output stream for BookKeeper Journal.
* Multiple complete edit log entries are packed into a single bookkeeper
@@ -44,20 +47,22 @@ import java.io.IOException;
*/
class BookKeeperEditLogOutputStream
extends EditLogOutputStream implements AddCallback {
+ static final Log LOG = LogFactory.getLog(BookKeeperEditLogOutputStream.class);
+
private final DataOutputBuffer bufCurrent;
private final AtomicInteger outstandingRequests;
private final int transmissionThreshold;
private final LedgerHandle lh;
private CountDownLatch syncLatch;
- private final WriteLock wl;
+ private final AtomicInteger transmitResult
+ = new AtomicInteger(BKException.Code.OK);
private final Writer writer;
/**
* Construct an edit log output stream which writes to a ledger.
*/
- protected BookKeeperEditLogOutputStream(Configuration conf,
- LedgerHandle lh, WriteLock wl)
+ protected BookKeeperEditLogOutputStream(Configuration conf, LedgerHandle lh)
throws IOException {
super();
@@ -65,8 +70,6 @@ class BookKeeperEditLogOutputStream
outstandingRequests = new AtomicInteger(0);
syncLatch = null;
this.lh = lh;
- this.wl = wl;
- this.wl.acquire();
this.writer = new Writer(bufCurrent);
this.transmissionThreshold
= conf.getInt(BookKeeperJournalManager.BKJM_OUTPUT_BUFFER_SIZE,
@@ -101,7 +104,6 @@ class BookKeeperEditLogOutputStream
throw new IOException("BookKeeper error during abort", bke);
}
- wl.release();
}
@Override
@@ -111,8 +113,6 @@ class BookKeeperEditLogOutputStream
@Override
public void write(FSEditLogOp op) throws IOException {
- wl.checkWriteLock();
-
writer.writeOp(op);
if (bufCurrent.getLength() > transmissionThreshold) {
@@ -122,25 +122,26 @@ class BookKeeperEditLogOutputStream
@Override
public void setReadyToFlush() throws IOException {
- wl.checkWriteLock();
-
transmit();
- synchronized(this) {
+ synchronized (this) {
syncLatch = new CountDownLatch(outstandingRequests.get());
}
}
@Override
public void flushAndSync() throws IOException {
- wl.checkWriteLock();
-
assert(syncLatch != null);
try {
syncLatch.await();
} catch (InterruptedException ie) {
throw new IOException("Interrupted waiting on latch", ie);
}
+ if (transmitResult.get() != BKException.Code.OK) {
+ throw new IOException("Failed to write to bookkeeper; Error is ("
+ + transmitResult.get() + ") "
+ + BKException.getMessage(transmitResult.get()));
+ }
syncLatch = null;
// wait for whatever we wait on
@@ -152,8 +153,12 @@ class BookKeeperEditLogOutputStream
* are never called at the same time.
*/
private void transmit() throws IOException {
- wl.checkWriteLock();
-
+ if (!transmitResult.compareAndSet(BKException.Code.OK,
+ BKException.Code.OK)) {
+ throw new IOException("Trying to write to an errored stream;"
+ + " Error code : (" + transmitResult.get()
+ + ") " + BKException.getMessage(transmitResult.get()));
+ }
if (bufCurrent.getLength() > 0) {
byte[] entry = Arrays.copyOf(bufCurrent.getData(),
bufCurrent.getLength());
@@ -168,6 +173,12 @@ class BookKeeperEditLogOutputStream
long entryId, Object ctx) {
synchronized(this) {
outstandingRequests.decrementAndGet();
+ if (!transmitResult.compareAndSet(BKException.Code.OK, rc)) {
+ LOG.warn("Tried to set transmit result to (" + rc + ") \""
+ + BKException.getMessage(rc) + "\""
+ + " but is already (" + transmitResult.get() + ") \""
+ + BKException.getMessage(transmitResult.get()) + "\"");
+ }
CountDownLatch l = syncLatch;
if (l != null) {
l.countDown();
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Wed Jun 6 00:17:38 2012
@@ -37,6 +37,7 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.ZooDefs.Ids;
+import java.util.Collection;
import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
@@ -48,7 +49,7 @@ import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
+import com.google.common.annotations.VisibleForTesting;
/**
* BookKeeper Journal Manager
*
@@ -61,7 +62,7 @@ import org.apache.commons.logging.LogFac
* </property>
*
* <property>
- * <name>dfs.namenode.edits.journalPlugin.bookkeeper</name>
+ * <name>dfs.namenode.edits.journal-plugin.bookkeeper</name>
* <value>org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager</value>
* </property>
* }
@@ -90,6 +91,10 @@ import org.apache.commons.logging.LogFac
* Default is 2.</li>
* <li><b>dfs.namenode.bookkeeperjournal.digestPw</b>
* Password to use when creating ledgers. </li>
+ * <li><b>dfs.namenode.bookkeeperjournal.zk.session.timeout</b>
+ * Session timeout for Zookeeper client from BookKeeper Journal Manager.
+ * Hadoop recommends that, this value should be less than the ZKFC
+ * session timeout value. Default value is 3000.</li>
* </ul>
*/
public class BookKeeperJournalManager implements JournalManager {
@@ -112,11 +117,17 @@ public class BookKeeperJournalManager im
public static final String BKJM_BOOKKEEPER_DIGEST_PW_DEFAULT = "";
private static final int BKJM_LAYOUT_VERSION = -1;
+
+ public static final String BKJM_ZK_SESSION_TIMEOUT
+ = "dfs.namenode.bookkeeperjournal.zk.session.timeout";
+ public static final int BKJM_ZK_SESSION_TIMEOUT_DEFAULT = 3000;
+
+ private static final String BKJM_EDIT_INPROGRESS = "inprogress_";
- private final ZooKeeper zkc;
+ private ZooKeeper zkc;
private final Configuration conf;
private final BookKeeper bkc;
- private final WriteLock wl;
+ private final CurrentInprogress ci;
private final String ledgerPath;
private final MaxTxId maxTxId;
private final int ensembleSize;
@@ -154,14 +165,15 @@ public class BookKeeperJournalManager im
ledgerPath = zkPath + "/ledgers";
String maxTxIdPath = zkPath + "/maxtxid";
- String lockPath = zkPath + "/lock";
+ String currentInprogressNodePath = zkPath + "/CurrentInprogress";
String versionPath = zkPath + "/version";
digestpw = conf.get(BKJM_BOOKKEEPER_DIGEST_PW,
BKJM_BOOKKEEPER_DIGEST_PW_DEFAULT);
try {
zkConnectLatch = new CountDownLatch(1);
- zkc = new ZooKeeper(zkConnect, 3000, new ZkConnectionWatcher());
+ zkc = new ZooKeeper(zkConnect, conf.getInt(BKJM_ZK_SESSION_TIMEOUT,
+ BKJM_ZK_SESSION_TIMEOUT_DEFAULT), new ZkConnectionWatcher());
if (!zkConnectLatch.await(6000, TimeUnit.MILLISECONDS)) {
throw new IOException("Error connecting to zookeeper");
}
@@ -187,11 +199,14 @@ public class BookKeeperJournalManager im
bkc = new BookKeeper(new ClientConfiguration(),
zkc);
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Error initializing zk", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while initializing bk journal manager",
+ ie);
}
- wl = new WriteLock(zkc, lockPath);
+ ci = new CurrentInprogress(zkc, currentInprogressNodePath);
maxTxId = new MaxTxId(zkc, maxTxIdPath);
}
@@ -206,21 +221,34 @@ public class BookKeeperJournalManager im
*/
@Override
public EditLogOutputStream startLogSegment(long txId) throws IOException {
- wl.acquire();
-
if (txId <= maxTxId.get()) {
throw new IOException("We've already seen " + txId
+ ". A new stream cannot be created with it");
}
- if (currentLedger != null) {
- throw new IOException("Already writing to a ledger, id="
- + currentLedger.getId());
- }
+
try {
+ String existingInprogressNode = ci.read();
+ if (null != existingInprogressNode
+ && zkc.exists(existingInprogressNode, false) != null) {
+ throw new IOException("Inprogress node already exists");
+ }
+ if (currentLedger != null) {
+ // bookkeeper errored on last stream, clean up ledger
+ currentLedger.close();
+ }
currentLedger = bkc.createLedger(ensembleSize, quorumSize,
BookKeeper.DigestType.MAC,
digestpw.getBytes());
- String znodePath = inprogressZNode();
+ } catch (BKException bke) {
+ throw new IOException("Error creating ledger", bke);
+ } catch (KeeperException ke) {
+ throw new IOException("Error in zookeeper while creating ledger", ke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted creating ledger", ie);
+ }
+
+ try {
+ String znodePath = inprogressZNode(txId);
EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath,
HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId);
/* Write the ledger metadata out to the inprogress ledger znode
@@ -232,20 +260,30 @@ public class BookKeeperJournalManager im
*/
l.write(zkc, znodePath);
- return new BookKeeperEditLogOutputStream(conf, currentLedger, wl);
- } catch (Exception e) {
- if (currentLedger != null) {
- try {
- currentLedger.close();
- } catch (Exception e2) {
- //log & ignore, an IOException will be thrown soon
- LOG.error("Error closing ledger", e2);
- }
- }
- throw new IOException("Error creating ledger", e);
+ maxTxId.store(txId);
+ ci.update(znodePath);
+ return new BookKeeperEditLogOutputStream(conf, currentLedger);
+ } catch (KeeperException ke) {
+ cleanupLedger(currentLedger);
+ throw new IOException("Error storing ledger metadata", ke);
}
}
+ private void cleanupLedger(LedgerHandle lh) {
+ try {
+ long id = currentLedger.getId();
+ currentLedger.close();
+ bkc.deleteLedger(id);
+ } catch (BKException bke) {
+ //log & ignore, an IOException will be thrown soon
+ LOG.error("Error closing ledger", bke);
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted while closing ledger", ie);
+ }
+ }
+
+
+
/**
* Finalize a log segment. If the journal manager is currently
* writing to a ledger, ensure that this is the ledger of the log segment
@@ -258,7 +296,7 @@ public class BookKeeperJournalManager im
@Override
public void finalizeLogSegment(long firstTxId, long lastTxId)
throws IOException {
- String inprogressPath = inprogressZNode();
+ String inprogressPath = inprogressZNode(firstTxId);
try {
Stat inprogressStat = zkc.exists(inprogressPath, false);
if (inprogressStat == null) {
@@ -266,7 +304,6 @@ public class BookKeeperJournalManager im
+ " doesn't exist");
}
- wl.checkWriteLock();
EditLogLedgerMetadata l
= EditLogLedgerMetadata.read(zkc, inprogressPath);
@@ -303,55 +340,94 @@ public class BookKeeperJournalManager im
}
maxTxId.store(lastTxId);
zkc.delete(inprogressPath, inprogressStat.getVersion());
+ String inprogressPathFromCI = ci.read();
+ if (inprogressPath.equals(inprogressPathFromCI)) {
+ ci.clear();
+ }
} catch (KeeperException e) {
throw new IOException("Error finalising ledger", e);
} catch (InterruptedException ie) {
throw new IOException("Error finalising ledger", ie);
- } finally {
- wl.release();
- }
+ }
}
- // TODO(HA): Handle inProgressOk
- @Override
- public EditLogInputStream getInputStream(long fromTxnId, boolean inProgressOk)
+ EditLogInputStream getInputStream(long fromTxId, boolean inProgressOk)
throws IOException {
- for (EditLogLedgerMetadata l : getLedgerList()) {
- if (l.getFirstTxId() == fromTxnId) {
+ for (EditLogLedgerMetadata l : getLedgerList(inProgressOk)) {
+ long lastTxId = l.getLastTxId();
+ if (l.isInProgress()) {
+ lastTxId = recoverLastTxId(l, false);
+ }
+
+ if (fromTxId >= l.getFirstTxId() && fromTxId <= lastTxId) {
try {
- LedgerHandle h = bkc.openLedger(l.getLedgerId(),
- BookKeeper.DigestType.MAC,
- digestpw.getBytes());
- return new BookKeeperEditLogInputStream(h, l);
- } catch (Exception e) {
- throw new IOException("Could not open ledger for " + fromTxnId, e);
+ LedgerHandle h;
+ if (l.isInProgress()) { // we don't want to fence the current journal
+ h = bkc.openLedgerNoRecovery(l.getLedgerId(),
+ BookKeeper.DigestType.MAC, digestpw.getBytes());
+ } else {
+ h = bkc.openLedger(l.getLedgerId(), BookKeeper.DigestType.MAC,
+ digestpw.getBytes());
+ }
+ BookKeeperEditLogInputStream s = new BookKeeperEditLogInputStream(h,
+ l);
+ s.skipTo(fromTxId);
+ return s;
+ } catch (BKException e) {
+ throw new IOException("Could not open ledger for " + fromTxId, e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted opening ledger for "
+ + fromTxId, ie);
}
}
}
- throw new IOException("No ledger for fromTxnId " + fromTxnId + " found.");
+ return null;
}
- // TODO(HA): Handle inProgressOk
@Override
- public long getNumberOfTransactions(long fromTxnId, boolean inProgressOk)
+ public void selectInputStreams(Collection<EditLogInputStream> streams,
+ long fromTxId, boolean inProgressOk) {
+ // NOTE: could probably be rewritten more efficiently
+ while (true) {
+ EditLogInputStream elis;
+ try {
+ elis = getInputStream(fromTxId, inProgressOk);
+ } catch (IOException e) {
+ LOG.error(e);
+ return;
+ }
+ if (elis == null) {
+ return;
+ }
+ streams.add(elis);
+ if (elis.getLastTxId() == HdfsConstants.INVALID_TXID) {
+ return;
+ }
+ fromTxId = elis.getLastTxId() + 1;
+ }
+ }
+
+ long getNumberOfTransactions(long fromTxId, boolean inProgressOk)
throws IOException {
long count = 0;
long expectedStart = 0;
- for (EditLogLedgerMetadata l : getLedgerList()) {
+ for (EditLogLedgerMetadata l : getLedgerList(inProgressOk)) {
+ long lastTxId = l.getLastTxId();
if (l.isInProgress()) {
- long endTxId = recoverLastTxId(l);
- if (endTxId == HdfsConstants.INVALID_TXID) {
+ lastTxId = recoverLastTxId(l, false);
+ if (lastTxId == HdfsConstants.INVALID_TXID) {
break;
}
- count += (endTxId - l.getFirstTxId()) + 1;
- break;
}
- if (l.getFirstTxId() < fromTxnId) {
+ assert lastTxId >= l.getFirstTxId();
+
+ if (lastTxId < fromTxId) {
continue;
- } else if (l.getFirstTxId() == fromTxnId) {
- count = (l.getLastTxId() - l.getFirstTxId()) + 1;
- expectedStart = l.getLastTxId() + 1;
+ } else if (l.getFirstTxId() <= fromTxId && lastTxId >= fromTxId) {
+ // we can start in the middle of a segment
+ count = (lastTxId - l.getFirstTxId()) + 1;
+ expectedStart = lastTxId + 1;
} else {
if (expectedStart != l.getFirstTxId()) {
if (count == 0) {
@@ -362,8 +438,8 @@ public class BookKeeperJournalManager im
break;
}
}
- count += (l.getLastTxId() - l.getFirstTxId()) + 1;
- expectedStart = l.getLastTxId() + 1;
+ count += (lastTxId - l.getFirstTxId()) + 1;
+ expectedStart = lastTxId + 1;
}
}
return count;
@@ -371,26 +447,50 @@ public class BookKeeperJournalManager im
@Override
public void recoverUnfinalizedSegments() throws IOException {
- wl.acquire();
-
synchronized (this) {
try {
- EditLogLedgerMetadata l
- = EditLogLedgerMetadata.read(zkc, inprogressZNode());
- long endTxId = recoverLastTxId(l);
- if (endTxId == HdfsConstants.INVALID_TXID) {
- LOG.error("Unrecoverable corruption has occurred in segment "
- + l.toString() + " at path " + inprogressZNode()
- + ". Unable to continue recovery.");
- throw new IOException("Unrecoverable corruption, please check logs.");
+ List<String> children = zkc.getChildren(ledgerPath, false);
+ for (String child : children) {
+ if (!child.startsWith(BKJM_EDIT_INPROGRESS)) {
+ continue;
+ }
+ String znode = ledgerPath + "/" + child;
+ EditLogLedgerMetadata l = EditLogLedgerMetadata.read(zkc, znode);
+ try {
+ long endTxId = recoverLastTxId(l, true);
+ if (endTxId == HdfsConstants.INVALID_TXID) {
+ LOG.error("Unrecoverable corruption has occurred in segment "
+ + l.toString() + " at path " + znode
+ + ". Unable to continue recovery.");
+ throw new IOException("Unrecoverable corruption,"
+ + " please check logs.");
+ }
+ finalizeLogSegment(l.getFirstTxId(), endTxId);
+ } catch (SegmentEmptyException see) {
+ LOG.warn("Inprogress znode " + child
+ + " refers to a ledger which is empty. This occurs when the NN"
+ + " crashes after opening a segment, but before writing the"
+ + " OP_START_LOG_SEGMENT op. It is safe to delete."
+ + " MetaData [" + l.toString() + "]");
+
+ // If the max seen transaction is the same as what would
+ // have been the first transaction of the failed ledger,
+ // decrement it, as that transaction never happened and as
+ // such, is _not_ the last seen
+ if (maxTxId.get() == l.getFirstTxId()) {
+ maxTxId.reset(maxTxId.get() - 1);
+ }
+
+ zkc.delete(znode, -1);
+ }
}
- finalizeLogSegment(l.getFirstTxId(), endTxId);
} catch (KeeperException.NoNodeException nne) {
// nothing to recover, ignore
- } finally {
- if (wl.haveLock()) {
- wl.release();
- }
+ } catch (KeeperException ke) {
+ throw new IOException("Couldn't get list of inprogress segments", ke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted getting list of inprogress segments",
+ ie);
}
}
}
@@ -398,9 +498,8 @@ public class BookKeeperJournalManager im
@Override
public void purgeLogsOlderThan(long minTxIdToKeep)
throws IOException {
- for (EditLogLedgerMetadata l : getLedgerList()) {
- if (!l.isInProgress()
- && l.getLastTxId() < minTxIdToKeep) {
+ for (EditLogLedgerMetadata l : getLedgerList(false)) {
+ if (l.getLastTxId() < minTxIdToKeep) {
try {
Stat stat = zkc.exists(l.getZkPath(), false);
zkc.delete(l.getZkPath(), stat.getVersion());
@@ -421,8 +520,10 @@ public class BookKeeperJournalManager im
try {
bkc.close();
zkc.close();
- } catch (Exception e) {
- throw new IOException("Couldn't close zookeeper client", e);
+ } catch (BKException bke) {
+ throw new IOException("Couldn't close bookkeeper client", bke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while closing journal manager", ie);
}
}
@@ -440,14 +541,34 @@ public class BookKeeperJournalManager im
* Find the id of the last edit log transaction writen to a edit log
* ledger.
*/
- private long recoverLastTxId(EditLogLedgerMetadata l) throws IOException {
+ private long recoverLastTxId(EditLogLedgerMetadata l, boolean fence)
+ throws IOException, SegmentEmptyException {
+ LedgerHandle lh = null;
+ try {
+ if (fence) {
+ lh = bkc.openLedger(l.getLedgerId(),
+ BookKeeper.DigestType.MAC,
+ digestpw.getBytes());
+ } else {
+ lh = bkc.openLedgerNoRecovery(l.getLedgerId(),
+ BookKeeper.DigestType.MAC,
+ digestpw.getBytes());
+ }
+ } catch (BKException bke) {
+ throw new IOException("Exception opening ledger for " + l, bke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted opening ledger for " + l, ie);
+ }
+
+ BookKeeperEditLogInputStream in = null;
+
try {
- LedgerHandle lh = bkc.openLedger(l.getLedgerId(),
- BookKeeper.DigestType.MAC,
- digestpw.getBytes());
long lastAddConfirmed = lh.getLastAddConfirmed();
- BookKeeperEditLogInputStream in
- = new BookKeeperEditLogInputStream(lh, l, lastAddConfirmed);
+ if (lastAddConfirmed == -1) {
+ throw new SegmentEmptyException();
+ }
+
+ in = new BookKeeperEditLogInputStream(lh, l, lastAddConfirmed);
long endTxId = HdfsConstants.INVALID_TXID;
FSEditLogOp op = in.readOp();
@@ -459,25 +580,41 @@ public class BookKeeperJournalManager im
op = in.readOp();
}
return endTxId;
- } catch (Exception e) {
- throw new IOException("Exception retreiving last tx id for ledger " + l,
- e);
+ } finally {
+ if (in != null) {
+ in.close();
+ }
}
}
/**
* Get a list of all segments in the journal.
*/
- private List<EditLogLedgerMetadata> getLedgerList() throws IOException {
+ List<EditLogLedgerMetadata> getLedgerList(boolean inProgressOk)
+ throws IOException {
List<EditLogLedgerMetadata> ledgers
= new ArrayList<EditLogLedgerMetadata>();
try {
List<String> ledgerNames = zkc.getChildren(ledgerPath, false);
- for (String n : ledgerNames) {
- ledgers.add(EditLogLedgerMetadata.read(zkc, ledgerPath + "/" + n));
+ for (String ledgerName : ledgerNames) {
+ if (!inProgressOk && ledgerName.contains(BKJM_EDIT_INPROGRESS)) {
+ continue;
+ }
+ String legderMetadataPath = ledgerPath + "/" + ledgerName;
+ try {
+ EditLogLedgerMetadata editLogLedgerMetadata = EditLogLedgerMetadata
+ .read(zkc, legderMetadataPath);
+ ledgers.add(editLogLedgerMetadata);
+ } catch (KeeperException.NoNodeException e) {
+ LOG.warn("ZNode: " + legderMetadataPath
+ + " might have finalized and deleted."
+ + " So ignoring NoNodeException.");
+ }
}
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Exception reading ledger list from zk", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted getting list of ledgers from zk", ie);
}
Collections.sort(ledgers, EditLogLedgerMetadata.COMPARATOR);
@@ -495,8 +632,13 @@ public class BookKeeperJournalManager im
/**
* Get the znode path for the inprogressZNode
*/
- String inprogressZNode() {
- return ledgerPath + "/inprogress";
+ String inprogressZNode(long startTxid) {
+ return ledgerPath + "/inprogress_" + Long.toString(startTxid, 16);
+ }
+
+ @VisibleForTesting
+ void setZooKeeper(ZooKeeper zk) {
+ this.zkc = zk;
}
/**
@@ -509,4 +651,7 @@ public class BookKeeperJournalManager im
}
}
}
+
+ private static class SegmentEmptyException extends IOException {
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java Wed Jun 6 00:17:38 2012
@@ -130,8 +130,10 @@ public class EditLogLedgerMetadata {
}
} catch(KeeperException.NoNodeException nne) {
throw nne;
- } catch(Exception e) {
- throw new IOException("Error reading from zookeeper", e);
+ } catch(KeeperException ke) {
+ throw new IOException("Error reading from zookeeper", ke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted reading from zookeeper", ie);
}
}
@@ -147,13 +149,15 @@ public class EditLogLedgerMetadata {
version, ledgerId, firstTxId, lastTxId);
}
try {
- zkc.create(path, finalisedData.getBytes(),
- Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ zkc.create(path, finalisedData.getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
} catch (KeeperException.NodeExistsException nee) {
throw nee;
- } catch (Exception e) {
- throw new IOException("Error creating ledger znode");
- }
+ } catch (KeeperException e) {
+ throw new IOException("Error creating ledger znode", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted creating ledger znode", ie);
+ }
}
boolean verify(ZooKeeper zkc, String path) {
@@ -163,10 +167,13 @@ public class EditLogLedgerMetadata {
LOG.trace("Verifying " + this.toString()
+ " against " + other);
}
- return other == this;
- } catch (Exception e) {
+ return other.equals(this);
+ } catch (KeeperException e) {
LOG.error("Couldn't verify data in " + path, e);
return false;
+ } catch (IOException ie) {
+ LOG.error("Couldn't verify data in " + path, ie);
+ return false;
}
}
@@ -181,12 +188,12 @@ public class EditLogLedgerMetadata {
&& version == ol.version;
}
- public int hashCode() {
+ public int hashCode() {
int hash = 1;
- hash = hash * 31 + (int)ledgerId;
- hash = hash * 31 + (int)firstTxId;
- hash = hash * 31 + (int)lastTxId;
- hash = hash * 31 + (int)version;
+ hash = hash * 31 + (int) ledgerId;
+ hash = hash * 31 + (int) firstTxId;
+ hash = hash * 31 + (int) lastTxId;
+ hash = hash * 31 + (int) version;
return hash;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java?rev=1346682&r1=1346681&r2=1346682&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java Wed Jun 6 00:17:38 2012
@@ -18,13 +18,14 @@
package org.apache.hadoop.contrib.bkjournal;
import java.io.IOException;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.data.Stat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.Stat;
/**
* Utility class for storing and reading
@@ -49,18 +50,24 @@ class MaxTxId {
if (LOG.isTraceEnabled()) {
LOG.trace("Setting maxTxId to " + maxTxId);
}
- String txidStr = Long.toString(maxTxId);
- try {
- if (currentStat != null) {
- currentStat = zkc.setData(path, txidStr.getBytes("UTF-8"),
- currentStat.getVersion());
- } else {
- zkc.create(path, txidStr.getBytes("UTF-8"),
- Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- } catch (Exception e) {
- throw new IOException("Error writing max tx id", e);
+ reset(maxTxId);
+ }
+ }
+
+ synchronized void reset(long maxTxId) throws IOException {
+ String txidStr = Long.toString(maxTxId);
+ try {
+ if (currentStat != null) {
+ currentStat = zkc.setData(path, txidStr.getBytes("UTF-8"), currentStat
+ .getVersion());
+ } else {
+ zkc.create(path, txidStr.getBytes("UTF-8"), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
}
+ } catch (KeeperException e) {
+ throw new IOException("Error writing max tx id", e);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted while writing max tx id", e);
}
}
@@ -74,8 +81,10 @@ class MaxTxId {
String txidString = new String(bytes, "UTF-8");
return Long.valueOf(txidString);
}
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Error reading the max tx id from zk", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while reading thr max tx id", ie);
}
}
}