You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2014/03/05 00:53:29 UTC
svn commit: r1574259 [1/7] - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/
src/main/java/ sr...
Author: szetszwo
Date: Tue Mar 4 23:53:26 2014
New Revision: 1574259
URL: http://svn.apache.org/r1574259
Log:
svn merge --reintegrate https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-5535 back to trunk.
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeLocalInfo.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeLocalInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeException.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeException.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeInfo.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeStatus.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/RollingUpgradeStatus.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeStartupOptions.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeStartupOptions.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java
- copied unchanged from r1574256, hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/IncorrectVersionException.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartupOptionUpgrade.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs:r1550130-1574256
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Mar 4 23:53:26 2014
@@ -250,6 +250,148 @@ Trunk (Unreleased)
HDFS-5794. Fix the inconsistency of layout version number of
ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9)
+BREAKDOWN OF HDFS-5535 ROLLING UPGRADE SUBTASKS AND RELATED JIRAS
+
+ HDFS-5496. Make replication queue initialization asynchronous. (Vinay via
+ jing9)
+
+ HDFS-5645. Support upgrade marker in editlog streams. (szetszwo)
+
+ HDFS-5752. Add a new DFSAdmin command to query, start and finalize rolling
+ upgrade. (szetszwo)
+
+ HDFS-5786. Support QUERY and FINALIZE actions of rolling upgrade. (szetszwo)
+
+ HDFS-5753. Add new Namenode startup options for downgrade and rollback using
+ upgrade marker. (szetszwo)
+
+ HDFS-5835. Add a new option for starting Namenode when rolling upgrade is
+ in progress. (szetszwo)
+
+ HDFS-5754. Split LayoutVerion into NameNodeLayoutVersion and
+ DataNodeLayoutVersion. (Brandon Li via szetszwo)
+
+ HDFS-5848. Add rolling upgrade status to heartbeat response. (szetszwo)
+
+ HDFS-5890. Avoid NPE in Datanode heartbeat. (Vinay via brandonli)
+
+ HDFS-5869. When starting rolling upgrade or NN restarts, NN should create
+ a checkpoint right before the upgrade marker. (szetszwo)
+
+ HDFS-5874. Should not compare DataNode current layout version with that of
+ NameNode in DataStrorage. (brandonli)
+
+ HDFS-5889. When starting rolling upgrade, create a fs image for rollback
+ so that the standby namenode can create checkpoints during upgrade.
+ (szetszwo & jing9)
+
+ HDFS-5907. Add BlockPoolSliceStorage 'trash' to handle block deletions
+ during rolling upgrades. (Arpit Agarwal)
+
+ HDFS-5494. Merge Protobuf-based-FSImage code from trunk - fix build
+ break after the merge. (Jing Zhao via Arpit Agarwal)
+
+ HDFS-5585. Provide admin commands for data node upgrade (kihwal)
+
+ HDFS-5920. Support rollback of rolling upgrade in NameNode and JournalNodes.
+ (jing9)
+
+ HDFS-5945. Add rolling upgrade information to fsimage; and disallow upgrade
+ and rolling upgrade to be started simultaneously. (szetszwo & jing9)
+
+ HDFS-5966. Fix rollback of rolling upgrade in NameNode HA setup. (jing9
+ via szetszwo)
+
+ HDFS-5974. Fix compilation error, NameNodeLayoutVersion and
+ DataNodeLayoutVersion after merge from trunk. (szetszwo)
+
+ HDFS-5963. TestRollingUpgrade#testSecondaryNameNode causes subsequent
+ tests to fail. (szetszwo via Arpit Agarwal)
+
+ HDFS-5976. Create unit tests for downgrade and finalize rolling upgrade.
+ (Haohui Mai via Arpit Agarwal)
+
+ HDFS-5980. Rollback does not need to load edits. (jing9 via szetszwo)
+
+ HDFS-5984. Fix TestEditLog and TestStandbyCheckpoints. (jing9 via szetszwo)
+
+ HDFS-5985. SimulatedFSDataset#disableAndPurgeTrashStorage should not throw
+ UnsupportedOperationException. (jing9 via kihwal)
+
+ HDFS-5987. Fix findbugs warnings in Rolling Upgrade branch. (seztszwo via
+ Arpit Agarwal)
+
+ HDFS-5992. Fix NPE in MD5FileUtils and update editsStored for
+ TestOfflineEditsViewer. (szetszwo)
+
+ HDFS-5994. Fix TestDataNodeRollingUpgrade. (Arpit Agarwal via szetszwo)
+
+ HDFS-5999. Do not create rollback fsimage when it already exists. (jing9)
+
+ HDFS-6005. Simplify Datanode rollback and downgrade. (Suresh Srinivas via
+ Arpit Agarwal)
+
+ HDFS-6004. Change DFSAdmin for rolling upgrade commands. (szetszwo via
+ Arpit Agarwal)
+
+ HDFS-5583. Make DN send an OOB Ack on shutdown before restarting. (kihwal)
+
+ HDFS-5778. Add rolling upgrade user document. (szetszwo)
+
+ HDFS-6003. Add the new -rollingUpgrade startup option to the namenode
+ usage message. (Vinayakumar B via szetszwo)
+
+ HDFS-6014. Fix findbug warnings introduced by HDFS-5583. (kihwal)
+
+ HDFS-6015. Fix TestBlockRecovery
+ #testRaceBetweenReplicaRecoveryAndFinalizeBlock. (kihwal)
+
+ HDFS-5924. Utilize OOB upgrade message processing for writes. (kihwal)
+
+ HDFS-5498. Improve datanode startup time. (kihwal)
+
+ HDFS-6000. Avoid saving namespace when starting rolling upgrade. (jing9)
+
+ HDFS-6017. Query the status of rolling upgrade in the preparation stage in
+ TestRollingUpgrade and TestRollingUpgradeRollback. (Haohui Mai via
+ Arpit Agarwal)
+
+ HDFS-6020. Fix the five findbugs warnings. (kihwal)
+
+ HDFS-6019. Standby NN might not checkpoint when processing the rolling
+ upgrade marker. (Haohui Mai via jing9)
+
+ HDFS-6023. Test whether the standby NN continues to checkpoint after the
+ prepare stage. (Haohui Mai via jing9)
+
+ HDFS-6024. Test whether the NN will reject the downgrade if it has a
+ fsimage from a newer release. (Haohui Mai via jing9)
+
+ HDFS-6026. Fix TestDFSUpgrade and TestDataNodeRollingUpgrade.
+ (jing9 via szetszwo)
+
+ HDFS-6029. Secondary NN fails to checkpoint after -rollingUpgrade prepare.
+ (jing9)
+
+ HDFS-6032. -rollingUpgrade query hits NPE after the NN restarts. (Haohui Mai
+ via jing9)
+
+ HDFS-6031. Add back the "-rollingUpgrade started" namenode startup option;
+ otherwise, namenode cannot start when the layout version is changed.
+ (szetszwo)
+
+ HDFS-6034. Use DataNodeLayoutVersion for DN registration check and do not
+ verify layout version if there is a rolling upgrade in progress. (szetszwo)
+
+ HDFS-6013. add rollingUpgrade information to latest UI.
+ (Vinayakumar B via wheat9)
+
+ HDFS-6042. Fix rolling upgrade documentation and error messages. (szetszwo
+ via Arpit Agarwal)
+
+ HDFS-6041. Downgrade/Finalize should rename the rollback image instead of
+ purging it. (jing9)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Tue Mar 4 23:53:26 2014
@@ -397,7 +397,7 @@ public class BookKeeperJournalManager im
try {
String znodePath = inprogressZNode(txId);
EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath,
- HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId);
+ HdfsConstants.NAMENODE_LAYOUT_VERSION, currentLedger.getId(), txId);
/* Write the ledger metadata out to the inprogress ledger znode
* This can fail if for some reason our write lock has
* expired (@see WriteLock) and another process has managed to
@@ -693,6 +693,11 @@ public class BookKeeperJournalManager im
}
@Override
+ public void discardSegments(long startTxId) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void close() throws IOException {
try {
bkc.close();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java Tue Mar 4 23:53:26 2014
@@ -68,7 +68,7 @@ public class TestBookKeeperEditLogStream
lh.close();
EditLogLedgerMetadata metadata = new EditLogLedgerMetadata("/foobar",
- HdfsConstants.LAYOUT_VERSION, lh.getId(), 0x1234);
+ HdfsConstants.NAMENODE_LAYOUT_VERSION, lh.getId(), 0x1234);
try {
new BookKeeperEditLogInputStream(lh, metadata, -1);
fail("Shouldn't get this far, should have thrown");
@@ -77,7 +77,7 @@ public class TestBookKeeperEditLogStream
}
metadata = new EditLogLedgerMetadata("/foobar",
- HdfsConstants.LAYOUT_VERSION, lh.getId(), 0x1234);
+ HdfsConstants.NAMENODE_LAYOUT_VERSION, lh.getId(), 0x1234);
try {
new BookKeeperEditLogInputStream(lh, metadata, 0);
fail("Shouldn't get this far, should have thrown");
Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1550130-1574256
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Tue Mar 4 23:53:26 2014
@@ -30,6 +30,8 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
@@ -136,11 +138,13 @@ import org.apache.hadoop.hdfs.protocol.E
import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
@@ -267,6 +271,7 @@ public class DFSClient implements java.i
final int getFileBlockStorageLocationsTimeoutMs;
final int retryTimesForGetLastBlockLength;
final int retryIntervalForGetLastBlockLength;
+ final long datanodeRestartTimeout;
final boolean useLegacyBlockReader;
final boolean useLegacyBlockReaderLocal;
@@ -417,6 +422,10 @@ public class DFSClient implements java.i
shortCircuitSharedMemoryWatcherInterruptCheckMs = conf.getInt(
DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS,
DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT);
+
+ datanodeRestartTimeout = conf.getLong(
+ DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
+ DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000;
}
private DataChecksum.Type getChecksumType(Configuration conf) {
@@ -2458,6 +2467,10 @@ public class DFSClient implements java.i
namenode.finalizeUpgrade();
}
+ RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException {
+ return namenode.rollingUpgrade(action);
+ }
+
/**
*/
@Deprecated
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Mar 4 23:53:26 2014
@@ -94,6 +94,10 @@ public class DFSConfigKeys extends Commo
public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 3000;
public static final String DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL = "dfs.client.write.exclude.nodes.cache.expiry.interval.millis";
public static final long DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT = 10 * 60 * 1000; // 10 minutes, in ms
+ public static final String DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY = "dfs.client.datanode-restart.timeout";
+ public static final long DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT = 30;
+ public static final String DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY = "dfs.datanode.restart.replica.expiration";
+ public static final long DFS_DATANODE_RESTART_REPLICA_EXPIRY_DEFAULT = 50;
public static final String DFS_NAMENODE_BACKUP_ADDRESS_KEY = "dfs.namenode.backup.address";
public static final String DFS_NAMENODE_BACKUP_ADDRESS_DEFAULT = "localhost:50100";
public static final String DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY = "dfs.namenode.backup.http-address";
@@ -225,6 +229,8 @@ public class DFSConfigKeys extends Commo
public static final boolean DFS_DATANODE_SYNCONCLOSE_DEFAULT = false;
public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive";
public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 4000;
+ public static final String DFS_DATANODE_OOB_TIMEOUT_KEY = "dfs.datanode.oob.timeout-ms";
+ public static final String DFS_DATANODE_OOB_TIMEOUT_DEFAULT = "1500,0,0,0"; // OOB_TYPE1, OOB_TYPE2, OOB_TYPE3, OOB_TYPE4
public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check";
public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true;
@@ -416,7 +422,9 @@ public class DFSConfigKeys extends Commo
public static final int DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT = 1000;
public static final String DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY = "dfs.corruptfilesreturned.max";
public static final int DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED = 500;
-
+ /* Maximum number of blocks to process for initializing replication queues */
+ public static final String DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT = "dfs.block.misreplication.processing.limit";
+ public static final int DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT = 10000;
public static final String DFS_CLIENT_READ_SHORTCIRCUIT_KEY = "dfs.client.read.shortcircuit";
public static final boolean DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT = false;
public static final String DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY = "dfs.client.read.shortcircuit.skip.checksum";
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Tue Mar 4 23:53:26 2014
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStream;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.BufferOverflowException;
@@ -335,6 +336,8 @@ public class DFSOutputStream extends FSO
private String[] favoredNodes;
volatile boolean hasError = false;
volatile int errorIndex = -1;
+ volatile int restartingNodeIndex = -1; // Restarting node index
+ private long restartDeadline = 0; // Deadline of DN restart
private BlockConstructionStage stage; // block construction stage
private long bytesSent = 0; // number of bytes that've been sent
@@ -471,7 +474,7 @@ public class DFSOutputStream extends FSO
try {
// process datanode IO errors if any
boolean doSleep = false;
- if (hasError && errorIndex>=0) {
+ if (hasError && (errorIndex >= 0 || restartingNodeIndex >= 0)) {
doSleep = processDatanodeError();
}
@@ -571,8 +574,12 @@ public class DFSOutputStream extends FSO
blockStream.flush();
} catch (IOException e) {
// HDFS-3398 treat primary DN is down since client is unable to
- // write to primary DN
- errorIndex = 0;
+ // write to primary DN. If a failed or restarting node has already
+ // been recorded by the responder, the following call will have no
+ // effect. Pipeline recovery can handle only one node error at a
+ // time. If the primary node fails again during the recovery, it
+ // will be taken out then.
+ tryMarkPrimaryDatanodeFailed();
throw e;
}
lastPacket = Time.now();
@@ -609,12 +616,16 @@ public class DFSOutputStream extends FSO
Thread.sleep(artificialSlowdown);
}
} catch (Throwable e) {
- DFSClient.LOG.warn("DataStreamer Exception", e);
+ // Log warning if there was a real error.
+ if (restartingNodeIndex == -1) {
+ DFSClient.LOG.warn("DataStreamer Exception", e);
+ }
if (e instanceof IOException) {
setLastException((IOException)e);
}
hasError = true;
- if (errorIndex == -1) { // not a datanode error
+ if (errorIndex == -1 && restartingNodeIndex == -1) {
+ // Not a datanode issue
streamerClosed = true;
}
}
@@ -694,6 +705,65 @@ public class DFSOutputStream extends FSO
}
}
+ // The following synchronized methods are used whenever
+ // errorIndex or restartingNodeIndex is set. This is because
+ // check & set needs to be atomic. Simply reading variables
+ // does not require a synchronization. When responder is
+ // not running (e.g. during pipeline recovery), there is no
+ // need to use these methods.
+
+ /** Set the error node index. Called by responder */
+ synchronized void setErrorIndex(int idx) {
+ errorIndex = idx;
+ }
+
+ /** Set the restarting node index. Called by responder */
+ synchronized void setRestartingNodeIndex(int idx) {
+ restartingNodeIndex = idx;
+ // If the data streamer has already set the primary node
+ // bad, clear it. It is likely that the write failed due to
+ // the DN shutdown. Even if it was a real failure, the pipeline
+ // recovery will take care of it.
+ errorIndex = -1;
+ }
+
+ /**
+ * This method is used when no explicit error report was received,
+ * but something failed. When the primary node is a suspect or
+ * unsure about the cause, the primary node is marked as failed.
+ */
+ synchronized void tryMarkPrimaryDatanodeFailed() {
+ // There should be no existing error and no ongoing restart.
+ if ((errorIndex == -1) && (restartingNodeIndex == -1)) {
+ errorIndex = 0;
+ }
+ }
+
+ /**
+ * Examine whether it is worth waiting for a node to restart.
+ * @param index the node index
+ */
+ boolean shouldWaitForRestart(int index) {
+ // Only one node in the pipeline.
+ if (nodes.length == 1) {
+ return true;
+ }
+
+ // Is it a local node?
+ InetAddress addr = null;
+ try {
+ addr = InetAddress.getByName(nodes[index].getIpAddr());
+ } catch (java.net.UnknownHostException e) {
+ // we are passing an ip address. this should not happen.
+ assert false;
+ }
+
+ if (addr != null && NetUtils.isLocalAddress(addr)) {
+ return true;
+ }
+ return false;
+ }
+
//
// Processes responses from the datanodes. A packet is removed
// from the ackQueue when its response arrives.
@@ -727,8 +797,20 @@ public class DFSOutputStream extends FSO
// processes response status from datanodes.
for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) {
final Status reply = ack.getReply(i);
+ // Restart will not be treated differently unless it is
+ // the local node or the only one in the pipeline.
+ if (PipelineAck.isRestartOOBStatus(reply) &&
+ shouldWaitForRestart(i)) {
+ restartDeadline = dfsClient.getConf().datanodeRestartTimeout +
+ Time.now();
+ setRestartingNodeIndex(i);
+ String message = "A datanode is restarting: " + targets[i];
+ DFSClient.LOG.info(message);
+ throw new IOException(message);
+ }
+ // node error
if (reply != SUCCESS) {
- errorIndex = i; // first bad datanode
+ setErrorIndex(i); // first bad datanode
throw new IOException("Bad response " + reply +
" for block " + block +
" from datanode " +
@@ -777,12 +859,16 @@ public class DFSOutputStream extends FSO
setLastException((IOException)e);
}
hasError = true;
- errorIndex = errorIndex==-1 ? 0 : errorIndex;
+ // If no explicit error report was received, mark the primary
+ // node as failed.
+ tryMarkPrimaryDatanodeFailed();
synchronized (dataQueue) {
dataQueue.notifyAll();
}
- DFSClient.LOG.warn("DFSOutputStream ResponseProcessor exception "
- + " for block " + block, e);
+ if (restartingNodeIndex == -1) {
+ DFSClient.LOG.warn("DFSOutputStream ResponseProcessor exception "
+ + " for block " + block, e);
+ }
responderClosed = true;
}
}
@@ -1001,6 +1087,24 @@ public class DFSOutputStream extends FSO
boolean success = false;
long newGS = 0L;
while (!success && !streamerClosed && dfsClient.clientRunning) {
+ // Sleep before reconnect if a dn is restarting.
+ // This process will be repeated until the deadline or the datanode
+ // starts back up.
+ if (restartingNodeIndex >= 0) {
+ // 4 seconds or the configured deadline period, whichever is shorter.
+ // This is the retry interval and recovery will be retried in this
+ // interval until timeout or success.
+ long delay = Math.min(dfsClient.getConf().datanodeRestartTimeout,
+ 4000L);
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException ie) {
+ lastException.set(new IOException("Interrupted while waiting for " +
+ "datanode to restart. " + nodes[restartingNodeIndex]));
+ streamerClosed = true;
+ return false;
+ }
+ }
boolean isRecovery = hasError;
// remove bad datanode from list of datanodes.
// If errorIndex was not set (i.e. appends), then do not remove
@@ -1037,7 +1141,24 @@ public class DFSOutputStream extends FSO
setPipeline(newnodes, newStorageIDs);
- hasError = false;
+ // Just took care of a node error while waiting for a node restart
+ if (restartingNodeIndex >= 0) {
+ // If the error came from a node further away than the restarting
+ // node, the restart must have been complete.
+ if (errorIndex > restartingNodeIndex) {
+ restartingNodeIndex = -1;
+ } else if (errorIndex < restartingNodeIndex) {
+ // the node index has shifted.
+ restartingNodeIndex--;
+ } else {
+ // this shouldn't happen...
+ assert false;
+ }
+ }
+
+ if (restartingNodeIndex == -1) {
+ hasError = false;
+ }
lastException.set(null);
errorIndex = -1;
}
@@ -1066,7 +1187,34 @@ public class DFSOutputStream extends FSO
} else {
success = createBlockOutputStream(nodes, newGS, isRecovery);
}
- }
+
+ if (restartingNodeIndex >= 0) {
+ assert hasError == true;
+ // check errorIndex set above
+ if (errorIndex == restartingNodeIndex) {
+ // ignore, if came from the restarting node
+ errorIndex = -1;
+ }
+ // still within the deadline
+ if (Time.now() < restartDeadline) {
+ continue; // with in the deadline
+ }
+ // expired. declare the restarting node dead
+ restartDeadline = 0;
+ int expiredNodeIndex = restartingNodeIndex;
+ restartingNodeIndex = -1;
+ DFSClient.LOG.warn("Datanode did not restart in time: " +
+ nodes[expiredNodeIndex]);
+ // Mark the restarting node as failed. If there is any other failed
+ // node during the last pipeline construction attempt, it will not be
+ // overwritten/dropped. In this case, the restarting node will get
+ // excluded in the following attempt, if it still does not come up.
+ if (errorIndex == -1) {
+ errorIndex = expiredNodeIndex;
+ }
+ // From this point on, normal pipeline recovery applies.
+ }
+ } // while
if (success) {
// update pipeline at the namenode
@@ -1144,6 +1292,7 @@ public class DFSOutputStream extends FSO
}
Status pipelineStatus = SUCCESS;
String firstBadLink = "";
+ boolean checkRestart = false;
if (DFSClient.LOG.isDebugEnabled()) {
for (int i = 0; i < nodes.length; i++) {
DFSClient.LOG.debug("pipeline = " + nodes[i]);
@@ -1192,6 +1341,16 @@ public class DFSOutputStream extends FSO
pipelineStatus = resp.getStatus();
firstBadLink = resp.getFirstBadLink();
+ // Got an restart OOB ack.
+ // If a node is already restarting, this status is not likely from
+ // the same node. If it is from a different node, it is not
+ // from the local datanode. Thus it is safe to treat this as a
+ // regular node error.
+ if (PipelineAck.isRestartOOBStatus(pipelineStatus) &&
+ restartingNodeIndex == -1) {
+ checkRestart = true;
+ throw new IOException("A datanode is restarting.");
+ }
if (pipelineStatus != SUCCESS) {
if (pipelineStatus == Status.ERROR_ACCESS_TOKEN) {
throw new InvalidBlockTokenException(
@@ -1205,9 +1364,12 @@ public class DFSOutputStream extends FSO
assert null == blockStream : "Previous blockStream unclosed";
blockStream = out;
result = true; // success
-
+ restartingNodeIndex = -1;
+ hasError = false;
} catch (IOException ie) {
- DFSClient.LOG.info("Exception in createBlockOutputStream", ie);
+ if (restartingNodeIndex == -1) {
+ DFSClient.LOG.info("Exception in createBlockOutputStream", ie);
+ }
if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
DFSClient.LOG.info("Will fetch a new encryption key and retry, "
+ "encryption key was invalid when connecting to "
@@ -1230,8 +1392,18 @@ public class DFSOutputStream extends FSO
}
}
} else {
+ assert checkRestart == false;
errorIndex = 0;
}
+ // Check whether there is a restart worth waiting for.
+ if (checkRestart && shouldWaitForRestart(errorIndex)) {
+ restartDeadline = dfsClient.getConf().datanodeRestartTimeout +
+ Time.now();
+ restartingNodeIndex = errorIndex;
+ errorIndex = -1;
+ DFSClient.LOG.info("Waiting for the datanode to be restarted: " +
+ nodes[restartingNodeIndex]);
+ }
hasError = true;
setLastException(ie);
result = false; // error
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Tue Mar 4 23:53:26 2014
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.client.Hdf
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -68,11 +69,12 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
@@ -993,6 +995,14 @@ public class DistributedFileSystem exten
dfs.finalizeUpgrade();
}
+ /**
+ * Rolling upgrade: start/finalize/query.
+ */
+ public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
+ throws IOException {
+ return dfs.rollingUpgrade(action);
+ }
+
/*
* Requests the namenode to dump data strcutures into specified
* file.
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java Tue Mar 4 23:53:26 2014
@@ -125,4 +125,22 @@ public interface ClientDatanodeProtocol
*/
HdfsBlocksMetadata getHdfsBlocksMetadata(String blockPoolId,
long []blockIds, List<Token<BlockTokenIdentifier>> tokens) throws IOException;
+
+ /**
+ * Shuts down a datanode.
+ *
+ * @param forUpgrade If true, data node does extra prep work before shutting
+ * down. The work includes advising clients to wait and saving
+ * certain states for quick restart. This should only be used when
+ * the stored data will remain the same during upgrade/restart.
+ * @throws IOException
+ */
+ void shutdownDatanode(boolean forUpgrade) throws IOException;
+
+ /**
+ * Obtains datanode info
+ *
+ * @return software/config version and uptime of the datanode
+ */
+ DatanodeLocalInfo getDatanodeInfo() throws IOException;
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Tue Mar 4 23:53:26 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.permission.A
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
@@ -764,6 +765,15 @@ public interface ClientProtocol {
public void finalizeUpgrade() throws IOException;
/**
+ * Rolling upgrade operations.
+ * @param action either query, start or finailze.
+ * @return rolling upgrade information.
+ */
+ @Idempotent
+ public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
+ throws IOException;
+
+ /**
* @return CorruptFileBlocks, containing a list of corrupt files (with
* duplicates if there is more than one corrupt block in a file)
* and a cookie
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java Tue Mar 4 23:53:26 2014
@@ -17,11 +17,16 @@
*/
package org.apache.hadoop.hdfs.protocol;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
/************************************
@@ -79,6 +84,24 @@ public class HdfsConstants {
SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET;
}
+ public static enum RollingUpgradeAction {
+ QUERY, PREPARE, FINALIZE;
+
+ private static final Map<String, RollingUpgradeAction> MAP
+ = new HashMap<String, RollingUpgradeAction>();
+ static {
+ MAP.put("", QUERY);
+ for(RollingUpgradeAction a : values()) {
+ MAP.put(a.name(), a);
+ }
+ }
+
+ /** Covert the given String to a RollingUpgradeAction. */
+ public static RollingUpgradeAction fromString(String s) {
+ return MAP.get(s.toUpperCase());
+ }
+ }
+
// type of the datanode report
public static enum DatanodeReportType {
ALL, LIVE, DEAD
@@ -105,10 +128,18 @@ public class HdfsConstants {
/**
- * Please see {@link LayoutVersion} on adding new layout version.
+ * Current layout version for NameNode.
+ * Please see {@link NameNodeLayoutVersion.Feature} on adding new layout version.
+ */
+ public static final int NAMENODE_LAYOUT_VERSION
+ = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;
+
+ /**
+ * Current layout version for DataNode.
+ * Please see {@link DataNodeLayoutVersion.Feature} on adding new layout version.
*/
- public static final int LAYOUT_VERSION = LayoutVersion
- .getCurrentLayoutVersion();
+ public static final int DATANODE_LAYOUT_VERSION
+ = DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION;
/**
* Path components that are reserved in HDFS.
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java Tue Mar 4 23:53:26 2014
@@ -17,9 +17,10 @@
*/
package org.apache.hadoop.hdfs.protocol;
-import java.util.EnumSet;
-import java.util.HashMap;
+import java.util.Comparator;
import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,7 +44,6 @@ import org.apache.hadoop.classification.
*/
@InterfaceAudience.Private
public class LayoutVersion {
-
/**
* Version in which HDFS-2991 was fixed. This bug caused OP_ADD to
* sometimes be skipped for append() calls. If we see such a case when
@@ -54,19 +54,27 @@ public class LayoutVersion {
public static final int BUGFIX_HDFS_2991_VERSION = -40;
/**
- * Enums for features that change the layout version.
+ * The interface to be implemented by NameNode and DataNode layout features
+ */
+ public interface LayoutFeature {
+ public FeatureInfo getInfo();
+ }
+
+ /**
+ * Enums for features that change the layout version before rolling
+ * upgrade is supported.
* <br><br>
* To add a new layout version:
* <ul>
* <li>Define a new enum constant with a short enum name, the new layout version
* and description of the added feature.</li>
* <li>When adding a layout version with an ancestor that is not same as
- * its immediate predecessor, use the constructor where a spacific ancestor
+ * its immediate predecessor, use the constructor where a specific ancestor
* can be passed.
* </li>
* </ul>
*/
- public static enum Feature {
+ public static enum Feature implements LayoutFeature {
NAMESPACE_QUOTA(-16, "Support for namespace quotas"),
FILE_ACCESS_TIME(-17, "Support for access time on files"),
DISKSPACE_QUOTA(-18, "Support for disk space quotas"),
@@ -119,12 +127,8 @@ public class LayoutVersion {
RESERVED_REL2_4_0(-54, -51, "Reserved for release 2.4.0", true,
PROTOBUF_FORMAT, EXTENDED_ACL);
- final int lv;
- final int ancestorLV;
- final String description;
- final boolean reserved;
- final Feature[] specialFeatures;
-
+ private final FeatureInfo info;
+
/**
* Feature that is added at layout version {@code lv} - 1.
* @param lv new layout version with the addition of this feature
@@ -140,16 +144,35 @@ public class LayoutVersion {
* @param ancestorLV layout version from which the new lv is derived from.
* @param description description of the feature
* @param reserved true when this is a layout version reserved for previous
- * verions
+ * version
* @param features set of features that are to be enabled for this version
*/
Feature(final int lv, final int ancestorLV, final String description,
boolean reserved, Feature... features) {
+ info = new FeatureInfo(lv, ancestorLV, description, reserved, features);
+ }
+
+ @Override
+ public FeatureInfo getInfo() {
+ return info;
+ }
+ }
+
+ /** Feature information. */
+ public static class FeatureInfo {
+ private final int lv;
+ private final int ancestorLV;
+ private final String description;
+ private final boolean reserved;
+ private final LayoutFeature[] specialFeatures;
+
+ public FeatureInfo(final int lv, final int ancestorLV, final String description,
+ boolean reserved, LayoutFeature... specialFeatures) {
this.lv = lv;
this.ancestorLV = ancestorLV;
this.description = description;
this.reserved = reserved;
- specialFeatures = features;
+ this.specialFeatures = specialFeatures;
}
/**
@@ -179,80 +202,91 @@ public class LayoutVersion {
public boolean isReservedForOldRelease() {
return reserved;
}
+
+ public LayoutFeature[] getSpecialFeatures() {
+ return specialFeatures;
+ }
}
-
- // Build layout version and corresponding feature matrix
- static final Map<Integer, EnumSet<Feature>>map =
- new HashMap<Integer, EnumSet<Feature>>();
-
- // Static initialization
- static {
- initMap();
+
+ static class LayoutFeatureComparator implements Comparator<LayoutFeature> {
+ @Override
+ public int compare(LayoutFeature arg0, LayoutFeature arg1) {
+ return arg0.getInfo().getLayoutVersion()
+ - arg1.getInfo().getLayoutVersion();
+ }
}
-
- /**
- * Initialize the map of a layout version and EnumSet of {@link Feature}s
- * supported.
- */
- private static void initMap() {
+
+ public static void updateMap(Map<Integer, SortedSet<LayoutFeature>> map,
+ LayoutFeature[] features) {
// Go through all the enum constants and build a map of
- // LayoutVersion <-> EnumSet of all supported features in that LayoutVersion
- for (Feature f : Feature.values()) {
- EnumSet<Feature> ancestorSet = map.get(f.ancestorLV);
+ // LayoutVersion <-> Set of all supported features in that LayoutVersion
+ for (LayoutFeature f : features) {
+ final FeatureInfo info = f.getInfo();
+ SortedSet<LayoutFeature> ancestorSet = map.get(info.getAncestorLayoutVersion());
if (ancestorSet == null) {
- ancestorSet = EnumSet.noneOf(Feature.class); // Empty enum set
- map.put(f.ancestorLV, ancestorSet);
+ // Empty set
+ ancestorSet = new TreeSet<LayoutFeature>(new LayoutFeatureComparator());
+ map.put(info.getAncestorLayoutVersion(), ancestorSet);
}
- EnumSet<Feature> featureSet = EnumSet.copyOf(ancestorSet);
- if (f.specialFeatures != null) {
- for (Feature specialFeature : f.specialFeatures) {
+ SortedSet<LayoutFeature> featureSet = new TreeSet<LayoutFeature>(ancestorSet);
+ if (info.getSpecialFeatures() != null) {
+ for (LayoutFeature specialFeature : info.getSpecialFeatures()) {
featureSet.add(specialFeature);
}
}
featureSet.add(f);
- map.put(f.lv, featureSet);
+ map.put(info.getLayoutVersion(), featureSet);
}
}
/**
* Gets formatted string that describes {@link LayoutVersion} information.
*/
- public static String getString() {
+ public String getString(Map<Integer, SortedSet<LayoutFeature>> map,
+ LayoutFeature[] values) {
final StringBuilder buf = new StringBuilder();
buf.append("Feature List:\n");
- for (Feature f : Feature.values()) {
+ for (LayoutFeature f : values) {
+ final FeatureInfo info = f.getInfo();
buf.append(f).append(" introduced in layout version ")
- .append(f.lv).append(" (").
- append(f.description).append(")\n");
+ .append(info.getLayoutVersion()).append(" (")
+ .append(info.getDescription()).append(")\n");
}
-
+
buf.append("\n\nLayoutVersion and supported features:\n");
- for (Feature f : Feature.values()) {
- buf.append(f.lv).append(": ").append(map.get(f.lv))
- .append("\n");
+ for (LayoutFeature f : values) {
+ final FeatureInfo info = f.getInfo();
+ buf.append(info.getLayoutVersion()).append(": ")
+ .append(map.get(info.getLayoutVersion())).append("\n");
}
return buf.toString();
}
/**
* Returns true if a given feature is supported in the given layout version
+ * @param map layout feature map
* @param f Feature
* @param lv LayoutVersion
* @return true if {@code f} is supported in layout version {@code lv}
*/
- public static boolean supports(final Feature f, final int lv) {
- final EnumSet<Feature> set = map.get(lv);
+ public static boolean supports(Map<Integer, SortedSet<LayoutFeature>> map,
+ final LayoutFeature f, final int lv) {
+ final SortedSet<LayoutFeature> set = map.get(lv);
return set != null && set.contains(f);
}
/**
* Get the current layout version
*/
- public static int getCurrentLayoutVersion() {
- Feature[] values = Feature.values();
- for (int i = values.length -1; i >= 0; i--) {
- if (!values[i].isReservedForOldRelease()) {
- return values[i].lv;
+ public static int getCurrentLayoutVersion(LayoutFeature[] features) {
+ return getLastNonReservedFeature(features).getInfo().getLayoutVersion();
+ }
+
+ static LayoutFeature getLastNonReservedFeature(LayoutFeature[] features) {
+ for (int i = features.length -1; i >= 0; i--) {
+ final FeatureInfo info = features[i].getInfo();
+ if (!info.isReservedForOldRelease()) {
+ return features[i];
}
}
throw new AssertionError("All layout versions are reserved.");
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java Tue Mar 4 23:53:26 2014
@@ -26,10 +26,12 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_DEFAULT;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
-
import com.google.protobuf.TextFormat;
/** Pipeline Acknowledgment **/
@@ -38,6 +40,21 @@ import com.google.protobuf.TextFormat;
public class PipelineAck {
PipelineAckProto proto;
public final static long UNKOWN_SEQNO = -2;
+ final static int OOB_START = Status.OOB_RESTART_VALUE; // the first OOB type
+ final static int OOB_END = Status.OOB_RESERVED3_VALUE; // the last OOB type
+ final static int NUM_OOB_TYPES = OOB_END - OOB_START + 1;
+ // place holder for timeout value of each OOB type
+ final static long[] OOB_TIMEOUT;
+
+ static {
+ OOB_TIMEOUT = new long[NUM_OOB_TYPES];
+ HdfsConfiguration conf = new HdfsConfiguration();
+ String[] ele = conf.get(DFS_DATANODE_OOB_TIMEOUT_KEY,
+ DFS_DATANODE_OOB_TIMEOUT_DEFAULT).split(",");
+ for (int i = 0; i < NUM_OOB_TYPES; i++) {
+ OOB_TIMEOUT[i] = (i < ele.length) ? Long.valueOf(ele[i]) : 0;
+ }
+ }
/** default constructor **/
public PipelineAck() {
@@ -103,14 +120,57 @@ public class PipelineAck {
* @return true if all statuses are SUCCESS
*/
public boolean isSuccess() {
- for (DataTransferProtos.Status reply : proto.getStatusList()) {
- if (reply != DataTransferProtos.Status.SUCCESS) {
+ for (Status reply : proto.getStatusList()) {
+ if (reply != Status.SUCCESS) {
return false;
}
}
return true;
}
-
+
+ /**
+ * Returns the OOB status if this ack contains one.
+ * @return null if it is not an OOB ack.
+ */
+ public Status getOOBStatus() {
+ // Normal data transfer acks will have a valid sequence number, so
+ // this will return right away in most cases.
+ if (getSeqno() != UNKOWN_SEQNO) {
+ return null;
+ }
+ for (Status reply : proto.getStatusList()) {
+ // The following check is valid because protobuf guarantees to
+ // preserve the ordering of enum elements.
+ if (reply.getNumber() >= OOB_START && reply.getNumber() <= OOB_END) {
+ return reply;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get the timeout to be used for transmitting the OOB type
+ * @return the timeout in milliseconds
+ */
+ public static long getOOBTimeout(Status status) throws IOException {
+ int index = status.getNumber() - OOB_START;
+ if (index >= 0 && index < NUM_OOB_TYPES) {
+ return OOB_TIMEOUT[index];
+ }
+ // Not an OOB.
+ throw new IOException("Not an OOB status: " + status);
+ }
+
+ /** Get the Restart OOB ack status */
+ public static Status getRestartOOBStatus() {
+ return Status.OOB_RESTART;
+ }
+
+ /** return true if it is the restart OOB status code */
+ public static boolean isRestartOOBStatus(Status st) {
+ return st.equals(Status.OOB_RESTART);
+ }
+
/**** Writable interface ****/
public void readFields(InputStream in) throws IOException {
proto = PipelineAckProto.parseFrom(vintPrefixed(in));
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java Tue Mar 4 23:53:26 2014
@@ -30,6 +30,8 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto.Builder;
@@ -37,10 +39,13 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ShutdownDatanodeRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ShutdownDatanodeResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.VersionInfo;
import com.google.common.primitives.Longs;
import com.google.protobuf.ByteString;
@@ -59,6 +64,8 @@ public class ClientDatanodeProtocolServe
RefreshNamenodesResponseProto.newBuilder().build();
private final static DeleteBlockPoolResponseProto DELETE_BLOCKPOOL_RESP =
DeleteBlockPoolResponseProto.newBuilder().build();
+ private final static ShutdownDatanodeResponseProto SHUTDOWN_DATANODE_RESP =
+ ShutdownDatanodeResponseProto.newBuilder().build();
private final ClientDatanodeProtocol impl;
@@ -151,4 +158,28 @@ public class ClientDatanodeProtocolServe
builder.addAllVolumeIndexes(resp.getVolumeIndexes());
return builder.build();
}
+
+ @Override
+ public ShutdownDatanodeResponseProto shutdownDatanode(
+ RpcController unused, ShutdownDatanodeRequestProto request)
+ throws ServiceException {
+ try {
+ impl.shutdownDatanode(request.getForUpgrade());
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return SHUTDOWN_DATANODE_RESP;
+ }
+
+ public GetDatanodeInfoResponseProto getDatanodeInfo(RpcController unused,
+ GetDatanodeInfoRequestProto request) throws ServiceException {
+ GetDatanodeInfoResponseProto res;
+ try {
+ res = GetDatanodeInfoResponseProto.newBuilder()
+ .setLocalInfo(PBHelper.convert(impl.getDatanodeInfo())).build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return res;
+ }
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java?rev=1574259&r1=1574258&r2=1574259&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java Tue Mar 4 23:53:26 2014
@@ -35,16 +35,20 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetDatanodeInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ShutdownDatanodeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.ipc.ProtobufHelper;
@@ -81,6 +85,8 @@ public class ClientDatanodeProtocolTrans
private final ClientDatanodeProtocolPB rpcProxy;
private final static RefreshNamenodesRequestProto VOID_REFRESH_NAMENODES =
RefreshNamenodesRequestProto.newBuilder().build();
+ private final static GetDatanodeInfoRequestProto VOID_GET_DATANODE_INFO =
+ GetDatanodeInfoRequestProto.newBuilder().build();
public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid,
Configuration conf, int socketTimeout, boolean connectToDnViaHostname,
@@ -253,4 +259,27 @@ public class ClientDatanodeProtocolTrans
return new HdfsBlocksMetadata(blockPoolId, blockIds,
volumeIds, volumeIndexes);
}
+
+ @Override
+ public void shutdownDatanode(boolean forUpgrade) throws IOException {
+ ShutdownDatanodeRequestProto request = ShutdownDatanodeRequestProto
+ .newBuilder().setForUpgrade(forUpgrade).build();
+ try {
+ rpcProxy.shutdownDatanode(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public DatanodeLocalInfo getDatanodeInfo() throws IOException {
+ GetDatanodeInfoResponseProto response;
+ try {
+ response = rpcProxy.getDatanodeInfo(NULL_CONTROLLER, VOID_GET_DATANODE_INFO);
+ return PBHelper.convert(response.getLocalInfo());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
}