You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ar...@apache.org on 2013/12/12 08:17:58 UTC
svn commit: r1550363 [1/8] - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/protocolPB/ src/main/java/...
Author: arp
Date: Thu Dec 12 07:17:51 2013
New Revision: 1550363
URL: http://svn.apache.org/r1550363
Log:
svn merge --reintegrate https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832 for merging Heterogeneous Storage feature branch
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
- copied unchanged from r1550362, hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
- copied unchanged from r1550362, hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
- copied unchanged from r1550362, hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
- copied unchanged from r1550362, hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/MutableBlockCollection.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs:r1513717-1550362
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Dec 12 07:17:51 2013
@@ -13,6 +13,10 @@ Trunk (Unreleased)
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
+ HDFS-2832. Heterogeneous Storages support in HDFS phase 1 - treat DataNode
+ as a collection of storages (see breakdown of tasks below for features and
+ contributors).
+
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
@@ -440,6 +444,131 @@ Trunk (Unreleased)
HDFS-5626. dfsadmin -report shows incorrect cache values. (cmccabe)
+ BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS
+
+ HDFS-4985. Add storage type to the protocol and expose it in block report
+ and block locations. (Arpit Agarwal)
+
+ HDFS-5115. Make StorageID a UUID. (Arpit Agarwal)
+
+ HDFS-5000. DataNode configuration should allow specifying storage type.
+ (Arpit Agarwal)
+
+ HDFS-4987. Namenode changes to track multiple storages per datanode.
+ (szetszwo)
+
+ HDFS-5154. Fix TestBlockManager and TestDatanodeDescriptor after HDFS-4987.
+ (Junping Du via szetszwo)
+
+ HDFS-5009. Include storage information in the LocatedBlock. (szetszwo)
+
+ HDFS-5134. Move blockContentsStale, heartbeatedSinceFailover and
+ firstBlockReport from DatanodeDescriptor to DatanodeStorageInfo; and
+ fix a synchronization problem in DatanodeStorageInfo. (szetszwo)
+
+ HDFS-5157. Add StorageType to FsVolume. (Junping Du via szetszwo)
+
+ HDFS-4990. Change BlockPlacementPolicy to choose storages instead of
+ datanodes. (szetszwo)
+
+ HDFS-5232. Protocol changes to transmit StorageUuid. (Arpit Agarwal)
+
+ HDFS-5233. Use Datanode UUID to identify Datanodes. (Arpit Agarwal)
+
+ HDFS-5222. Move block schedule information from DatanodeDescriptor to
+ DatanodeStorageInfo. (szetszwo)
+
+ HDFS-4988. Datanode must support all the volumes as individual storages.
+ (Arpit Agarwal)
+
+ HDFS-5377. Heartbeats from Datandode should include one storage report
+ per storage directory. (Arpit Agarwal)
+
+ HDFS-5398. NameNode changes to process storage reports per storage
+ directory. (Arpit Agarwal)
+
+ HDFS-5390. Send one incremental block report per storage directory.
+ (Arpit Agarwal)
+
+ HDFS-5401. Fix NPE in Directory Scanner. (Arpit Agarwal)
+
+ HDFS-5417. Fix storage IDs in PBHelper and UpgradeUtilities. (szetszwo)
+
+ HDFS-5214. Fix NPEs in BlockManager and DirectoryScanner. (Arpit Agarwal)
+
+ HDFS-5435. File append fails to initialize storageIDs. (Junping Du via
+ Arpit Agarwal)
+
+ HDFS-5437. Fix TestBlockReport and TestBPOfferService failures. (Arpit
+ Agarwal)
+
+ HDFS-5447. Fix TestJspHelper. (Arpit Agarwal)
+
+ HDFS-5452. Fix TestReplicationPolicy and TestBlocksScheduledCounter.
+
+ HDFS-5448. Datanode should generate its ID on first registration. (Arpit
+ Agarwal)
+
+ HDFS-5448. Fix break caused by previous checkin for HDFS-5448. (Arpit
+ Agarwal)
+
+ HDFS-5455. NN should update storageMap on first heartbeat. (Arpit Agarwal)
+
+ HDFS-5457. Fix TestDatanodeRegistration, TestFsck and TestAddBlockRetry.
+ (Contributed by szetszwo)
+
+ HDFS-5466. Update storage IDs when the pipeline is updated. (Contributed
+ by szetszwo)
+
+ HDFS-5439. Fix TestPendingReplication. (Contributed by Junping Du, Arpit
+ Agarwal)
+
+ HDFS-5470. Add back trunk's reportDiff algorithm to the branch.
+ (Contributed by szetszwo)
+
+ HDFS-5472. Fix TestDatanodeManager, TestSafeMode and
+ TestNNThroughputBenchmark (Contributed by szetszwo)
+
+ HDFS-5475. NN incorrectly tracks more than one replica per DN. (Arpit
+ Agarwal)
+
+ HDFS-5481. Fix TestDataNodeVolumeFailure in branch HDFS-2832. (Contributed
+ by Junping Du)
+
+ HDFS-5480. Update Balancer for HDFS-2832. (Contributed by szetszwo)
+
+ HDFS-5486. Fix TestNameNodeMetrics for HDFS-2832. (Arpit Agarwal)
+
+ HDFS-5491. Update editsStored for HDFS-2832. (Arpit Agarwal)
+
+ HDFS-5494. Fix findbugs warnings for HDFS-2832. (Arpit Agarwal)
+
+ HDFS-5508. Fix compilation error after merge. (Contributed by szetszwo)
+
+ HDFS-5501. Fix pendingReceivedRequests tracking in BPServiceActor. (Arpit
+ Agarwal)
+
+ HDFS-5510. Fix a findbug warning in DataStorage.java on HDFS-2832 branch.
+ (Junping Du via Arpit Agarwal)
+
+ HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. (Arpit Agarwal)
+
+ HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. (Arpit
+ Agarwal)
+
+ HDFS-5547. Fix build break after merge from trunk to HDFS-2832. (Arpit
+ Agarwal)
+
+ HDFS-5542. Fix TODO and clean up the code in HDFS-2832. (Contributed by
+ szetszwo)
+
+ HDFS-5559. Fix TestDatanodeConfig in HDFS-2832. (Contributed by szetszwo)
+
+ HDFS-5484. StorageType and State in DatanodeStorageInfo in NameNode is
+ not accurate. (Eric Sirianni via Arpit Agarwal)
+
+ HDFS-5648. Get rid of FsDatasetImpl#perVolumeReplicaMap. (Arpit Agarwal)
+
Release 2.4.0 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1513717-1550362
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Thu Dec 12 07:17:51 2013
@@ -1347,7 +1347,7 @@ implements ByteBufferReadable, CanSetDro
* deadNodes and added currentNode again. Thats ok. */
deadNodes.remove(oldNode);
}
- if (!oldNode.getStorageID().equals(newNode.getStorageID())) {
+ if (!oldNode.getDatanodeUuid().equals(newNode.getDatanodeUuid())) {
currentNode = newNode;
return true;
} else {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Thu Dec 12 07:17:51 2013
@@ -312,6 +312,7 @@ public class DFSOutputStream extends FSO
private DataInputStream blockReplyStream;
private ResponseProcessor response = null;
private volatile DatanodeInfo[] nodes = null; // list of targets for current block
+ private volatile String[] storageIDs = null;
private LoadingCache<DatanodeInfo, DatanodeInfo> excludedNodes =
CacheBuilder.newBuilder()
.expireAfterWrite(
@@ -402,7 +403,7 @@ public class DFSOutputStream extends FSO
}
// setup pipeline to append to the last block XXX retries??
- nodes = lastBlock.getLocations();
+ setPipeline(lastBlock);
errorIndex = -1; // no errors yet.
if (nodes.length < 1) {
throw new IOException("Unable to retrieve blocks locations " +
@@ -411,6 +412,14 @@ public class DFSOutputStream extends FSO
}
}
+
+ private void setPipeline(LocatedBlock lb) {
+ setPipeline(lb.getLocations(), lb.getStorageIDs());
+ }
+ private void setPipeline(DatanodeInfo[] nodes, String[] storageIDs) {
+ this.nodes = nodes;
+ this.storageIDs = storageIDs;
+ }
private void setFavoredNodes(String[] favoredNodes) {
this.favoredNodes = favoredNodes;
@@ -434,7 +443,7 @@ public class DFSOutputStream extends FSO
this.setName("DataStreamer for file " + src);
closeResponder();
closeStream();
- nodes = null;
+ setPipeline(null, null);
stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
}
@@ -503,7 +512,7 @@ public class DFSOutputStream extends FSO
if(DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("Allocating new block");
}
- nodes = nextBlockOutputStream();
+ setPipeline(nextBlockOutputStream());
initDataStreaming();
} else if (stage == BlockConstructionStage.PIPELINE_SETUP_APPEND) {
if(DFSClient.LOG.isDebugEnabled()) {
@@ -917,9 +926,10 @@ public class DFSOutputStream extends FSO
//get a new datanode
final DatanodeInfo[] original = nodes;
final LocatedBlock lb = dfsClient.namenode.getAdditionalDatanode(
- src, block, nodes, failed.toArray(new DatanodeInfo[failed.size()]),
+ src, block, nodes, storageIDs,
+ failed.toArray(new DatanodeInfo[failed.size()]),
1, dfsClient.clientName);
- nodes = lb.getLocations();
+ setPipeline(lb);
//find the new datanode
final int d = findNewDatanode(original);
@@ -1019,7 +1029,14 @@ public class DFSOutputStream extends FSO
System.arraycopy(nodes, 0, newnodes, 0, errorIndex);
System.arraycopy(nodes, errorIndex+1, newnodes, errorIndex,
newnodes.length-errorIndex);
- nodes = newnodes;
+
+ final String[] newStorageIDs = new String[newnodes.length];
+ System.arraycopy(storageIDs, 0, newStorageIDs, 0, errorIndex);
+ System.arraycopy(storageIDs, errorIndex+1, newStorageIDs, errorIndex,
+ newStorageIDs.length-errorIndex);
+
+ setPipeline(newnodes, newStorageIDs);
+
hasError = false;
lastException.set(null);
errorIndex = -1;
@@ -1055,7 +1072,8 @@ public class DFSOutputStream extends FSO
// update pipeline at the namenode
ExtendedBlock newBlock = new ExtendedBlock(
block.getBlockPoolId(), block.getBlockId(), block.getNumBytes(), newGS);
- dfsClient.namenode.updatePipeline(dfsClient.clientName, block, newBlock, nodes);
+ dfsClient.namenode.updatePipeline(dfsClient.clientName, block, newBlock,
+ nodes, storageIDs);
// update client side generation stamp
block = newBlock;
}
@@ -1068,7 +1086,7 @@ public class DFSOutputStream extends FSO
* Must get block ID and the IDs of the destinations from the namenode.
* Returns the list of target datanodes.
*/
- private DatanodeInfo[] nextBlockOutputStream() throws IOException {
+ private LocatedBlock nextBlockOutputStream() throws IOException {
LocatedBlock lb = null;
DatanodeInfo[] nodes = null;
int count = dfsClient.getConf().nBlockWriteRetry;
@@ -1110,7 +1128,7 @@ public class DFSOutputStream extends FSO
if (!success) {
throw new IOException("Unable to create new block.");
}
- return nodes;
+ return lb;
}
// connects to the first datanode in the pipeline
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Thu Dec 12 07:17:51 2013
@@ -145,6 +145,23 @@ public class DFSUtil {
return SECURE_RANDOM.get();
}
+ /** Shuffle the elements in the given array. */
+ public static <T> T[] shuffle(final T[] array) {
+ if (array != null && array.length > 0) {
+ final Random random = getRandom();
+ for (int n = array.length; n > 1; ) {
+ final int randomIndex = random.nextInt(n);
+ n--;
+ if (n != randomIndex) {
+ final T tmp = array[randomIndex];
+ array[randomIndex] = array[n];
+ array[n] = tmp;
+ }
+ }
+ }
+ return array;
+ }
+
/**
* Compartor for sorting DataNodeInfo[] based on decommissioned states.
* Decommissioned nodes are moved to the end of the array on sorting with
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java Thu Dec 12 07:17:51 2013
@@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.protocol;
import java.util.Iterator;
import java.util.List;
+import java.util.Random;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -250,33 +252,28 @@ public class BlockListAsLongs implements
}
/**
- * The block-id of the indexTh block
- * @param index - the block whose block-id is desired
- * @return the block-id
- */
- @Deprecated
- public long getBlockId(final int index) {
- return blockId(index);
- }
-
- /**
- * The block-len of the indexTh block
- * @param index - the block whose block-len is desired
- * @return - the block-len
+ * Corrupt the generation stamp of the block with the given index.
+ * Not meant to be used outside of tests.
*/
- @Deprecated
- public long getBlockLen(final int index) {
- return blockLength(index);
+ @VisibleForTesting
+ public long corruptBlockGSForTesting(final int blockIndex, Random rand) {
+ long oldGS = blockList[index2BlockId(blockIndex) + 2];
+ while (blockList[index2BlockId(blockIndex) + 2] == oldGS) {
+ blockList[index2BlockId(blockIndex) + 2] = rand.nextInt();
+ }
+ return oldGS;
}
/**
- * The generation stamp of the indexTh block
- * @param index - the block whose block-len is desired
- * @return - the generation stamp
+ * Corrupt the length of the block with the given index by truncation.
+ * Not meant to be used outside of tests.
*/
- @Deprecated
- public long getBlockGenStamp(final int index) {
- return blockGenerationStamp(index);
+ @VisibleForTesting
+ public long corruptBlockLengthForTesting(final int blockIndex, Random rand) {
+ long oldLength = blockList[index2BlockId(blockIndex) + 1];
+ blockList[index2BlockId(blockIndex) + 1] =
+ rand.nextInt((int) oldLength - 1);
+ return oldLength;
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Thu Dec 12 07:17:51 2013
@@ -354,7 +354,8 @@ public interface ClientProtocol {
*/
@Idempotent
public LocatedBlock getAdditionalDatanode(final String src, final ExtendedBlock blk,
- final DatanodeInfo[] existings, final DatanodeInfo[] excludes,
+ final DatanodeInfo[] existings, final String[] existingStorageIDs,
+ final DatanodeInfo[] excludes,
final int numAdditionalNodes, final String clientName
) throws AccessControlException, FileNotFoundException,
SafeModeException, UnresolvedLinkException, IOException;
@@ -983,7 +984,7 @@ public interface ClientProtocol {
*/
@AtMostOnce
public void updatePipeline(String clientName, ExtendedBlock oldBlock,
- ExtendedBlock newBlock, DatanodeID[] newNodes)
+ ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
throws IOException;
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java Thu Dec 12 07:17:51 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.hdfs.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* This class represents the primary identifier for a Datanode.
* Datanodes are identified by how they can be contacted (hostname
@@ -40,37 +42,46 @@ public class DatanodeID implements Compa
private String ipAddr; // IP address
private String hostName; // hostname claimed by datanode
private String peerHostName; // hostname from the actual connection
- private String storageID; // unique per cluster storageID
private int xferPort; // data streaming port
private int infoPort; // info server port
private int infoSecurePort; // info server port
private int ipcPort; // IPC server port
+ /**
+ * UUID identifying a given datanode. For upgraded Datanodes this is the
+ * same as the StorageID that was previously used by this Datanode.
+ * For newly formatted Datanodes it is a UUID.
+ */
+ private String datanodeUuid = null;
+
public DatanodeID(DatanodeID from) {
this(from.getIpAddr(),
from.getHostName(),
- from.getStorageID(),
+ from.getDatanodeUuid(),
from.getXferPort(),
from.getInfoPort(),
from.getInfoSecurePort(),
from.getIpcPort());
this.peerHostName = from.getPeerHostName();
}
-
+
/**
* Create a DatanodeID
* @param ipAddr IP
* @param hostName hostname
- * @param storageID data storage ID
+ * @param datanodeUuid data node ID, UUID for new Datanodes, may be the
+ * storage ID for pre-UUID datanodes. NULL if unknown
+ * e.g. if this is a new datanode. A new UUID will
+ * be assigned by the namenode.
* @param xferPort data transfer port
* @param infoPort info server port
* @param ipcPort ipc server port
*/
- public DatanodeID(String ipAddr, String hostName, String storageID,
+ public DatanodeID(String ipAddr, String hostName, String datanodeUuid,
int xferPort, int infoPort, int infoSecurePort, int ipcPort) {
this.ipAddr = ipAddr;
this.hostName = hostName;
- this.storageID = storageID;
+ this.datanodeUuid = checkDatanodeUuid(datanodeUuid);
this.xferPort = xferPort;
this.infoPort = infoPort;
this.infoSecurePort = infoSecurePort;
@@ -85,8 +96,24 @@ public class DatanodeID implements Compa
this.peerHostName = peerHostName;
}
- public void setStorageID(String storageID) {
- this.storageID = storageID;
+ /**
+ * @return data node ID.
+ */
+ public String getDatanodeUuid() {
+ return datanodeUuid;
+ }
+
+ @VisibleForTesting
+ public void setDatanodeUuidForTesting(String datanodeUuid) {
+ this.datanodeUuid = datanodeUuid;
+ }
+
+ private String checkDatanodeUuid(String uuid) {
+ if (uuid == null || uuid.isEmpty()) {
+ return null;
+ } else {
+ return uuid;
+ }
}
/**
@@ -169,13 +196,6 @@ public class DatanodeID implements Compa
}
/**
- * @return data storage ID.
- */
- public String getStorageID() {
- return storageID;
- }
-
- /**
* @return xferPort (the port for data streaming)
*/
public int getXferPort() {
@@ -212,12 +232,12 @@ public class DatanodeID implements Compa
return false;
}
return (getXferAddr().equals(((DatanodeID)to).getXferAddr()) &&
- storageID.equals(((DatanodeID)to).getStorageID()));
+ datanodeUuid.equals(((DatanodeID)to).getDatanodeUuid()));
}
@Override
public int hashCode() {
- return getXferAddr().hashCode()^ storageID.hashCode();
+ return getXferAddr().hashCode()^ datanodeUuid.hashCode();
}
@Override
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Thu Dec 12 07:17:51 2013
@@ -115,7 +115,7 @@ public class DatanodeInfo extends Datano
final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
final long lastUpdate, final int xceiverCount,
final AdminStates adminState) {
- this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getStorageID(),
+ this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(),
nodeID.getXferPort(), nodeID.getInfoPort(), nodeID.getInfoSecurePort(),
nodeID.getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed,
cacheCapacity, cacheUsed, lastUpdate, xceiverCount, location,
@@ -124,13 +124,13 @@ public class DatanodeInfo extends Datano
/** Constructor */
public DatanodeInfo(final String ipAddr, final String hostName,
- final String storageID, final int xferPort, final int infoPort,
+ final String datanodeUuid, final int xferPort, final int infoPort,
final int infoSecurePort, final int ipcPort,
final long capacity, final long dfsUsed, final long remaining,
final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
final long lastUpdate, final int xceiverCount,
final String networkLocation, final AdminStates adminState) {
- super(ipAddr, hostName, storageID, xferPort, infoPort,
+ super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
infoSecurePort, ipcPort);
this.capacity = capacity;
this.dfsUsed = dfsUsed;
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java Thu Dec 12 07:17:51 2013
@@ -107,7 +107,10 @@ public class LayoutVersion {
"block IDs in the edits log and image files"),
EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to "
+ "enable rebuilding retry cache in case of HA failover"),
- CACHING(-48, "Support for cache pools and path-based caching");
+ CACHING(-48, "Support for cache pools and path-based caching"),
+ ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid."
+ + " Use distinct StorageUuid per storage directory.");
+
final int lv;
final int ancestorLV;
@@ -248,3 +251,4 @@ public class LayoutVersion {
throw new AssertionError("All layout versions are reserved.");
}
}
+
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java Thu Dec 12 07:17:51 2013
@@ -21,7 +21,9 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.security.token.Token;
import com.google.common.base.Preconditions;
@@ -40,6 +42,10 @@ public class LocatedBlock {
private ExtendedBlock b;
private long offset; // offset of the first byte of the block in the file
private DatanodeInfo[] locs;
+ /** Storage ID for each replica */
+ private String[] storageIDs;
+ // Storage type for each replica, if reported.
+ private StorageType[] storageTypes;
// corrupt flag is true if all of the replicas of a block are corrupt.
// else false. If block has few corrupt replicas, they are filtered and
// their locations are not part of this object
@@ -54,20 +60,34 @@ public class LocatedBlock {
private static final DatanodeInfo[] EMPTY_LOCS = new DatanodeInfo[0];
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) {
- this(b, locs, -1); // startOffset is unknown
- }
-
- public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset) {
- this(b, locs, startOffset, false);
+ this(b, locs, -1, false); // startOffset is unknown
}
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset,
boolean corrupt) {
- this(b, locs, startOffset, corrupt, EMPTY_LOCS);
+ this(b, locs, null, null, startOffset, corrupt, EMPTY_LOCS);
+ }
+
+ public LocatedBlock(ExtendedBlock b, DatanodeStorageInfo[] storages) {
+ this(b, storages, -1, false); // startOffset is unknown
}
- public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset,
- boolean corrupt, DatanodeInfo[] cachedLocs) {
+ public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs,
+ String[] storageIDs, StorageType[] storageTypes) {
+ this(b, locs, storageIDs, storageTypes, -1, false, EMPTY_LOCS);
+ }
+
+ public LocatedBlock(ExtendedBlock b, DatanodeStorageInfo[] storages,
+ long startOffset, boolean corrupt) {
+ this(b, DatanodeStorageInfo.toDatanodeInfos(storages),
+ DatanodeStorageInfo.toStorageIDs(storages),
+ DatanodeStorageInfo.toStorageTypes(storages),
+ startOffset, corrupt, EMPTY_LOCS); // startOffset is unknown
+ }
+
+ public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, String[] storageIDs,
+ StorageType[] storageTypes, long startOffset,
+ boolean corrupt, DatanodeInfo[] cachedLocs) {
this.b = b;
this.offset = startOffset;
this.corrupt = corrupt;
@@ -76,6 +96,8 @@ public class LocatedBlock {
} else {
this.locs = locs;
}
+ this.storageIDs = storageIDs;
+ this.storageTypes = storageTypes;
Preconditions.checkArgument(cachedLocs != null,
"cachedLocs should not be null, use a different constructor");
if (cachedLocs.length == 0) {
@@ -100,7 +122,15 @@ public class LocatedBlock {
public DatanodeInfo[] getLocations() {
return locs;
}
+
+ public StorageType[] getStorageTypes() {
+ return storageTypes;
+ }
+ public String[] getStorageIDs() {
+ return storageIDs;
+ }
+
public long getStartOffset() {
return offset;
}
@@ -161,3 +191,4 @@ public class LocatedBlock {
+ "}";
}
}
+
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java Thu Dec 12 07:17:51 2013
@@ -51,7 +51,7 @@ public class UnregisteredNodeException e
*/
public UnregisteredNodeException(DatanodeID nodeID, DatanodeInfo storedNode) {
super("Data node " + nodeID + " is attempting to report storage ID "
- + nodeID.getStorageID() + ". Node "
+ + nodeID.getDatanodeUuid() + ". Node "
+ storedNode + " is expected to serve this storage.");
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java Thu Dec 12 07:17:51 2013
@@ -425,14 +425,17 @@ public class ClientNamenodeProtocolServe
throws ServiceException {
try {
List<DatanodeInfoProto> existingList = req.getExistingsList();
+ List<String> existingStorageIDsList = req.getExistingStorageUuidsList();
List<DatanodeInfoProto> excludesList = req.getExcludesList();
- LocatedBlock result = server.getAdditionalDatanode(
- req.getSrc(), PBHelper.convert(req.getBlk()),
+ LocatedBlock result = server.getAdditionalDatanode(req.getSrc(),
+ PBHelper.convert(req.getBlk()),
PBHelper.convert(existingList.toArray(
new DatanodeInfoProto[existingList.size()])),
+ existingStorageIDsList.toArray(
+ new String[existingStorageIDsList.size()]),
PBHelper.convert(excludesList.toArray(
new DatanodeInfoProto[excludesList.size()])),
- req.getNumAdditionalNodes(), req.getClientName());
+ req.getNumAdditionalNodes(), req.getClientName());
return GetAdditionalDatanodeResponseProto.newBuilder().setBlock(
PBHelper.convert(result))
.build();
@@ -833,10 +836,12 @@ public class ClientNamenodeProtocolServe
UpdatePipelineRequestProto req) throws ServiceException {
try {
List<DatanodeIDProto> newNodes = req.getNewNodesList();
- server
- .updatePipeline(req.getClientName(), PBHelper.convert(req
- .getOldBlock()), PBHelper.convert(req.getNewBlock()), PBHelper
- .convert(newNodes.toArray(new DatanodeIDProto[newNodes.size()])));
+ List<String> newStorageIDs = req.getStorageIDsList();
+ server.updatePipeline(req.getClientName(),
+ PBHelper.convert(req.getOldBlock()),
+ PBHelper.convert(req.getNewBlock()),
+ PBHelper.convert(newNodes.toArray(new DatanodeIDProto[newNodes.size()])),
+ newStorageIDs.toArray(new String[newStorageIDs.size()]));
return VOID_UPDATEPIPELINE_RESPONSE;
} catch (IOException e) {
throw new ServiceException(e);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java Thu Dec 12 07:17:51 2013
@@ -351,7 +351,8 @@ public class ClientNamenodeProtocolTrans
@Override
public LocatedBlock getAdditionalDatanode(String src, ExtendedBlock blk,
- DatanodeInfo[] existings, DatanodeInfo[] excludes,
+ DatanodeInfo[] existings, String[] existingStorageIDs,
+ DatanodeInfo[] excludes,
int numAdditionalNodes, String clientName) throws AccessControlException,
FileNotFoundException, SafeModeException, UnresolvedLinkException,
IOException {
@@ -360,6 +361,7 @@ public class ClientNamenodeProtocolTrans
.setSrc(src)
.setBlk(PBHelper.convert(blk))
.addAllExistings(PBHelper.convert(existings))
+ .addAllExistingStorageUuids(Arrays.asList(existingStorageIDs))
.addAllExcludes(PBHelper.convert(excludes))
.setNumAdditionalNodes(numAdditionalNodes)
.setClientName(clientName)
@@ -796,12 +798,13 @@ public class ClientNamenodeProtocolTrans
@Override
public void updatePipeline(String clientName, ExtendedBlock oldBlock,
- ExtendedBlock newBlock, DatanodeID[] newNodes) throws IOException {
+ ExtendedBlock newBlock, DatanodeID[] newNodes, String[] storageIDs) throws IOException {
UpdatePipelineRequestProto req = UpdatePipelineRequestProto.newBuilder()
.setClientName(clientName)
.setOldBlock(PBHelper.convert(oldBlock))
.setNewBlock(PBHelper.convert(newBlock))
.addAllNewNodes(Arrays.asList(PBHelper.convert(newNodes)))
+ .addAllStorageIDs(storageIDs == null ? null : Arrays.asList(storageIDs))
.build();
try {
rpcProxy.updatePipeline(null, req);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java Thu Dec 12 07:17:51 2013
@@ -245,7 +245,7 @@ public class DatanodeProtocolClientSideT
for (StorageReceivedDeletedBlocks storageBlock : receivedAndDeletedBlocks) {
StorageReceivedDeletedBlocksProto.Builder repBuilder =
StorageReceivedDeletedBlocksProto.newBuilder();
- repBuilder.setStorageID(storageBlock.getStorageID());
+ repBuilder.setStorageUuid(storageBlock.getStorageID());
for (ReceivedDeletedBlockInfo rdBlock : storageBlock.getBlocks()) {
repBuilder.addBlocks(PBHelper.convert(rdBlock));
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java Thu Dec 12 07:17:51 2013
@@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReportBadBlocksResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageBlockReportProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto;
@@ -102,14 +101,8 @@ public class DatanodeProtocolServerSideT
HeartbeatRequestProto request) throws ServiceException {
HeartbeatResponse response;
try {
- List<StorageReportProto> list = request.getReportsList();
- StorageReport[] report = new StorageReport[list.size()];
- int i = 0;
- for (StorageReportProto p : list) {
- report[i++] = new StorageReport(p.getStorageID(), p.getFailed(),
- p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
- p.getBlockPoolUsed());
- }
+ final StorageReport[] report = PBHelper.convertStorageReports(
+ request.getReportsList());
response = impl.sendHeartbeat(PBHelper.convert(request.getRegistration()),
report, request.getCacheCapacity(), request.getCacheUsed(),
request.getXmitsInProgress(),
@@ -198,7 +191,7 @@ public class DatanodeProtocolServerSideT
for (int j = 0; j < list.size(); j++) {
rdBlocks[j] = PBHelper.convert(list.get(j));
}
- info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageID(), rdBlocks);
+ info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageUuid(), rdBlocks);
}
try {
impl.blockReceivedAndDeleted(PBHelper.convert(request.getRegistration()),
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java Thu Dec 12 07:17:51 2013
@@ -82,6 +82,6 @@ public class InterDatanodeProtocolServer
throw new ServiceException(e);
}
return UpdateReplicaUnderRecoveryResponseProto.newBuilder()
- .setStorageID(storageID).build();
+ .setStorageUuid(storageID).build();
}
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java Thu Dec 12 07:17:51 2013
@@ -109,7 +109,7 @@ public class InterDatanodeProtocolTransl
.setNewLength(newLength).setRecoveryId(recoveryId).build();
try {
return rpcProxy.updateReplicaUnderRecovery(NULL_CONTROLLER, req
- ).getStorageID();
+ ).getStorageUuid();
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Thu Dec 12 07:17:51 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats;
@@ -52,12 +53,12 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto;
@@ -122,6 +123,8 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
@@ -242,17 +245,20 @@ public class PBHelper {
// DatanodeId
public static DatanodeID convert(DatanodeIDProto dn) {
- return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getStorageID(),
+ return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(),
dn.getXferPort(), dn.getInfoPort(), dn.hasInfoSecurePort() ? dn
.getInfoSecurePort() : 0, dn.getIpcPort());
}
public static DatanodeIDProto convert(DatanodeID dn) {
+ // For wire compatibility with older versions we transmit the StorageID
+ // which is the same as the DatanodeUuid. Since StorageID is a required
+ // field we pass the empty string if the DatanodeUuid is not yet known.
return DatanodeIDProto.newBuilder()
.setIpAddr(dn.getIpAddr())
.setHostName(dn.getHostName())
- .setStorageID(dn.getStorageID())
.setXferPort(dn.getXferPort())
+ .setDatanodeUuid(dn.getDatanodeUuid() != null ? dn.getDatanodeUuid() : "")
.setInfoPort(dn.getInfoPort())
.setInfoSecurePort(dn.getInfoSecurePort())
.setIpcPort(dn.getIpcPort()).build();
@@ -294,12 +300,16 @@ public class PBHelper {
public static BlockWithLocationsProto convert(BlockWithLocations blk) {
return BlockWithLocationsProto.newBuilder()
.setBlock(convert(blk.getBlock()))
- .addAllStorageIDs(Arrays.asList(blk.getStorageIDs())).build();
+ .addAllDatanodeUuids(Arrays.asList(blk.getDatanodeUuids()))
+ .addAllStorageUuids(Arrays.asList(blk.getStorageIDs())).build();
}
public static BlockWithLocations convert(BlockWithLocationsProto b) {
- return new BlockWithLocations(convert(b.getBlock()), b.getStorageIDsList()
- .toArray(new String[0]));
+ final List<String> datanodeUuids = b.getDatanodeUuidsList();
+ final List<String> storageUuids = b.getStorageUuidsList();
+ return new BlockWithLocations(convert(b.getBlock()),
+ datanodeUuids.toArray(new String[datanodeUuids.size()]),
+ storageUuids.toArray(new String[storageUuids.size()]));
}
public static BlocksWithLocationsProto convert(BlocksWithLocations blks) {
@@ -601,6 +611,17 @@ public class PBHelper {
"Found additional cached replica locations that are not in the set of"
+ " storage-backed locations!");
+ StorageType[] storageTypes = b.getStorageTypes();
+ if (storageTypes != null) {
+ for (int i = 0; i < storageTypes.length; ++i) {
+ builder.addStorageTypes(PBHelper.convertStorageType(storageTypes[i]));
+ }
+ }
+ final String[] storageIDs = b.getStorageIDs();
+ if (storageIDs != null) {
+ builder.addAllStorageIDs(Arrays.asList(storageIDs));
+ }
+
return builder.setB(PBHelper.convert(b.getBlock()))
.setBlockToken(PBHelper.convert(b.getBlockToken()))
.setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build();
@@ -613,6 +634,25 @@ public class PBHelper {
for (int i = 0; i < locs.size(); i++) {
targets[i] = PBHelper.convert(locs.get(i));
}
+
+ final int storageTypesCount = proto.getStorageTypesCount();
+ final StorageType[] storageTypes;
+ if (storageTypesCount == 0) {
+ storageTypes = null;
+ } else {
+ Preconditions.checkState(storageTypesCount == locs.size());
+ storageTypes = convertStorageTypeProtos(proto.getStorageTypesList());
+ }
+
+ final int storageIDsCount = proto.getStorageIDsCount();
+ final String[] storageIDs;
+ if (storageIDsCount == 0) {
+ storageIDs = null;
+ } else {
+ Preconditions.checkState(storageIDsCount == locs.size());
+ storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]);
+ }
+
// Set values from the isCached list, re-using references from loc
List<DatanodeInfo> cachedLocs = new ArrayList<DatanodeInfo>(locs.size());
List<Boolean> isCachedList = proto.getIsCachedList();
@@ -623,7 +663,7 @@ public class PBHelper {
}
LocatedBlock lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets,
- proto.getOffset(), proto.getCorrupt(),
+ storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt(),
cachedLocs.toArray(new DatanodeInfo[0]));
lb.setBlockToken(PBHelper.convert(proto.getBlockToken()));
@@ -766,7 +806,8 @@ public class PBHelper {
for (int i = 0; i < blocks.length; i++) {
builder.addBlocks(PBHelper.convert(blocks[i]));
}
- builder.addAllTargets(PBHelper.convert(cmd.getTargets()));
+ builder.addAllTargets(convert(cmd.getTargets()))
+ .addAllTargetStorageUuids(convert(cmd.getTargetStorageIDs()));
return builder.build();
}
@@ -799,6 +840,15 @@ public class PBHelper {
return Arrays.asList(ret);
}
+ private static List<StorageUuidsProto> convert(String[][] targetStorageUuids) {
+ StorageUuidsProto[] ret = new StorageUuidsProto[targetStorageUuids.length];
+ for (int i = 0; i < targetStorageUuids.length; i++) {
+ ret[i] = StorageUuidsProto.newBuilder()
+ .addAllStorageUuids(Arrays.asList(targetStorageUuids[i])).build();
+ }
+ return Arrays.asList(ret);
+ }
+
public static DatanodeCommandProto convert(DatanodeCommand datanodeCommand) {
DatanodeCommandProto.Builder builder = DatanodeCommandProto.newBuilder();
if (datanodeCommand == null) {
@@ -878,6 +928,14 @@ public class PBHelper {
for (int i = 0; i < targetList.size(); i++) {
targets[i] = PBHelper.convert(targetList.get(i));
}
+
+ List<StorageUuidsProto> targetStorageUuidsList = blkCmd.getTargetStorageUuidsList();
+ String[][] targetStorageIDs = new String[targetStorageUuidsList.size()][];
+ for(int i = 0; i < targetStorageIDs.length; i++) {
+ List<String> storageIDs = targetStorageUuidsList.get(i).getStorageUuidsList();
+ targetStorageIDs[i] = storageIDs.toArray(new String[storageIDs.size()]);
+ }
+
int action = DatanodeProtocol.DNA_UNKNOWN;
switch (blkCmd.getAction()) {
case TRANSFER:
@@ -892,7 +950,8 @@ public class PBHelper {
default:
throw new AssertionError("Unknown action type: " + blkCmd.getAction());
}
- return new BlockCommand(action, blkCmd.getBlockPoolId(), blocks, targets);
+ return new BlockCommand(action, blkCmd.getBlockPoolId(), blocks, targets,
+ targetStorageIDs);
}
public static BlockIdCommand convert(BlockIdCommandProto blkIdCmd) {
@@ -1422,11 +1481,12 @@ public class PBHelper {
public static DatanodeStorageProto convert(DatanodeStorage s) {
return DatanodeStorageProto.newBuilder()
- .setState(PBHelper.convert(s.getState()))
- .setStorageID(s.getStorageID()).build();
+ .setState(PBHelper.convertState(s.getState()))
+ .setStorageType(PBHelper.convertStorageType(s.getStorageType()))
+ .setStorageUuid(s.getStorageID()).build();
}
- private static StorageState convert(State state) {
+ private static StorageState convertState(State state) {
switch(state) {
case READ_ONLY:
return StorageState.READ_ONLY;
@@ -1436,11 +1496,26 @@ public class PBHelper {
}
}
+ private static StorageTypeProto convertStorageType(
+ StorageType type) {
+ switch(type) {
+ case DISK:
+ return StorageTypeProto.DISK;
+ case SSD:
+ return StorageTypeProto.SSD;
+ default:
+ throw new IllegalStateException(
+ "BUG: StorageType not found, type=" + type);
+ }
+ }
+
public static DatanodeStorage convert(DatanodeStorageProto s) {
- return new DatanodeStorage(s.getStorageID(), PBHelper.convert(s.getState()));
+ return new DatanodeStorage(s.getStorageUuid(),
+ PBHelper.convertState(s.getState()),
+ PBHelper.convertType(s.getStorageType()));
}
- private static State convert(StorageState state) {
+ private static State convertState(StorageState state) {
switch(state) {
case READ_ONLY:
return DatanodeStorage.State.READ_ONLY;
@@ -1450,14 +1525,50 @@ public class PBHelper {
}
}
+ private static StorageType convertType(StorageTypeProto type) {
+ switch(type) {
+ case DISK:
+ return StorageType.DISK;
+ case SSD:
+ return StorageType.SSD;
+ default:
+ throw new IllegalStateException(
+ "BUG: StorageTypeProto not found, type=" + type);
+ }
+ }
+
+ private static StorageType[] convertStorageTypeProtos(
+ List<StorageTypeProto> storageTypesList) {
+ final StorageType[] storageTypes = new StorageType[storageTypesList.size()];
+ for (int i = 0; i < storageTypes.length; ++i) {
+ storageTypes[i] = PBHelper.convertType(storageTypesList.get(i));
+ }
+ return storageTypes;
+ }
+
public static StorageReportProto convert(StorageReport r) {
StorageReportProto.Builder builder = StorageReportProto.newBuilder()
.setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
.setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
- .setStorageID(r.getStorageID());
+ .setStorageUuid(r.getStorageID());
return builder.build();
}
+ public static StorageReport convert(StorageReportProto p) {
+ return new StorageReport(p.getStorageUuid(), p.getFailed(),
+ p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
+ p.getBlockPoolUsed());
+ }
+
+ public static StorageReport[] convertStorageReports(
+ List<StorageReportProto> list) {
+ final StorageReport[] report = new StorageReport[list.size()];
+ for (int i = 0; i < report.length; i++) {
+ report[i] = convert(list.get(i));
+ }
+ return report;
+ }
+
public static JournalInfo convert(JournalInfoProto info) {
int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0;
int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0;
@@ -1756,3 +1867,4 @@ public class PBHelper {
return new ExactSizeInputStream(input, size);
}
}
+
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java Thu Dec 12 07:17:51 2013
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.balancer;
import static com.google.common.base.Preconditions.checkArgument;
-
import static org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed;
import java.io.BufferedInputStream;
@@ -221,9 +220,9 @@ public class Balancer {
private Map<Block, BalancerBlock> globalBlockList
= new HashMap<Block, BalancerBlock>();
private MovedBlocks movedBlocks = new MovedBlocks();
- // Map storage IDs to BalancerDatanodes
- private Map<String, BalancerDatanode> datanodes
- = new HashMap<String, BalancerDatanode>();
+ /** Map (datanodeUuid -> BalancerDatanodes) */
+ private final Map<String, BalancerDatanode> datanodeMap
+ = new HashMap<String, BalancerDatanode>();
private NetworkTopology cluster;
@@ -241,6 +240,14 @@ public class Balancer {
private PendingBlockMove() {
}
+ @Override
+ public String toString() {
+ final Block b = block.getBlock();
+ return b + " with size=" + b.getNumBytes() + " from "
+ + source.getDisplayName() + " to " + target.getDisplayName()
+ + " through " + proxySource.getDisplayName();
+ }
+
/* choose a block & a proxy source for this pendingMove
* whose source & target have already been chosen.
*
@@ -272,11 +279,7 @@ public class Balancer {
if ( chooseProxySource() ) {
movedBlocks.add(block);
if (LOG.isDebugEnabled()) {
- LOG.debug("Decided to move block "+ block.getBlockId()
- +" with a length of "+StringUtils.byteDesc(block.getNumBytes())
- + " bytes from " + source.getDisplayName()
- + " to " + target.getDisplayName()
- + " using proxy source " + proxySource.getDisplayName() );
+ LOG.debug("Decided to move " + this);
}
return true;
}
@@ -353,17 +356,9 @@ public class Balancer {
sendRequest(out);
receiveResponse(in);
bytesMoved.inc(block.getNumBytes());
- LOG.info( "Moving block " + block.getBlock().getBlockId() +
- " from "+ source.getDisplayName() + " to " +
- target.getDisplayName() + " through " +
- proxySource.getDisplayName() +
- " is succeeded." );
+ LOG.info("Successfully moved " + this);
} catch (IOException e) {
- LOG.warn("Error moving block "+block.getBlockId()+
- " from " + source.getDisplayName() + " to " +
- target.getDisplayName() + " through " +
- proxySource.getDisplayName() +
- ": "+e.getMessage());
+ LOG.warn("Failed to move " + this + ": " + e.getMessage());
} finally {
IOUtils.closeStream(out);
IOUtils.closeStream(in);
@@ -415,9 +410,7 @@ public class Balancer {
@Override
public void run() {
if (LOG.isDebugEnabled()) {
- LOG.debug("Starting moving "+ block.getBlockId() +
- " from " + proxySource.getDisplayName() + " to " +
- target.getDisplayName());
+ LOG.debug("Start moving " + PendingBlockMove.this);
}
dispatch();
}
@@ -464,11 +457,6 @@ public class Balancer {
return block;
}
- /* Return the block id */
- private long getBlockId() {
- return block.getBlockId();
- }
-
/* Return the length of the block */
private long getNumBytes() {
return block.getNumBytes();
@@ -552,7 +540,7 @@ public class Balancer {
/* Get the storage id of the datanode */
protected String getStorageID() {
- return datanode.getStorageID();
+ return datanode.getDatanodeUuid();
}
/** Decide if still need to move more bytes */
@@ -675,10 +663,10 @@ public class Balancer {
synchronized (block) {
// update locations
- for ( String storageID : blk.getStorageIDs() ) {
- BalancerDatanode datanode = datanodes.get(storageID);
+ for (String datanodeUuid : blk.getDatanodeUuids()) {
+ final BalancerDatanode d = datanodeMap.get(datanodeUuid);
if (datanode != null) { // not an unknown datanode
- block.addLocation(datanode);
+ block.addLocation(d);
}
}
}
@@ -852,16 +840,6 @@ public class Balancer {
DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_DEFAULT));
}
- /* Shuffle datanode array */
- static private void shuffleArray(DatanodeInfo[] datanodes) {
- for (int i=datanodes.length; i>1; i--) {
- int randomIndex = DFSUtil.getRandom().nextInt(i);
- DatanodeInfo tmp = datanodes[randomIndex];
- datanodes[randomIndex] = datanodes[i-1];
- datanodes[i-1] = tmp;
- }
- }
-
/* Given a data node set, build a network topology and decide
* over-utilized datanodes, above average utilized datanodes,
* below average utilized datanodes, and underutilized datanodes.
@@ -891,8 +869,7 @@ public class Balancer {
* an increasing order or a decreasing order.
*/
long overLoadedBytes = 0L, underLoadedBytes = 0L;
- shuffleArray(datanodes);
- for (DatanodeInfo datanode : datanodes) {
+ for (DatanodeInfo datanode : DFSUtil.shuffle(datanodes)) {
if (datanode.isDecommissioned() || datanode.isDecommissionInProgress()) {
continue; // ignore decommissioning or decommissioned nodes
}
@@ -923,13 +900,13 @@ public class Balancer {
datanodeS.utilization)*datanodeS.datanode.getCapacity()/100.0);
}
}
- this.datanodes.put(datanode.getStorageID(), datanodeS);
+ datanodeMap.put(datanode.getDatanodeUuid(), datanodeS);
}
//logging
logNodes();
- assert (this.datanodes.size() ==
+ assert (this.datanodeMap.size() ==
overUtilizedDatanodes.size()+underUtilizedDatanodes.size()+
aboveAvgUtilizedDatanodes.size()+belowAvgUtilizedDatanodes.size())
: "Mismatched number of datanodes";
@@ -1001,9 +978,9 @@ public class Balancer {
// At last, match all remaining nodes
chooseNodes(ANY_OTHER);
- assert (datanodes.size() >= sources.size()+targets.size())
+ assert (datanodeMap.size() >= sources.size()+targets.size())
: "Mismatched number of datanodes (" +
- datanodes.size() + " total, " +
+ datanodeMap.size() + " total, " +
sources.size() + " sources, " +
targets.size() + " targets)";
@@ -1304,7 +1281,7 @@ public class Balancer {
this.aboveAvgUtilizedDatanodes.clear();
this.belowAvgUtilizedDatanodes.clear();
this.underUtilizedDatanodes.clear();
- this.datanodes.clear();
+ this.datanodeMap.clear();
this.sources.clear();
this.targets.clear();
this.policy.reset();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java Thu Dec 12 07:17:51 2013
@@ -75,7 +75,7 @@ public interface BlockCollection {
* and set the locations.
*/
public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock,
- DatanodeDescriptor[] locations) throws IOException;
+ DatanodeStorageInfo[] targets) throws IOException;
/**
* @return whether the block collection is under construction.