You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2015/06/16 20:42:09 UTC
[50/50] [abbrv] hadoop git commit: Merge remote-tracking branch
'apache-commit/trunk' into HDFS-7240
Merge remote-tracking branch 'apache-commit/trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cec4909
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cec4909
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cec4909
Branch: refs/heads/HDFS-7240
Commit: 5cec49090dd48bb9fdbe25745cf3d88bf7ee7342
Parents: 1f32463 32ffda1
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Jun 16 11:40:26 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Jun 16 11:40:26 2015 -0700
----------------------------------------------------------------------
.gitignore | 1 +
BUILDING.txt | 1 +
dev-support/smart-apply-patch.sh | 52 +-
dev-support/test-patch.d/shellcheck.sh | 2 +-
dev-support/test-patch.sh | 300 +-
hadoop-client/pom.xml | 8 +
.../src/site/markdown/Configuration.md | 18 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 130 +
hadoop-common-project/hadoop-common/pom.xml | 6 +-
.../hadoop-common/src/main/bin/hadoop | 4 +
.../src/main/bin/hadoop-config.cmd | 2 +-
.../src/main/bin/hadoop-functions.sh | 46 +-
.../main/conf/hadoop-user-functions.sh.example | 29 +-
.../src/main/conf/log4j.properties | 21 +-
.../src/main/docs/changes/ChangesFancyStyle.css | 170 -
.../main/docs/changes/ChangesSimpleStyle.css | 49 -
.../src/main/docs/changes/changes2html.pl | 286 -
.../src/main/docs/releasenotes.html | 29099 -----------------
.../src/main/docs/src/documentation/README.txt | 7 -
.../classes/CatalogManager.properties | 40 -
.../main/docs/src/documentation/conf/cli.xconf | 327 -
.../src/documentation/content/xdocs/index.xml | 48 -
.../src/documentation/content/xdocs/site.xml | 263 -
.../src/documentation/content/xdocs/tabs.xml | 37 -
.../resources/images/architecture.gif | Bin 15461 -> 0 bytes
.../resources/images/common-logo.jpg | Bin 5887 -> 0 bytes
.../resources/images/core-logo.gif | Bin 6665 -> 0 bytes
.../documentation/resources/images/favicon.ico | Bin 766 -> 0 bytes
.../resources/images/hadoop-logo-big.jpg | Bin 127869 -> 0 bytes
.../resources/images/hadoop-logo.jpg | Bin 9443 -> 0 bytes
.../resources/images/hdfsarchitecture.gif | Bin 17653 -> 0 bytes
.../resources/images/hdfsarchitecture.odg | Bin 41298 -> 0 bytes
.../resources/images/hdfsarchitecture.png | Bin 40571 -> 0 bytes
.../resources/images/hdfsdatanodes.gif | Bin 16060 -> 0 bytes
.../resources/images/hdfsdatanodes.odg | Bin 37296 -> 0 bytes
.../resources/images/hdfsdatanodes.png | Bin 30012 -> 0 bytes
.../main/docs/src/documentation/skinconf.xml | 366 -
.../hadoop-common/src/main/docs/status.xml | 75 -
.../fs/CommonConfigurationKeysPublic.java | 6 +
.../java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 164 +-
.../apache/hadoop/fs/RawLocalFileSystem.java | 51 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 7 +-
.../org/apache/hadoop/fs/shell/FsCommand.java | 30 +-
.../java/org/apache/hadoop/fs/shell/Ls.java | 8 +
.../apache/hadoop/ha/ZKFailoverController.java | 2 +-
.../main/java/org/apache/hadoop/io/MD5Hash.java | 3 +-
.../main/java/org/apache/hadoop/io/Text.java | 4 +-
.../org/apache/hadoop/io/nativeio/NativeIO.java | 11 +
.../apache/hadoop/io/retry/RetryPolicies.java | 6 +-
.../org/apache/hadoop/io/retry/RetryUtils.java | 7 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 8 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 1 +
.../hadoop/net/unix/DomainSocketWatcher.java | 21 +-
.../apache/hadoop/security/ProviderUtils.java | 30 +
.../alias/AbstractJavaKeyStoreProvider.java | 352 +
.../security/alias/JavaKeyStoreProvider.java | 257 +-
.../alias/LocalJavaKeyStoreProvider.java | 215 +
.../org/apache/hadoop/security/token/Token.java | 8 +-
.../apache/hadoop/tracing/SpanReceiverHost.java | 45 +-
.../java/org/apache/hadoop/util/ConfTest.java | 368 +
.../org/apache/hadoop/util/DiskChecker.java | 24 +-
.../org/apache/hadoop/util/ReflectionUtils.java | 6 +-
.../main/java/org/apache/hadoop/util/Shell.java | 16 +-
.../hadoop/net/unix/DomainSocketWatcher.c | 2 +-
...oop.security.alias.CredentialProviderFactory | 1 +
.../src/main/resources/core-default.xml | 8 +
.../src/site/markdown/CLIMiniCluster.md.vm | 2 +-
.../src/site/markdown/ClusterSetup.md | 2 +-
.../src/site/markdown/CommandsManual.md | 4 +-
.../src/site/markdown/FileSystemShell.md | 2 +-
.../hadoop-common/src/site/markdown/Tracing.md | 12 +-
.../apache/hadoop/fs/TestLocalFileSystem.java | 35 +
.../fs/TestLocalFileSystemPermission.java | 111 +-
.../java/org/apache/hadoop/fs/shell/TestLs.java | 25 +
.../hadoop/io/retry/TestDefaultRetryPolicy.java | 101 +
.../java/org/apache/hadoop/ipc/TestIPC.java | 78 +-
.../net/unix/TestDomainSocketWatcher.java | 75 +
.../alias/TestCredentialProviderFactory.java | 42 +-
.../token/delegation/TestDelegationToken.java | 15 +
.../apache/hadoop/tracing/SetSpanReceiver.java | 109 +
.../org/apache/hadoop/util/TestConfTest.java | 204 +
.../org/apache/hadoop/util/TestDiskChecker.java | 22 +
.../org/apache/hadoop/util/TestStringUtils.java | 3 -
hadoop-common-project/hadoop-kms/pom.xml | 4 +-
.../crypto/key/kms/server/KMSMDCFilter.java | 2 +-
hadoop-dist/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +-
.../apache/hadoop/lib/servlet/ServerWebApp.java | 3 +-
.../org/apache/hadoop/test/TestDirHelper.java | 2 +-
.../org/apache/hadoop/test/TestHdfsHelper.java | 4 +-
.../org/apache/hadoop/test/TestJettyHelper.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 133 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 24 +-
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 13 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 30 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 12 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 7 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 161 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 22 +-
.../apache/hadoop/hdfs/KeyProviderCache.java | 4 +-
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 43 +-
.../hadoop/hdfs/protocol/LayoutVersion.java | 51 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 8 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 3 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 3 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 78 +-
.../hadoop/hdfs/server/balancer/ExitStatus.java | 3 +-
.../hdfs/server/balancer/NameNodeConnector.java | 16 +
.../server/blockmanagement/BlockCollection.java | 8 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 359 +
.../blockmanagement/BlockInfoContiguous.java | 373 +-
.../BlockInfoContiguousUnderConstruction.java | 402 -
.../BlockInfoUnderConstruction.java | 405 +
.../BlockInfoUnderConstructionContiguous.java | 110 +
.../server/blockmanagement/BlockManager.java | 206 +-
.../BlockManagerFaultInjector.java | 52 +
.../BlockPlacementPolicyRackFaultTolarent.java | 154 -
.../BlockPlacementPolicyRackFaultTolerant.java | 154 +
.../BlockReportLeaseManager.java | 355 +
.../hdfs/server/blockmanagement/BlocksMap.java | 49 +-
.../CacheReplicationMonitor.java | 14 +-
.../ContiguousBlockStorageOp.java | 106 +
.../blockmanagement/DatanodeDescriptor.java | 29 +-
.../server/blockmanagement/DatanodeManager.java | 6 +-
.../blockmanagement/DatanodeStorageInfo.java | 22 +-
.../blockmanagement/DecommissionManager.java | 22 +-
.../hdfs/server/datanode/BPServiceActor.java | 71 +-
.../hdfs/server/datanode/BlockReceiver.java | 10 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 4 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 36 +-
.../hdfs/server/datanode/DataStorage.java | 2 +-
.../hdfs/server/datanode/ReplicaInPipeline.java | 15 +-
.../datanode/ReplicaInPipelineInterface.java | 10 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 31 +-
.../server/datanode/web/DatanodeHttpServer.java | 96 +-
.../web/PortUnificationServerHandler.java | 85 +
.../datanode/web/SimpleHttpProxyHandler.java | 18 +-
.../hdfs/server/datanode/web/URLDispatcher.java | 9 +-
.../datanode/web/dtp/DtpHttp2FrameListener.java | 52 +
.../datanode/web/dtp/DtpHttp2Handler.java | 34 +
.../datanode/web/webhdfs/ExceptionHandler.java | 31 +-
.../server/datanode/web/webhdfs/HdfsWriter.java | 10 +-
.../datanode/web/webhdfs/WebHdfsHandler.java | 57 +-
.../hdfs/server/namenode/BackupImage.java | 2 +-
.../hdfs/server/namenode/Checkpointer.java | 4 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 29 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 32 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 24 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 21 +-
.../hdfs/server/namenode/FSImageFormat.java | 18 +-
.../server/namenode/FSImageFormatPBINode.java | 32 +-
.../server/namenode/FSImageFormatProtobuf.java | 33 +-
.../server/namenode/FSImageSerialization.java | 7 +-
.../hdfs/server/namenode/FSNamesystem.java | 280 +-
.../namenode/FileUnderConstructionFeature.java | 16 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 88 +-
.../hdfs/server/namenode/LeaseManager.java | 6 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 5 +-
.../server/namenode/NameNodeLayoutVersion.java | 46 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 34 +-
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 4 +-
.../hdfs/server/namenode/SecondaryNameNode.java | 4 +-
.../snapshot/FSImageFormatPBSnapshot.java | 5 +-
.../hdfs/server/namenode/snapshot/FileDiff.java | 10 +-
.../server/namenode/snapshot/FileDiffList.java | 24 +-
.../snapshot/FileWithSnapshotFeature.java | 4 +-
.../top/window/RollingWindowManager.java | 8 +-
.../server/protocol/BlockReportContext.java | 25 +-
.../hdfs/server/protocol/DatanodeProtocol.java | 5 +-
.../hdfs/server/protocol/HeartbeatResponse.java | 10 +-
.../hdfs/server/protocol/RegisterCommand.java | 2 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 +-
.../org/apache/hadoop/hdfs/tools/DFSck.java | 9 +-
.../offlineImageViewer/FSImageHandler.java | 39 +-
.../StorageContainerManager.java | 5 +-
.../storagecontainer/StorageContainerMap.java | 7 +-
.../StorageContainerNameService.java | 4 +-
.../src/main/native/libhdfs/expect.h | 18 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 1 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 3 +-
.../main/native/libhdfs/test/test_libhdfs_ops.c | 9 +
.../main/native/libhdfs/test_libhdfs_threaded.c | 17 +-
.../src/main/proto/DatanodeProtocol.proto | 6 +
.../src/main/resources/hdfs-default.xml | 37 +-
.../src/site/markdown/HDFSCommands.md | 22 +-
.../src/site/markdown/HdfsNfsGateway.md | 2 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 12 +-
.../apache/hadoop/hdfs/TestDFSInputStream.java | 25 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 297 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 18 +
.../apache/hadoop/hdfs/TestDatanodeDeath.java | 1 -
.../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 8 +-
.../apache/hadoop/hdfs/TestDecommission.java | 4 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 2 +-
.../apache/hadoop/hdfs/TestFileCorruption.java | 19 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 6 -
.../hdfs/protocol/TestBlockListAsLongs.java | 4 +-
.../hadoop/hdfs/protocol/TestLayoutVersion.java | 101 +-
.../hdfs/server/balancer/TestBalancer.java | 83 +-
.../server/blockmanagement/TestBlockInfo.java | 18 +-
.../TestBlockInfoUnderConstruction.java | 6 +-
.../blockmanagement/TestBlockManager.java | 40 +-
.../TestBlockReportRateLimiting.java | 246 +
.../blockmanagement/TestDatanodeDescriptor.java | 4 +-
.../blockmanagement/TestDatanodeManager.java | 21 +-
.../blockmanagement/TestHeartbeatHandling.java | 8 +-
.../TestNameNodePrunesMissingStorages.java | 2 +-
.../blockmanagement/TestPendingReplication.java | 2 +-
.../blockmanagement/TestReplicationPolicy.java | 14 +-
.../server/datanode/SimulatedFSDataset.java | 5 +
.../server/datanode/TestBPOfferService.java | 7 +-
.../TestBlockHasMultipleReplicasOnSameDN.java | 2 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 6 +-
.../datanode/TestBpServiceActorScheduler.java | 2 +-
.../TestDatanodeProtocolRetryPolicy.java | 8 +-
.../server/datanode/TestFsDatasetCache.java | 9 +-
.../TestNNHandlesBlockReportPerStorage.java | 2 +-
.../TestNNHandlesCombinedBlockReport.java | 2 +-
.../hdfs/server/datanode/TestStorageReport.java | 2 +-
.../extdataset/ExternalReplicaInPipeline.java | 7 +
.../fsdataset/impl/TestFsDatasetImpl.java | 34 +
.../datanode/web/dtp/Http2ResponseHandler.java | 65 +
.../server/datanode/web/dtp/TestDtpHttp2.java | 147 +
.../hdfs/server/namenode/CreateEditsLog.java | 9 +-
.../hdfs/server/namenode/FSImageTestUtil.java | 2 +-
.../server/namenode/NNThroughputBenchmark.java | 8 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../hdfs/server/namenode/TestAddBlock.java | 12 +-
...stBlockPlacementPolicyRackFaultTolarent.java | 209 -
...stBlockPlacementPolicyRackFaultTolerant.java | 209 +
.../namenode/TestBlockUnderConstruction.java | 10 +-
.../hdfs/server/namenode/TestClusterId.java | 6 +-
.../TestCommitBlockSynchronization.java | 17 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 6 +-
.../hdfs/server/namenode/TestEditLog.java | 20 +-
.../server/namenode/TestFSEditLogLoader.java | 2 +-
.../hdfs/server/namenode/TestFSImage.java | 45 +-
.../hdfs/server/namenode/TestFSNamesystem.java | 20 +
.../hdfs/server/namenode/TestFileTruncate.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
.../server/namenode/TestGetBlockLocations.java | 4 +-
.../hdfs/server/namenode/TestINodeFile.java | 3 +-
.../namenode/TestTruncateQuotaUpdate.java | 13 +-
.../namenode/ha/TestRetryCacheWithHA.java | 43 +-
.../namenode/snapshot/SnapshotTestHelper.java | 8 +-
.../snapshot/TestFileWithSnapshotFeature.java | 3 +-
.../snapshot/TestSnapshotBlocksMap.java | 24 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 16 +-
.../org/apache/hadoop/tracing/TestTracing.java | 101 +-
.../TestTracingShortCircuitLocalRead.java | 4 +-
hadoop-mapreduce-project/CHANGES.txt | 49 +
.../jobhistory/JobHistoryEventHandler.java | 6 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 6 +
.../v2/app/rm/RMContainerAllocator.java | 19 +-
.../v2/app/rm/RMContainerRequestor.java | 32 +-
.../mapreduce/v2/app/webapp/AttemptsPage.java | 5 +-
.../mapreduce/v2/app/webapp/TaskPage.java | 89 +-
.../hadoop/mapreduce/jobhistory/TestEvents.java | 148 +-
.../mapreduce/jobhistory/TestJobSummary.java | 10 +-
.../v2/app/rm/TestRMContainerAllocator.java | 89 +-
.../v2/jobhistory/FileNameIndexUtils.java | 17 +-
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 +
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 5 -
.../v2/jobhistory/TestFileNameIndexUtils.java | 26 +
.../hadoop-mapreduce-client-core/pom.xml | 15 +
.../java/org/apache/hadoop/mapred/Task.java | 4 +-
.../org/apache/hadoop/mapred/lib/Chain.java | 2 +-
.../hadoop/mapred/pipes/PipesPartitioner.java | 6 +-
.../org/apache/hadoop/mapreduce/MRConfig.java | 7 +
.../apache/hadoop/mapreduce/MRJobConfig.java | 20 +
.../ClientDistributedCacheManager.java | 1 +
.../mapreduce/jobhistory/AMStartedEvent.java | 24 +-
.../mapreduce/jobhistory/AvroArrayUtils.java | 2 +-
.../mapreduce/jobhistory/EventReader.java | 22 +-
.../mapreduce/jobhistory/EventWriter.java | 24 +-
.../mapreduce/jobhistory/JobFinishedEvent.java | 39 +-
.../mapreduce/jobhistory/JobHistoryParser.java | 2 +-
.../jobhistory/JobInfoChangeEvent.java | 12 +-
.../mapreduce/jobhistory/JobInitedEvent.java | 24 +-
.../jobhistory/JobPriorityChangeEvent.java | 10 +-
.../jobhistory/JobStatusChangedEvent.java | 8 +-
.../mapreduce/jobhistory/JobSubmittedEvent.java | 62 +-
.../JobUnsuccessfulCompletionEvent.java | 4 +-
.../jobhistory/MapAttemptFinishedEvent.java | 66 +-
.../jobhistory/ReduceAttemptFinishedEvent.java | 70 +-
.../jobhistory/TaskAttemptFinishedEvent.java | 34 +-
.../jobhistory/TaskAttemptStartedEvent.java | 49 +-
.../TaskAttemptUnsuccessfulCompletionEvent.java | 66 +-
.../mapreduce/jobhistory/TaskFailedEvent.java | 32 +-
.../mapreduce/jobhistory/TaskFinishedEvent.java | 26 +-
.../mapreduce/jobhistory/TaskStartedEvent.java | 20 +-
.../mapreduce/jobhistory/TaskUpdatedEvent.java | 10 +-
.../task/reduce/IFileWrappedMapOutput.java | 72 +
.../task/reduce/InMemoryMapOutput.java | 26 +-
.../mapreduce/task/reduce/MergeManagerImpl.java | 5 +-
.../mapreduce/task/reduce/OnDiskMapOutput.java | 33 +-
.../task/reduce/ShuffleSchedulerImpl.java | 7 +-
.../src/main/resources/mapred-default.xml | 45 +-
.../src/site/markdown/MapredCommands.md | 14 +-
.../TestClientDistributedCacheManager.java | 28 +
.../mapreduce/task/reduce/TestFetcher.java | 27 +-
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 2 +-
.../hadoop/mapreduce/v2/hs/webapp/HsView.java | 5 +-
.../org/apache/hadoop/mapred/YARNRunner.java | 30 +-
.../apache/hadoop/mapred/TestJobCounters.java | 5 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 16 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 2 +-
.../apache/hadoop/mapred/ShuffleHandler.java | 15 +-
.../hadoop/examples/BaileyBorweinPlouffe.java | 2 +-
.../apache/hadoop/examples/QuasiMonteCarlo.java | 4 +
hadoop-mapreduce-project/pom.xml | 4 +-
hadoop-project-dist/pom.xml | 20 +-
hadoop-project/pom.xml | 12 +-
hadoop-project/src/site/site.xml | 44 +-
.../org/apache/hadoop/fs/s3/S3Credentials.java | 10 +-
.../apache/hadoop/fs/s3/TestS3Credentials.java | 107 +-
.../fs/azure/AzureNativeFileSystemStore.java | 15 +-
.../hadoop/fs/azure/PageBlobInputStream.java | 32 +-
.../hadoop/fs/azure/PageBlobOutputStream.java | 10 +-
.../hadoop/fs/azure/StorageInterface.java | 6 +-
.../hadoop/fs/azure/StorageInterfaceImpl.java | 4 +-
.../hadoop/fs/azure/MockStorageInterface.java | 4 +-
.../fs/azure/NativeAzureFileSystemBaseTest.java | 79 +-
.../TestAzureFileSystemErrorConditions.java | 1 +
.../hadoop/fs/azure/TestBlobDataValidation.java | 1 +
...tiveAzureFileSystemContractPageBlobLive.java | 90 +
.../apache/hadoop/tools/util/DistCpUtils.java | 2 +-
.../gridmix/DummyResourceCalculatorPlugin.java | 6 +
hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 9 +
.../apache/hadoop/record/BinaryRecordInput.java | 7 +-
.../hadoop/record/BinaryRecordOutput.java | 17 +-
.../org/apache/hadoop/streaming/PipeMapRed.java | 19 +-
.../org/apache/hadoop/streaming/StreamJob.java | 5 +-
.../hadoop/typedbytes/TypedBytesInput.java | 8 +-
.../hadoop/typedbytes/TypedBytesOutput.java | 8 +-
.../typedbytes/TypedBytesRecordInput.java | 8 +-
.../typedbytes/TypedBytesRecordOutput.java | 8 +-
.../typedbytes/TypedBytesWritableInput.java | 8 +-
.../typedbytes/TypedBytesWritableOutput.java | 8 +-
.../src/site/markdown/HadoopStreaming.md.vm | 9 +
hadoop-yarn-project/CHANGES.txt | 125 +
.../hadoop/yarn/api/ApplicationConstants.java | 58 +-
.../yarn/api/records/ApplicationReport.java | 19 +
.../hadoop/yarn/api/records/NodeState.java | 10 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 59 +-
.../src/main/proto/yarn_protos.proto | 1 +
.../hadoop/yarn/client/cli/ClusterCLI.java | 2 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 22 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 26 +-
...estApplicationMasterServiceProtocolOnHA.java | 10 +-
.../hadoop/yarn/client/TestRMFailover.java | 26 +-
.../hadoop/yarn/client/cli/TestClusterCLI.java | 4 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 13 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 16 +-
.../api/records/impl/pb/QueueInfoPBImpl.java | 2 +-
.../records/impl/pb/ResourceRequestPBImpl.java | 5 +-
.../client/api/impl/TimelineClientImpl.java | 28 +-
.../nodelabels/CommonNodeLabelsManager.java | 17 +-
.../util/LinuxResourceCalculatorPlugin.java | 88 +-
.../yarn/util/ResourceCalculatorPlugin.java | 11 +-
.../util/WindowsResourceCalculatorPlugin.java | 6 +
.../resource/DominantResourceCalculator.java | 15 +
.../hadoop/yarn/webapp/YarnWebParams.java | 3 +
.../hadoop/yarn/webapp/view/JQueryUI.java | 8 +-
.../src/main/resources/yarn-default.xml | 66 +-
.../hadoop/yarn/conf/TestYarnConfiguration.java | 22 +
.../util/TestLinuxResourceCalculatorPlugin.java | 101 +-
.../hadoop/yarn/util/TestWebAppUtils.java | 81 -
.../util/resource/TestResourceCalculator.java | 125 +
.../yarn/webapp/util/TestWebAppUtils.java | 52 +
.../ApplicationHistoryClientService.java | 9 +-
.../ApplicationHistoryManager.java | 16 +-
.../ApplicationHistoryManagerImpl.java | 4 +-
...pplicationHistoryManagerOnTimelineStore.java | 76 +-
.../webapp/AHSView.java | 2 +-
.../TestApplicationHistoryClientService.java | 96 +-
...pplicationHistoryManagerOnTimelineStore.java | 54 +-
.../webapp/TestAHSWebServices.java | 63 +-
.../TestTimelineAuthenticationFilter.java | 11 +
.../hadoop-yarn-server-common/pom.xml | 14 +
.../hadoop/yarn/server/api/ResourceTracker.java | 16 +-
.../pb/client/ResourceTrackerPBClientImpl.java | 18 +
.../service/ResourceTrackerPBServiceImpl.java | 27 +-
.../UnRegisterNodeManagerRequest.java | 38 +
.../UnRegisterNodeManagerResponse.java | 30 +
.../pb/UnRegisterNodeManagerRequestPBImpl.java | 108 +
.../pb/UnRegisterNodeManagerResponsePBImpl.java | 70 +
.../metrics/ApplicationMetricsConstants.java | 1 +
.../hadoop/yarn/server/webapp/AppsBlock.java | 37 +-
.../hadoop/yarn/server/webapp/WebPageUtils.java | 8 +-
.../hadoop/yarn/server/webapp/WebServices.java | 53 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 20 +-
.../src/main/proto/ResourceTracker.proto | 1 +
.../yarn_server_common_service_protos.proto | 7 +
.../yarn/TestResourceTrackerPBClientImpl.java | 34 +-
.../apache/hadoop/yarn/TestYSCRPCFactories.java | 10 +-
.../hadoop/yarn/TestYarnServerApiClasses.java | 12 +
.../hadoop-yarn-server-nodemanager/pom.xml | 2 +-
.../server/nodemanager/ContainerExecutor.java | 111 +-
.../nodemanager/DefaultContainerExecutor.java | 46 +-
.../server/nodemanager/DeletionService.java | 13 +-
.../nodemanager/DockerContainerExecutor.java | 45 +-
.../nodemanager/LinuxContainerExecutor.java | 62 +-
.../yarn/server/nodemanager/NodeManager.java | 17 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 47 +-
.../WindowsSecureContainerExecutor.java | 184 +-
.../container/ContainerImpl.java | 7 -
.../launcher/ContainerLaunch.java | 31 +-
.../launcher/RecoveredContainerLaunch.java | 7 +-
.../localizer/ResourceLocalizationService.java | 17 +-
.../monitor/ContainersMonitorImpl.java | 11 +-
.../executor/ContainerLivenessContext.java | 70 +
.../executor/ContainerReacquisitionContext.java | 71 +
.../executor/ContainerSignalContext.java | 83 +
.../executor/ContainerStartContext.java | 147 +
.../executor/DeletionAsUserContext.java | 91 +
.../executor/LocalizerStartContext.java | 122 +
.../util/CgroupsLCEResourcesHandler.java | 13 +-
.../util/NodeManagerHardwareUtils.java | 238 +-
.../server/nodemanager/LocalRMInterface.java | 10 +
.../nodemanager/MockNodeStatusUpdater.java | 9 +
.../nodemanager/TestContainerExecutor.java | 56 +-
.../TestDefaultContainerExecutor.java | 51 +-
.../server/nodemanager/TestDeletionService.java | 19 +-
.../TestDockerContainerExecutor.java | 14 +-
.../TestDockerContainerExecutorWithMocks.java | 42 +-
.../nodemanager/TestLinuxContainerExecutor.java | 72 +-
.../TestLinuxContainerExecutorWithMocks.java | 124 +-
.../TestLocalDirsHandlerService.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 44 +
.../TestNodeStatusUpdaterForLabels.java | 8 +
.../BaseContainerManagerTest.java | 8 +-
.../launcher/TestContainerLaunch.java | 8 +-
.../TestResourceLocalizationService.java | 16 +-
.../TestLogAggregationService.java | 9 +-
.../monitor/TestContainersMonitor.java | 8 +-
.../util/TestCgroupsLCEResourcesHandler.java | 2 +
.../util/TestNodeManagerHardwareUtils.java | 143 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 8 +
.../ApplicationMasterService.java | 19 +-
.../server/resourcemanager/ClusterMetrics.java | 14 +
.../server/resourcemanager/ResourceManager.java | 2 +-
.../resourcemanager/ResourceTrackerService.java | 23 +
.../resourcemanager/amlauncher/AMLauncher.java | 29 +-
.../metrics/ApplicationCreatedEvent.java | 10 +-
.../metrics/SystemMetricsPublisher.java | 4 +-
.../nodelabels/RMNodeLabelsManager.java | 1 +
.../recovery/FileSystemRMStateStore.java | 20 +-
.../recovery/ZKRMStateStore.java | 770 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 16 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 4 +-
.../resourcemanager/rmnode/RMNodeEventType.java | 1 +
.../resourcemanager/rmnode/RMNodeImpl.java | 23 +-
.../scheduler/AppSchedulingInfo.java | 7 +-
.../scheduler/SchedulerApplicationAttempt.java | 5 +-
.../scheduler/capacity/CapacityScheduler.java | 30 +-
.../CapacitySchedulerConfiguration.java | 7 +-
.../scheduler/fair/FSAppAttempt.java | 128 +-
.../scheduler/fair/FSLeafQueue.java | 43 +-
.../scheduler/fair/FSOpDurations.java | 6 +
.../scheduler/fair/FSParentQueue.java | 219 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 15 +
.../scheduler/fair/FairScheduler.java | 79 +-
.../scheduler/fair/QueueManager.java | 3 +-
.../AbstractComparatorOrderingPolicy.java | 25 +
.../scheduler/policy/FairOrderingPolicy.java | 11 +-
.../scheduler/policy/FifoOrderingPolicy.java | 6 +-
.../scheduler/policy/OrderingPolicy.java | 5 +
.../webapp/CapacitySchedulerPage.java | 6 +-
.../webapp/FairSchedulerAppsBlock.java | 6 +
.../webapp/JAXBContextResolver.java | 3 +-
.../webapp/MetricsOverviewTable.java | 2 +
.../resourcemanager/webapp/NodesPage.java | 1 +
.../resourcemanager/webapp/RMAppsBlock.java | 8 +
.../resourcemanager/webapp/RMWebServices.java | 27 +-
.../webapp/dao/ClusterMetricsInfo.java | 8 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 31 +-
.../webapp/dao/FairSchedulerQueueInfoList.java | 49 +
.../yarn/server/resourcemanager/HATestUtil.java | 38 +
.../yarn/server/resourcemanager/MockRM.java | 28 +-
.../TestApplicationMasterLauncher.java | 7 +-
.../resourcemanager/TestRMEmbeddedElector.java | 28 +-
.../resourcemanager/TestRMNodeTransitions.java | 14 +
.../TestResourceTrackerService.java | 123 +-
.../TestWorkPreservingRMRestart.java | 10 +-
.../metrics/TestSystemMetricsPublisher.java | 37 +
.../recovery/RMStateStoreTestBase.java | 3 +-
.../recovery/TestFSRMStateStore.java | 21 +-
.../recovery/TestZKRMStateStore.java | 83 +-
.../recovery/TestZKRMStateStorePerf.java | 12 +-
.../TestZKRMStateStoreZKClientConnections.java | 181 +-
.../scheduler/TestSchedulerUtils.java | 47 +
.../capacity/TestCapacityScheduler.java | 197 +-
.../scheduler/capacity/TestQueueParsing.java | 33 +
.../scheduler/fair/TestFairScheduler.java | 326 +
.../scheduler/fair/TestSchedulingUpdate.java | 135 +
.../security/TestDelegationTokenRenewer.java | 27 +-
.../resourcemanager/webapp/TestNodesPage.java | 4 +-
.../webapp/TestRMWebServices.java | 21 +-
.../webapp/TestRMWebServicesCapacitySched.java | 10 +
.../webapp/TestRMWebServicesFairScheduler.java | 35 +
.../webapp/TestRMWebServicesNodeLabels.java | 51 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 32 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 37 +
.../src/site/markdown/ResourceManagerRest.md | 248 +-
.../src/site/markdown/TimelineServer.md | 51 +-
.../src/site/markdown/YarnCommands.md | 10 +-
hadoop-yarn-project/pom.xml | 2 +-
pom.xml | 2 +
512 files changed, 14262 insertions(+), 36212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index d286784,b9abcd0..52c0d66
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@@ -19,53 -19,20 +19,22 @@@ package org.apache.hadoop.hdfs.server.b
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
- import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
- import org.apache.hadoop.util.LightWeightGSet;
+ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import java.util.LinkedList;
+
/**
- * BlockInfo class maintains for a given block
- * the {@link BlockCollection} it is part of and datanodes where the replicas of
- * the block are stored.
+ * Subclass of {@link BlockInfo}, used for a block with replication scheme.
*/
@InterfaceAudience.Private
- public class BlockInfoContiguous extends Block
- implements LightWeightGSet.LinkedElement {
- public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
+ public class BlockInfoContiguous extends BlockInfo {
- private BlockCollection bc;
-
- /** For implementing {@link LightWeightGSet.LinkedElement} interface */
- private LightWeightGSet.LinkedElement nextLinkedElement;
-
- /**
- * This array contains triplets of references. For each i-th storage, the
- * block belongs to triplets[3*i] is the reference to the
- * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
- * references to the previous and the next blocks, respectively, in the list
- * of blocks belonging to this storage.
- *
- * Using previous and next in Object triplets is done instead of a
- * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
- * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
- * bytes using the triplets.
- */
- private Object[] triplets;
-
- /**
- * Construct an entry for blocksmap
- * @param replication the block's replication factor
- */
- public BlockInfoContiguous(short replication) {
- this.triplets = new Object[3*replication];
- this.bc = null;
+ public BlockInfoContiguous(short size) {
+ super(size);
}
-
- public BlockInfoContiguous(Block blk, short replication) {
- super(blk);
- this.triplets = new Object[3*replication];
- this.bc = null;
+
+ public BlockInfoContiguous(Block blk, short size) {
+ super(blk, size);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 19a20bd,ebc9017..420ae56
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@@ -44,9 -70,17 +44,12 @@@ import org.apache.hadoop.hdfs.server.na
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
-import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
-import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.*;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
+ import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
+ import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
-import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
-import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
-import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index 465f095,9eb1059..9d0e66c
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@@ -17,28 -17,27 +17,30 @@@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import java.util.Iterator;
-
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
import org.apache.hadoop.hdfs.protocol.Block;
- import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
++import com.google.common.base.Preconditions;
+ import com.google.common.base.Predicate;
+ import com.google.common.collect.Iterables;
+import java.util.Iterator;
/**
* This class maintains the map from a block to its metadata.
* block's metadata currently includes blockCollection it belongs to and
* the datanodes that store the block.
*/
-class BlocksMap {
+public class BlocksMap {
private static class StorageIterator implements Iterator<DatanodeStorageInfo> {
- private final BlockInfoContiguous blockInfo;
+ private final BlockInfo blockInfo;
private int nextIdx = 0;
- StorageIterator(BlockInfoContiguous blkInfo) {
+ StorageIterator(BlockInfo blkInfo) {
this.blockInfo = blkInfo;
}
@@@ -62,13 -61,8 +64,13 @@@
/** Constant {@link LightWeightGSet} capacity. */
private final int capacity;
- private GSet<Block, BlockInfoContiguous> blocks;
+ private GSet<Block, BlockInfo> blocks;
- public BlocksMap(int capacity, GSet<Block, BlockInfoContiguous> b) {
++ public BlocksMap(int capacity, GSet<Block, BlockInfo> b) {
+ this.capacity = capacity;
+ this.blocks = b;
+ }
+
BlocksMap(int capacity) {
// Use 2% of total memory to size the GSet capacity
this.capacity = capacity;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 1f27e74,dd7b301..b24ab19
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@@ -323,7 -335,7 +323,7 @@@ public class DatanodeDescriptor extend
* Remove block from the list of blocks belonging to the data-node. Remove
* data-node from the block.
*/
- public boolean removeBlock(BlockInfoContiguous b) {
- boolean removeBlock(BlockInfo b) {
++ public boolean removeBlock(BlockInfo b) {
final DatanodeStorageInfo s = b.findStorageInfo(this);
// if block exists on this datanode
if (s != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 1fe1c9e,ea1abbd..c906636
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@@ -540,9 -537,10 +539,10 @@@ class BPServiceActor implements Runnabl
return cmd;
}
- HeartbeatResponse sendHeartBeat() throws IOException {
+ HeartbeatResponse sendHeartBeat(boolean requestBlockReportLease)
+ throws IOException {
StorageReport[] reports =
- dn.getFSDataset().getStorageReports(bpos.getBlockPoolId());
+ dataset.getStorageReports(bpos.getBlockPoolId());
if (LOG.isDebugEnabled()) {
LOG.debug("Sending heartbeat with " + reports.length +
" storage reports from service actor: " + this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java
index 06b99a8,0000000..ab48542
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java
@@@ -1,320 -1,0 +1,321 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.storagecontainer;
+
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.Namesystem;
+import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.WritableRpcEngine;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.storagecontainer.protocol.ContainerLocationProtocol;
+import org.apache.hadoop.util.LightWeightGSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+
+/**
+ * Service that allocates storage containers and tracks their
+ * location.
+ */
+public class StorageContainerManager
+ implements DatanodeProtocol, ContainerLocationProtocol {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(StorageContainerManager.class);
+
+ private final Namesystem ns = new StorageContainerNameService();
+ private final BlockManager blockManager;
+
+ private long txnId = 234;
+
+ /** The RPC server that listens to requests from DataNodes. */
+ private final RPC.Server serviceRpcServer;
+ private final InetSocketAddress serviceRPCAddress;
+
+ /** The RPC server that listens to requests from clients. */
+ private final RPC.Server clientRpcServer;
+ private final InetSocketAddress clientRpcAddress;
+
+ public StorageContainerManager(StorageContainerConfiguration conf)
+ throws IOException {
+ BlocksMap containerMap = new BlocksMap(
+ LightWeightGSet.computeCapacity(2.0, "BlocksMap"),
+ new StorageContainerMap());
+ this.blockManager = new BlockManager(ns, conf, containerMap);
+
+ int handlerCount =
+ conf.getInt(DFS_NAMENODE_HANDLER_COUNT_KEY,
+ DFS_NAMENODE_HANDLER_COUNT_DEFAULT);
+
+ RPC.setProtocolEngine(conf, DatanodeProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ DatanodeProtocolServerSideTranslatorPB dnProtoPbTranslator =
+ new DatanodeProtocolServerSideTranslatorPB(this);
+ BlockingService dnProtoPbService =
+ DatanodeProtocolProtos.DatanodeProtocolService
+ .newReflectiveBlockingService(dnProtoPbTranslator);
+
+ WritableRpcEngine.ensureInitialized();
+
+ InetSocketAddress serviceRpcAddr = NameNode.getServiceAddress(conf, false);
+ if (serviceRpcAddr != null) {
+ String bindHost =
+ conf.getTrimmed(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY);
+ if (bindHost == null || bindHost.isEmpty()) {
+ bindHost = serviceRpcAddr.getHostName();
+ }
+ LOG.info("Service RPC server is binding to " + bindHost + ":" +
+ serviceRpcAddr.getPort());
+
+ int serviceHandlerCount =
+ conf.getInt(DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
+ DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
+ serviceRpcServer = new RPC.Builder(conf)
+ .setProtocol(
+ org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB.class)
+ .setInstance(dnProtoPbService)
+ .setBindAddress(bindHost)
+ .setPort(serviceRpcAddr.getPort())
+ .setNumHandlers(serviceHandlerCount)
+ .setVerbose(false)
+ .setSecretManager(null)
+ .build();
+
+ DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService,
+ serviceRpcServer);
+
+ InetSocketAddress listenAddr = serviceRpcServer.getListenerAddress();
+ serviceRPCAddress = new InetSocketAddress(
+ serviceRpcAddr.getHostName(), listenAddr.getPort());
+ conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+ NetUtils.getHostPortString(serviceRPCAddress));
+ } else {
+ serviceRpcServer = null;
+ serviceRPCAddress = null;
+ }
+
+ InetSocketAddress rpcAddr = NameNode.getAddress(conf);
+ String bindHost = conf.getTrimmed(DFS_NAMENODE_RPC_BIND_HOST_KEY);
+ if (bindHost == null || bindHost.isEmpty()) {
+ bindHost = rpcAddr.getHostName();
+ }
+ LOG.info("RPC server is binding to " + bindHost + ":" + rpcAddr.getPort());
+
+ clientRpcServer = new RPC.Builder(conf)
+ .setProtocol(
+ org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB.class)
+ .setInstance(dnProtoPbService)
+ .setBindAddress(bindHost)
+ .setPort(rpcAddr.getPort())
+ .setNumHandlers(handlerCount)
+ .setVerbose(false)
+ .setSecretManager(null)
+ .build();
+
+ DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService,
+ clientRpcServer);
+
+ // The rpc-server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddr = clientRpcServer.getListenerAddress();
+ clientRpcAddress = new InetSocketAddress(
+ rpcAddr.getHostName(), listenAddr.getPort());
+ conf.set(FS_DEFAULT_NAME_KEY,
+ NameNode.getUri(clientRpcAddress).toString());
+ }
+
+ @Override
+ public DatanodeRegistration registerDatanode(
+ DatanodeRegistration registration) throws IOException {
+ ns.writeLock();
+ try {
+ blockManager.getDatanodeManager().registerDatanode(registration);
+ } finally {
+ ns.writeUnlock();
+ }
+ return registration;
+ }
+
+ @Override
+ public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration,
+ StorageReport[] reports, long dnCacheCapacity, long dnCacheUsed,
+ int xmitsInProgress, int xceiverCount, int failedVolumes,
- VolumeFailureSummary volumeFailureSummary) throws IOException {
++ VolumeFailureSummary volumeFailureSummary,
++ boolean requestFullBlockReportLease) throws IOException {
+ ns.readLock();
+ try {
+ final int maxTransfer = blockManager.getMaxReplicationStreams()
+ - xmitsInProgress;
+ DatanodeCommand[] cmds = blockManager.getDatanodeManager()
+ .handleHeartbeat(registration, reports, ns.getBlockPoolId(), 0, 0,
+ xceiverCount, maxTransfer, failedVolumes, volumeFailureSummary);
+
+ return new HeartbeatResponse(cmds,
+ new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.ACTIVE,
- txnId), null);
++ txnId), null, 0);
+ } finally {
+ ns.readUnlock();
+ }
+ }
+
+ @Override
+ public DatanodeCommand blockReport(DatanodeRegistration registration,
+ String poolId, StorageBlockReport[] reports,
+ BlockReportContext context) throws IOException {
+ for (int r = 0; r < reports.length; r++) {
+ final BlockListAsLongs storageContainerList = reports[r].getBlocks();
+ blockManager.processReport(registration, reports[r].getStorage(),
+ storageContainerList, context, (r == reports.length - 1));
+ }
+ return null;
+ }
+
+ @Override
+ public DatanodeCommand cacheReport(DatanodeRegistration registration,
+ String poolId, List<Long> blockIds) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void blockReceivedAndDeleted(DatanodeRegistration registration,
+ String poolId, StorageReceivedDeletedBlocks[] rcvdAndDeletedBlocks)
+ throws IOException {
+ for(StorageReceivedDeletedBlocks r : rcvdAndDeletedBlocks) {
+ ns.writeLock();
+ try {
+ blockManager.processIncrementalBlockReport(registration, r);
+ } finally {
+ ns.writeUnlock();
+ }
+ }
+ }
+
+ @Override
+ public void errorReport(DatanodeRegistration registration,
+ int errorCode, String msg) throws IOException {
+ String dnName =
+ (registration == null) ? "Unknown DataNode" : registration.toString();
+
+ if (errorCode == DatanodeProtocol.NOTIFY) {
+ LOG.info("Error report from " + dnName + ": " + msg);
+ return;
+ }
+
+ if (errorCode == DatanodeProtocol.DISK_ERROR) {
+ LOG.warn("Disk error on " + dnName + ": " + msg);
+ } else if (errorCode == DatanodeProtocol.FATAL_DISK_ERROR) {
+ LOG.warn("Fatal disk error on " + dnName + ": " + msg);
+ blockManager.getDatanodeManager().removeDatanode(registration);
+ } else {
+ LOG.info("Error report from " + dnName + ": " + msg);
+ }
+ }
+
+ @Override
+ public NamespaceInfo versionRequest() throws IOException {
+ ns.readLock();
+ try {
+ return unprotectedGetNamespaceInfo();
+ } finally {
+ ns.readUnlock();
+ }
+ }
+
+ private NamespaceInfo unprotectedGetNamespaceInfo() {
+ return new NamespaceInfo(1, "random", "random", 2,
+ NodeType.STORAGE_CONTAINER_SERVICE);
+ }
+
+ @Override
+ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
+ // It doesn't make sense to have LocatedBlock in this API.
+ ns.writeLock();
+ try {
+ for (int i = 0; i < blocks.length; i++) {
+ ExtendedBlock blk = blocks[i].getBlock();
+ DatanodeInfo[] nodes = blocks[i].getLocations();
+ String[] storageIDs = blocks[i].getStorageIDs();
+ for (int j = 0; j < nodes.length; j++) {
+ blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j],
+ storageIDs == null ? null: storageIDs[j],
+ "client machine reported it");
+ }
+ }
+ } finally {
+ ns.writeUnlock();
+ }
+ }
+
+ /**
+ * Start client and service RPC servers.
+ */
+ void start() {
+ clientRpcServer.start();
+ if (serviceRpcServer != null) {
+ serviceRpcServer.start();
+ }
+ }
+
+ /**
+ * Wait until the RPC servers have shutdown.
+ */
+ void join() throws InterruptedException {
+ clientRpcServer.join();
+ if (serviceRpcServer != null) {
+ serviceRpcServer.join();
+ }
+ }
+
+ @Override
+ public void commitBlockSynchronization(ExtendedBlock block,
+ long newgenerationstamp, long newlength, boolean closeFile,
+ boolean deleteblock, DatanodeID[] newtargets, String[] newtargetstorages)
+ throws IOException {
+ // Not needed for the purpose of object store
+ throw new UnsupportedOperationException();
+ }
+
+ public static void main(String[] argv) throws IOException {
+ StorageContainerConfiguration conf = new StorageContainerConfiguration();
+ StorageContainerManager scm = new StorageContainerManager(conf);
+ scm.start();
+ try {
+ scm.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java
index 117ee3b,0000000..1496da2
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java
@@@ -1,123 -1,0 +1,124 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.storagecontainer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.protocol.Block;
++import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.util.GSet;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Maps a storage container to its location on datanodes. Similar to
+ * {@link org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap}
+ */
- public class StorageContainerMap implements GSet<Block, BlockInfoContiguous> {
++public class StorageContainerMap implements GSet<Block, BlockInfo> {
+
+ private Map<Long, BitWiseTrieContainerMap> containerPrefixMap
+ = new HashMap<Long, BitWiseTrieContainerMap>();
+ private int size;
+ public static final int PREFIX_LENGTH = 28;
+
+ @Override
+ public int size() {
+ // TODO: update size when new containers created
+ return size;
+ }
+
+ @Override
+ public boolean contains(Block key) {
+ return getBlockInfoContiguous(key.getBlockId()) != null;
+ }
+
+ @Override
+ public BlockInfoContiguous get(Block key) {
+ return getBlockInfoContiguous(key.getBlockId());
+ }
+
+ @Override
- public BlockInfoContiguous put(BlockInfoContiguous element) {
++ public BlockInfoContiguous put(BlockInfo element) {
+ BlockInfoContiguous info = getBlockInfoContiguous(element.getBlockId());
+ if (info == null) {
+ throw new IllegalStateException(
+ "The containers are created by splitting");
+ }
+ // TODO: replace
+ return info;
+ }
+
+ @Override
+ public BlockInfoContiguous remove(Block key) {
+ // It doesn't remove
+ return getBlockInfoContiguous(key.getBlockId());
+ }
+
+ @Override
+ public void clear() {
+ containerPrefixMap.clear();
+ }
+
+ @Override
- public Iterator<BlockInfoContiguous> iterator() {
++ public Iterator<BlockInfo> iterator() {
+ // TODO : Support iteration
+ throw new UnsupportedOperationException("");
+ }
+
+ /**
+ * Initialize a new trie for a new bucket.
+ */
+ public synchronized void initPrefix(long prefix) {
+ Preconditions.checkArgument((prefix >>> PREFIX_LENGTH) == 0,
+ "Prefix shouldn't be longer than "+PREFIX_LENGTH+" bits");
+ if (getTrieMap(prefix << (64 - PREFIX_LENGTH)) != null) {
+ // Already initialized
+ return;
+ }
+ BitWiseTrieContainerMap newTrie = new BitWiseTrieContainerMap(prefix,
+ PREFIX_LENGTH);
+ containerPrefixMap.put(prefix, newTrie);
+ }
+
+ @VisibleForTesting
+ synchronized BitWiseTrieContainerMap getTrieMap(long containerId) {
+ long prefix = containerId >>> (64 - PREFIX_LENGTH);
+ return containerPrefixMap.get(prefix);
+ }
+
+ @VisibleForTesting
+ BlockInfoContiguous getBlockInfoContiguous(long containerId) {
+ BitWiseTrieContainerMap map = getTrieMap(containerId);
+ if (map == null) {
+ return null;
+ }
+ return map.get(containerId);
+ }
+
+ public void splitContainer(long key) {
+ BitWiseTrieContainerMap map = getTrieMap(key);
+ if (map == null) {
+ throw new IllegalArgumentException("No container exists");
+ }
+ map.addBit(key);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
index 4c42f11,0000000..23f2ade
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
@@@ -1,155 -1,0 +1,155 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.storagecontainer;
+
+import org.apache.hadoop.hdfs.protocol.Block;
- import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
++import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.Namesystem;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.AccessControlException;
+
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Namesystem implementation to be used by StorageContainerManager.
+ */
+public class StorageContainerNameService implements Namesystem {
+
+ private ReentrantReadWriteLock coarseLock = new ReentrantReadWriteLock();
+ private String blockPoolId;
+ private volatile boolean serviceRunning = true;
+
+ public void shutdown() {
+ serviceRunning = false;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return serviceRunning;
+ }
+
+ @Override
+ public void checkSuperuserPrivilege() throws AccessControlException {
+ // TBD
+ }
+
+ @Override
+ public String getBlockPoolId() {
+ return blockPoolId;
+ }
+
+ public void setBlockPoolId(String id) {
+ this.blockPoolId = id;
+ }
+
+ @Override
+ public boolean isInStandbyState() {
+ // HA mode is not supported
+ return false;
+ }
+
+ @Override
+ public boolean isGenStampInFuture(Block block) {
+ // HA mode is not supported
+ return false;
+ }
+
+ @Override
+ public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal) {
+ // TBD
+ }
+
+ @Override
+ public void checkOperation(NameNode.OperationCategory read)
+ throws StandbyException {
+ // HA mode is not supported
+ }
+
+ @Override
- public boolean isInSnapshot(BlockInfoContiguousUnderConstruction blockUC) {
++ public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
+ // Snapshots not supported
+ return false;
+ }
+
+ @Override
+ public void readLock() {
+ coarseLock.readLock().lock();
+ }
+
+ @Override
+ public void readUnlock() {
+ coarseLock.readLock().unlock();
+ }
+
+ @Override
+ public boolean hasReadLock() {
+ return coarseLock.getReadHoldCount() > 0 || hasWriteLock();
+ }
+
+ @Override
+ public void writeLock() {
+ coarseLock.writeLock().lock();
+ }
+
+ @Override
+ public void writeLockInterruptibly() throws InterruptedException {
+ coarseLock.writeLock().lockInterruptibly();
+ }
+
+ @Override
+ public void writeUnlock() {
+ coarseLock.writeLock().unlock();
+ }
+
+ @Override
+ public boolean hasWriteLock() {
+ return coarseLock.isWriteLockedByCurrentThread();
+ }
+
+ @Override
+ public void checkSafeMode() {
+ // TBD
+ }
+
+ @Override
+ public boolean isInSafeMode() {
+ return false;
+ }
+
+ @Override
+ public boolean isInStartupSafeMode() {
+ return false;
+ }
+
+ @Override
+ public boolean isPopulatingReplQueues() {
+ return false;
+ }
+
+ @Override
+ public void incrementSafeBlockCount(int replication) {
+ // Do nothing
+ }
+
+ @Override
+ public void decrementSafeBlockCount(Block b) {
+ // Do nothing
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cec4909/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
----------------------------------------------------------------------