You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2015/08/26 00:22:44 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a070ee4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a070ee4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a070ee4
Branch: refs/heads/HDFS-7240
Commit: 8a070ee489127fdb9d7048b7dbcb7eae74ba539d
Parents: c785187 eee0d45
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Aug 25 15:20:49 2015 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Aug 25 15:20:49 2015 -0700
----------------------------------------------------------------------
BUILDING.txt | 38 +-
dev-support/docker/Dockerfile | 8 +
dev-support/test-patch.sh | 6 +
.../server/AuthenticationFilter.java | 63 +-
.../server/AuthenticationToken.java | 12 +
.../security/authentication/util/AuthToken.java | 35 +-
.../server/TestAuthenticationFilter.java | 163 +++-
hadoop-common-project/hadoop-common/CHANGES.txt | 99 ++-
hadoop-common-project/hadoop-common/pom.xml | 33 +
.../hadoop-common/src/main/bin/hadoop | 8 +-
.../hadoop-common/src/main/bin/hadoop-config.sh | 73 +-
.../src/main/bin/hadoop-functions.sh | 369 +++++++--
.../hadoop-common/src/main/bin/slaves.sh | 3 +-
.../src/main/conf/log4j.properties | 13 +
.../org/apache/hadoop/conf/Configuration.java | 9 +-
.../org/apache/hadoop/fs/BlockLocation.java | 55 +-
.../fs/CommonConfigurationKeysPublic.java | 5 +
.../java/org/apache/hadoop/fs/CreateFlag.java | 2 +-
.../apache/hadoop/fs/DelegateToFileSystem.java | 19 +-
.../main/java/org/apache/hadoop/fs/FsShell.java | 19 +-
.../apache/hadoop/fs/RawLocalFileSystem.java | 36 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 11 +-
.../hadoop/fs/permission/UmaskParser.java | 2 +-
.../org/apache/hadoop/io/ReadaheadPool.java | 4 +-
.../org/apache/hadoop/io/nativeio/NativeIO.java | 83 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 60 ++
.../apache/hadoop/ipc/WritableRpcEngine.java | 3 +
.../apache/hadoop/ipc/metrics/RpcMetrics.java | 54 +-
.../metrics2/impl/MetricsSinkAdapter.java | 6 +-
.../apache/hadoop/metrics2/lib/MutableStat.java | 7 +-
.../org/apache/hadoop/metrics2/util/MBeans.java | 37 +-
.../org/apache/hadoop/net/NetworkTopology.java | 38 +-
.../apache/hadoop/security/SaslPlainServer.java | 2 +-
.../org/apache/hadoop/util/HostsFileReader.java | 7 +-
.../org/apache/hadoop/util/JvmPauseMonitor.java | 12 +-
.../java/org/apache/hadoop/util/RunJar.java | 3 +-
.../main/java/org/apache/hadoop/util/Shell.java | 11 +-
.../org/apache/hadoop/util/StringUtils.java | 29 +-
.../org/apache/hadoop/io/nativeio/NativeIO.c | 149 ++--
.../src/main/resources/core-default.xml | 27 +
.../src/site/markdown/HttpAuthentication.md | 8 +-
.../hadoop-common/src/site/markdown/Metrics.md | 4 +-
.../src/site/markdown/SingleCluster.md.vm | 2 +-
.../apache/hadoop/crypto/TestCryptoCodec.java | 11 +-
...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 2 +-
.../fs/FileContextMainOperationsBaseTest.java | 4 +-
.../org/apache/hadoop/fs/TestBlockLocation.java | 23 +-
.../java/org/apache/hadoop/fs/TestFsShell.java | 28 +
.../fs/contract/AbstractContractAppendTest.java | 4 +-
.../AbstractContractGetFileStatusTest.java | 61 ++
.../contract/AbstractContractSetTimesTest.java | 61 ++
.../hadoop/fs/contract/ContractOptions.java | 12 +
.../TestLocalFSContractGetFileStatus.java | 33 +
.../localfs/TestLocalFSContractSetTimes.java | 33 +
.../TestRawlocalContractGetFileStatus.java | 33 +
.../rawlocal/TestRawlocalContractSetTimes.java | 33 +
.../hadoop/io/TestSequenceFileAppend.java | 6 +-
.../apache/hadoop/io/nativeio/TestNativeIO.java | 90 ++-
.../org/apache/hadoop/ipc/TestProtoBufRpc.java | 77 +-
.../metrics2/impl/TestMetricsSystemImpl.java | 60 ++
.../apache/hadoop/test/GenericTestUtils.java | 9 +-
.../org/apache/hadoop/test/MetricsAsserts.java | 2 +-
.../apache/hadoop/tracing/SetSpanReceiver.java | 2 +-
.../java/org/apache/hadoop/util/TestShell.java | 39 +
.../hadoop-common/src/test/proto/test.proto | 7 +
.../src/test/proto/test_rpc_service.proto | 1 +
.../src/test/resources/contract/localfs.xml | 10 +
.../src/test/resources/contract/rawlocal.xml | 10 +
.../scripts/hadoop-functions_test_helper.bash | 56 ++
.../src/test/scripts/hadoop_add_classpath.bats | 100 +++
.../src/test/scripts/hadoop_add_colonpath.bats | 96 +++
.../scripts/hadoop_add_common_to_classpath.bats | 71 ++
.../test/scripts/hadoop_add_javalibpath.bats | 98 +++
.../src/test/scripts/hadoop_add_ldlibpath.bats | 97 +++
.../src/test/scripts/hadoop_add_param.bats | 49 ++
.../hadoop_add_to_classpath_userpath.bats | 98 +++
.../src/test/scripts/hadoop_basic_init.bats | 94 +++
.../src/test/scripts/hadoop_bootstrap.bats | 51 ++
.../src/test/scripts/hadoop_confdir.bats | 92 +++
.../test/scripts/hadoop_deprecate_envvar.bats | 32 +
.../src/test/scripts/hadoop_finalize.bats | 206 +++++
.../scripts/hadoop_finalize_catalina_opts.bats | 56 ++
.../test/scripts/hadoop_finalize_classpath.bats | 64 ++
.../scripts/hadoop_finalize_hadoop_heap.bats | 87 ++
.../scripts/hadoop_finalize_hadoop_opts.bats | 52 ++
.../test/scripts/hadoop_finalize_libpaths.bats | 30 +
.../src/test/scripts/hadoop_java_setup.bats | 47 ++
.../src/test/scripts/hadoop_os_tricks.bats | 34 +
.../src/test/scripts/hadoop_rotate_log.bats | 52 ++
.../src/test/scripts/hadoop_shellprofile.bats | 91 +++
.../src/test/scripts/hadoop_slaves.bats | 37 +
.../src/test/scripts/hadoop_ssh.bats | 51 ++
.../scripts/hadoop_translate_cygwin_path.bats | 48 ++
.../test/scripts/hadoop_validate_classname.bats | 26 +
.../hadoop-common/src/test/scripts/run-bats.sh | 43 +
.../hadoop-kms/src/main/sbin/kms.sh | 29 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 2 +
.../org/apache/hadoop/hdfs/ExtendedBlockId.java | 82 ++
.../org/apache/hadoop/hdfs/ReplicaAccessor.java | 88 +++
.../hadoop/hdfs/ReplicaAccessorBuilder.java | 101 +++
.../hdfs/client/HdfsClientConfigKeys.java | 76 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 785 +++++++++++++++++++
.../hadoop/hdfs/client/impl/package-info.java | 18 +
.../org/apache/hadoop/hdfs/net/DomainPeer.java | 132 ++++
.../java/org/apache/hadoop/hdfs/net/Peer.java | 123 +++
.../hadoop/hdfs/protocol/ClientProtocol.java | 14 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 10 +
.../datatransfer/BlockConstructionStage.java | 62 ++
.../datatransfer/DataTransferProtoUtil.java | 146 ++++
.../datatransfer/DataTransferProtocol.java | 202 +++++
.../hadoop/hdfs/protocol/datatransfer/Op.java | 66 ++
.../hdfs/protocol/datatransfer/Sender.java | 261 ++++++
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 254 ++++++
.../token/block/InvalidBlockTokenException.java | 41 +
.../hdfs/server/datanode/CachingStrategy.java | 76 ++
.../hadoop/hdfs/shortcircuit/DfsClientShm.java | 119 +++
.../hdfs/shortcircuit/DfsClientShmManager.java | 522 ++++++++++++
.../hdfs/shortcircuit/ShortCircuitShm.java | 647 +++++++++++++++
.../hadoop/hdfs/util/ByteArrayManager.java | 422 ++++++++++
.../hadoop/hdfs/util/ExactSizeInputStream.java | 125 +++
.../apache/hadoop/hdfs/util/package-info.java | 18 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 20 +
.../hdfs/web/resources/CreateFlagParam.java | 48 ++
.../hdfs/web/resources/CreateParentParam.java | 2 +-
.../src/main/proto/ClientDatanodeProtocol.proto | 33 -
.../src/main/proto/ClientNamenodeProtocol.proto | 10 +
.../src/main/proto/datatransfer.proto | 4 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +-
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 29 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 208 ++++-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +
.../hadoop-hdfs/src/CMakeLists.txt | 1 +
.../hadoop-hdfs/src/main/bin/hdfs | 10 +-
.../hadoop-hdfs/src/main/bin/start-balancer.sh | 12 +-
.../hadoop-hdfs/src/main/bin/stop-balancer.sh | 10 +-
.../apache/hadoop/fs/BlockStorageLocation.java | 51 --
.../java/org/apache/hadoop/fs/HdfsVolumeId.java | 73 --
.../java/org/apache/hadoop/fs/SWebHdfs.java | 51 ++
.../java/org/apache/hadoop/fs/VolumeId.java | 40 -
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 51 ++
.../main/java/org/apache/hadoop/fs/package.html | 26 +
.../apache/hadoop/hdfs/BlockReaderFactory.java | 60 +-
.../hadoop/hdfs/BlockStorageLocationUtil.java | 368 ---------
.../org/apache/hadoop/hdfs/ClientContext.java | 5 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 132 +---
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 195 +++--
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 3 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 6 +-
.../hadoop/hdfs/DistributedFileSystem.java | 33 -
.../org/apache/hadoop/hdfs/ExtendedBlockId.java | 82 --
.../apache/hadoop/hdfs/ExternalBlockReader.java | 120 +++
.../apache/hadoop/hdfs/HdfsConfiguration.java | 8 +-
.../apache/hadoop/hdfs/RemoteBlockReader.java | 4 +-
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 4 +-
.../org/apache/hadoop/hdfs/XAttrHelper.java | 13 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 747 ------------------
.../org/apache/hadoop/hdfs/net/DomainPeer.java | 132 ----
.../java/org/apache/hadoop/hdfs/net/Peer.java | 123 ---
.../hdfs/protocol/ClientDatanodeProtocol.java | 19 -
.../hdfs/protocol/HdfsBlocksMetadata.java | 111 ---
.../datatransfer/BlockConstructionStage.java | 62 --
.../datatransfer/DataTransferProtoUtil.java | 148 ----
.../datatransfer/DataTransferProtocol.java | 201 -----
.../hadoop/hdfs/protocol/datatransfer/Op.java | 65 --
.../hdfs/protocol/datatransfer/PipelineAck.java | 2 +-
.../hdfs/protocol/datatransfer/Receiver.java | 8 +-
.../hdfs/protocol/datatransfer/Sender.java | 261 ------
.../datatransfer/sasl/DataTransferSaslUtil.java | 2 +-
...tDatanodeProtocolServerSideTranslatorPB.java | 43 +-
.../ClientDatanodeProtocolTranslatorPB.java | 49 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 23 +-
.../ClientNamenodeProtocolTranslatorPB.java | 41 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 4 +-
.../InterDatanodeProtocolTranslatorPB.java | 2 +-
.../NamenodeProtocolTranslatorPB.java | 2 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 228 +-----
.../token/block/InvalidBlockTokenException.java | 41 -
.../hadoop/hdfs/server/balancer/Balancer.java | 182 +++--
.../hadoop/hdfs/server/balancer/Dispatcher.java | 186 +++--
.../hdfs/server/balancer/MovedBlocks.java | 5 +
.../server/blockmanagement/BlockCollection.java | 9 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 164 ++--
.../blockmanagement/BlockInfoContiguous.java | 82 +-
.../BlockInfoUnderConstruction.java | 405 ----------
.../BlockInfoUnderConstructionContiguous.java | 110 ---
.../server/blockmanagement/BlockManager.java | 780 ++++++++----------
.../BlockPlacementPolicyDefault.java | 177 +++--
.../BlockPlacementPolicyWithNodeGroup.java | 35 +-
.../BlockStoragePolicySuite.java | 11 +-
.../BlockUnderConstructionFeature.java | 254 ++++++
.../ContiguousBlockStorageOp.java | 106 ---
.../blockmanagement/CorruptReplicasMap.java | 4 +-
.../blockmanagement/DatanodeDescriptor.java | 75 +-
.../server/blockmanagement/DatanodeManager.java | 19 +-
.../blockmanagement/DatanodeStorageInfo.java | 15 +-
.../blockmanagement/DecommissionManager.java | 15 +-
.../blockmanagement/InvalidateBlocks.java | 2 +-
.../ReplicaUnderConstruction.java | 119 +++
.../blockmanagement/UnderReplicatedBlocks.java | 33 +-
.../hdfs/server/common/HdfsServerConstants.java | 9 -
.../hdfs/server/datanode/BlockReceiver.java | 14 +-
.../hdfs/server/datanode/BlockSender.java | 11 +-
.../hdfs/server/datanode/CachingStrategy.java | 76 --
.../hadoop/hdfs/server/datanode/DNConf.java | 30 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 49 +-
.../hdfs/server/datanode/DataXceiver.java | 77 +-
.../server/datanode/SecureDataNodeStarter.java | 4 +-
.../hdfs/server/datanode/VolumeScanner.java | 12 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 18 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 45 --
.../datanode/web/webhdfs/ParameterParser.java | 14 +
.../datanode/web/webhdfs/WebHdfsHandler.java | 23 +-
.../hdfs/server/namenode/Checkpointer.java | 4 +-
.../hdfs/server/namenode/FSDirAppendOp.java | 4 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 73 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 9 +-
.../hdfs/server/namenode/FSDirDeleteOp.java | 5 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 7 +-
.../hdfs/server/namenode/FSDirSnapshotOp.java | 2 +
.../server/namenode/FSDirStatAndListingOp.java | 9 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 40 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 30 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 29 +-
.../hdfs/server/namenode/FSDirectory.java | 77 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 11 +
.../hdfs/server/namenode/FSEditLogLoader.java | 30 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 6 +-
.../hdfs/server/namenode/FSImageFormat.java | 6 +-
.../server/namenode/FSImageFormatPBINode.java | 18 +-
.../server/namenode/FSImageSerialization.java | 10 +-
.../hdfs/server/namenode/FSNamesystem.java | 161 ++--
.../hdfs/server/namenode/FSNamesystemLock.java | 11 +
.../namenode/FileUnderConstructionFeature.java | 10 +-
.../hadoop/hdfs/server/namenode/INode.java | 52 +-
.../hdfs/server/namenode/INodeDirectory.java | 11 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 60 +-
.../hdfs/server/namenode/LeaseManager.java | 17 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 176 ++++-
.../hdfs/server/namenode/NameNodeRpcServer.java | 6 +
.../hdfs/server/namenode/NamenodeFsck.java | 25 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 4 +-
.../server/namenode/SerialNumberManager.java | 44 --
.../hdfs/server/namenode/SerialNumberMap.java | 79 ++
.../hdfs/server/namenode/XAttrFeature.java | 78 +-
.../hdfs/server/namenode/XAttrFormat.java | 155 ++++
.../server/namenode/XAttrPermissionFilter.java | 6 +-
.../hdfs/server/namenode/XAttrStorage.java | 62 +-
.../hdfs/server/namenode/ha/StandbyState.java | 3 +-
.../namenode/metrics/FSNamesystemMBean.java | 19 +
.../snapshot/FSImageFormatPBSnapshot.java | 12 +-
.../server/namenode/snapshot/FileDiffList.java | 3 +-
.../snapshot/FileWithSnapshotFeature.java | 44 +-
.../web/resources/NamenodeWebHdfsMethods.java | 20 +-
.../hdfs/server/protocol/NamespaceInfo.java | 17 +-
.../hadoop/hdfs/shortcircuit/DfsClientShm.java | 119 ---
.../hdfs/shortcircuit/DfsClientShmManager.java | 514 ------------
.../hdfs/shortcircuit/DomainSocketFactory.java | 4 +-
.../hdfs/shortcircuit/ShortCircuitCache.java | 4 +-
.../hdfs/shortcircuit/ShortCircuitShm.java | 646 ---------------
.../offlineImageViewer/FSImageHandler.java | 3 +
.../tools/offlineImageViewer/FSImageLoader.java | 90 +++
.../hadoop/hdfs/util/ByteArrayManager.java | 418 ----------
.../hadoop/hdfs/util/ExactSizeInputStream.java | 125 ---
.../hadoop/hdfs/util/LightWeightHashSet.java | 21 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 6 +-
.../StorageContainerNameService.java | 3 +-
.../src/main/resources/hdfs-default.xml | 40 +-
.../src/main/webapps/hdfs/dfshealth.html | 62 +-
.../src/main/webapps/hdfs/dfshealth.js | 63 +-
.../webapps/static/dataTables.bootstrap.css | 371 +++++++++
.../main/webapps/static/dataTables.bootstrap.js | 205 +++++
.../src/main/webapps/static/hadoop.css | 51 ++
.../webapps/static/jquery.dataTables.min.js | 160 ++++
.../src/main/webapps/static/moment.min.js | 7 +
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 5 +-
.../src/site/markdown/HdfsImageViewer.md | 1 +
.../src/site/markdown/HdfsMultihoming.md | 2 +
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 23 +-
.../org/apache/hadoop/hdfs/TestFiPipelines.java | 9 +-
.../datanode/TestFiDataTransferProtocol.java | 3 +-
.../datanode/TestFiDataTransferProtocol2.java | 5 +-
.../hadoop/fs/TestEnhancedByteBufferAccess.java | 10 +-
.../TestSWebHdfsFileContextMainOperations.java | 110 +++
.../java/org/apache/hadoop/fs/TestUnbuffer.java | 7 +-
.../java/org/apache/hadoop/fs/TestVolumeId.java | 146 ----
.../TestWebHdfsFileContextMainOperations.java | 157 ++++
.../contract/hdfs/TestHDFSContractAppend.java | 15 -
.../hdfs/TestHDFSContractGetFileStatus.java | 46 ++
.../contract/hdfs/TestHDFSContractSetTimes.java | 45 ++
.../fs/viewfs/TestViewFsDefaultValue.java | 8 +-
.../apache/hadoop/hdfs/BlockReaderTestUtil.java | 6 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 11 +-
.../org/apache/hadoop/hdfs/FileAppendTest4.java | 5 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 30 +-
.../hadoop/hdfs/TestAppendSnapshotTruncate.java | 13 +-
.../hadoop/hdfs/TestBlockReaderFactory.java | 4 +-
.../hadoop/hdfs/TestBlockReaderLocal.java | 4 +-
.../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 6 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 141 +++-
.../TestClientProtocolForPipelineRecovery.java | 4 +-
.../org/apache/hadoop/hdfs/TestConnCache.java | 5 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 8 +-
.../apache/hadoop/hdfs/TestDFSInputStream.java | 2 +-
.../hadoop/hdfs/TestDataTransferKeepalive.java | 8 +-
.../hadoop/hdfs/TestDataTransferProtocol.java | 6 +-
.../apache/hadoop/hdfs/TestDatanodeDeath.java | 5 +-
.../apache/hadoop/hdfs/TestDecommission.java | 98 +++
.../hadoop/hdfs/TestDisableConnCache.java | 3 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 263 ++-----
.../hadoop/hdfs/TestExternalBlockReader.java | 298 +++++++
.../org/apache/hadoop/hdfs/TestFileAppend.java | 4 +-
.../org/apache/hadoop/hdfs/TestFileAppend2.java | 6 +-
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 5 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 165 ++--
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 3 +-
.../apache/hadoop/hdfs/TestParallelRead.java | 2 +-
.../TestParallelShortCircuitLegacyRead.java | 4 +-
.../TestParallelShortCircuitReadUnCached.java | 6 +-
.../hadoop/hdfs/TestParallelUnixDomainRead.java | 2 +-
.../org/apache/hadoop/hdfs/TestPipelines.java | 9 +-
.../java/org/apache/hadoop/hdfs/TestPread.java | 6 +-
.../java/org/apache/hadoop/hdfs/TestRead.java | 5 +-
.../hadoop/hdfs/TestRemoteBlockReader.java | 4 +-
.../hdfs/TestReplaceDatanodeOnFailure.java | 4 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 26 +-
.../hdfs/protocol/TestBlockListAsLongs.java | 9 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 20 +-
.../hdfs/server/balancer/TestBalancer.java | 188 ++++-
.../server/blockmanagement/TestBlockInfo.java | 2 +-
.../TestBlockInfoUnderConstruction.java | 82 --
.../blockmanagement/TestBlockManager.java | 16 +-
.../TestBlockReportRateLimiting.java | 2 -
.../blockmanagement/TestBlockTokenWithDFS.java | 6 +-
.../TestBlockUnderConstructionFeature.java | 80 ++
.../blockmanagement/TestHeartbeatHandling.java | 22 +-
.../server/blockmanagement/TestNodeCount.java | 2 +-
.../TestOverReplicatedBlocks.java | 4 +-
.../blockmanagement/TestPendingReplication.java | 1 -
.../blockmanagement/TestReplicationPolicy.java | 98 ++-
.../TestUnderReplicatedBlocks.java | 48 --
.../server/datanode/SimulatedFSDataset.java | 8 -
.../server/datanode/TestBlockReplacement.java | 7 +-
.../server/datanode/TestCachingStrategy.java | 12 +-
.../datanode/TestDataNodeVolumeFailure.java | 6 +-
.../TestDataXceiverLazyPersistHint.java | 180 +++++
.../extdataset/ExternalDatasetImpl.java | 6 -
.../fsdataset/impl/LazyPersistTestCase.java | 5 +-
.../fsdataset/impl/TestDatanodeRestart.java | 7 +-
.../hdfs/server/mover/TestStorageMover.java | 8 +-
.../namenode/TestBlockUnderConstruction.java | 6 +-
.../TestCommitBlockSynchronization.java | 12 +-
.../TestDefaultBlockPlacementPolicy.java | 49 +-
.../hdfs/server/namenode/TestFSNamesystem.java | 28 +
.../server/namenode/TestFSNamesystemMBean.java | 34 +-
.../hdfs/server/namenode/TestFileTruncate.java | 5 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 2 +
.../hdfs/server/namenode/TestHDFSConcat.java | 24 +-
.../hdfs/server/namenode/TestINodeFile.java | 7 +-
.../hdfs/server/namenode/TestLeaseManager.java | 65 +-
.../server/namenode/TestNameNodeMXBean.java | 23 +-
.../namenode/TestNameNodeMetricsLogger.java | 193 +++++
.../server/namenode/TestQuotaByStorageType.java | 14 +-
.../hdfs/server/namenode/TestStartup.java | 27 +-
.../hdfs/server/namenode/TestXAttrFeature.java | 107 +++
.../namenode/ha/TestRetryCacheWithHA.java | 11 +-
.../server/namenode/ha/TestStandbyIsHot.java | 2 +
.../namenode/snapshot/SnapshotTestHelper.java | 9 +-
.../snapshot/TestFileWithSnapshotFeature.java | 7 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 16 +-
.../snapshot/TestSnapshotReplication.java | 31 +-
.../shortcircuit/TestShortCircuitCache.java | 13 +-
.../shortcircuit/TestShortCircuitLocalRead.java | 6 +-
...TestOfflineImageViewerForContentSummary.java | 248 ++++++
.../hdfs/util/TestLightWeightHashSet.java | 29 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 25 +
.../apache/hadoop/net/TestNetworkTopology.java | 1 +
.../apache/hadoop/security/TestPermission.java | 26 +
.../src/test/resources/contract/hdfs.xml | 10 +
.../src/test/resources/log4j.properties | 13 +
.../src/test/resources/testHDFSConf.xml | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 34 +
hadoop-mapreduce-project/bin/mapred | 2 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +-
.../v2/app/job/event/JobStartEvent.java | 2 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 40 +-
.../mapreduce/v2/app/rm/RMCommunicator.java | 51 +-
.../app/rm/RMContainerAllocationException.java | 31 +
.../v2/app/rm/RMContainerAllocator.java | 4 +-
.../v2/api/records/TestTaskAttemptReport.java | 131 ++++
.../v2/api/records/TestTaskReport.java | 139 ++++
.../mapreduce/v2/app/TestMRAppMaster.java | 88 ++-
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 130 ++-
.../mapreduce/v2/app/rm/TestRMCommunicator.java | 99 +++
.../v2/app/rm/TestRMContainerAllocator.java | 48 +-
.../v2/api/records/TaskAttemptReport.java | 3 +
.../mapreduce/v2/api/records/TaskReport.java | 5 +-
.../impl/pb/TaskAttemptReportPBImpl.java | 38 +-
.../api/records/impl/pb/TaskReportPBImpl.java | 38 +-
.../java/org/apache/hadoop/mapred/TaskLog.java | 4 +-
.../mapreduce/jobhistory/EventWriter.java | 19 +-
.../mapreduce/lib/output/MultipleOutputs.java | 14 +-
.../src/main/resources/mapred-default.xml | 20 +
.../src/site/markdown/EncryptedShuffle.md | 8 +-
.../hadoop/mapreduce/v2/hs/CompletedTask.java | 2 +-
.../mapreduce/v2/hs/CompletedTaskAttempt.java | 2 +-
.../mapreduce/v2/hs/JobHistoryServer.java | 14 +-
.../hadoop/mapred/ResourceMgrDelegate.java | 7 +
.../hadoop/mapred/TestClientRedirect.java | 9 +
.../hadoop/mapred/FadvisedChunkedFile.java | 5 +-
.../hadoop/mapred/FadvisedFileRegion.java | 6 +-
hadoop-project/pom.xml | 4 +-
hadoop-tools/hadoop-aws/pom.xml | 2 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 9 +-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 26 +-
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 15 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 6 +
.../fs/azure/TestWasbUriAndConfiguration.java | 30 +-
.../src/test/resources/azure-test.xml | 4 +
.../org/apache/hadoop/tools/CopyListing.java | 15 +-
.../java/org/apache/hadoop/tools/DiffInfo.java | 32 +-
.../java/org/apache/hadoop/tools/DistCp.java | 27 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 4 +-
.../org/apache/hadoop/tools/DistCpSync.java | 308 +++++++-
.../apache/hadoop/tools/SimpleCopyListing.java | 151 +++-
.../org/apache/hadoop/tools/TestDistCpSync.java | 345 +++++++-
.../apache/hadoop/tools/TestOptionsParser.java | 22 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 3 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 5 +
hadoop-yarn-project/CHANGES.txt | 135 +++-
.../hadoop-yarn/bin/start-yarn.sh | 7 +-
.../hadoop-yarn/bin/stop-yarn.sh | 7 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 7 +
.../yarn/api/ApplicationClientProtocol.java | 18 +
.../UpdateApplicationPriorityRequest.java | 80 ++
.../UpdateApplicationPriorityResponse.java | 47 ++
.../yarn/api/records/ApplicationReport.java | 31 +-
.../yarn/api/records/LogAggregationContext.java | 95 +++
.../yarn/api/records/QueueStatistics.java | 36 +
.../hadoop/yarn/conf/YarnConfiguration.java | 61 +-
.../api/ContainerLogAggregationPolicy.java | 54 ++
.../yarn/server/api/ContainerLogContext.java | 71 ++
.../main/proto/applicationclient_protocol.proto | 1 +
.../src/main/proto/yarn_protos.proto | 7 +
.../src/main/proto/yarn_service_protos.proto | 8 +
.../distributedshell/ApplicationMaster.java | 31 +-
.../distributedshell/TestDSAppMaster.java | 11 +-
.../hadoop/yarn/client/api/YarnClient.java | 17 +
.../yarn/client/api/impl/YarnClientImpl.java | 11 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 35 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 24 +-
.../yarn/client/TestResourceTrackerOnHA.java | 2 +-
.../yarn/client/api/impl/TestYarnClient.java | 10 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 34 +-
.../ApplicationClientProtocolPBClientImpl.java | 20 +
.../ApplicationClientProtocolPBServiceImpl.java | 22 +
.../UpdateApplicationPriorityRequestPBImpl.java | 171 ++++
...UpdateApplicationPriorityResponsePBImpl.java | 69 ++
.../impl/pb/ApplicationReportPBImpl.java | 49 ++
.../impl/pb/LogAggregationContextPBImpl.java | 40 +
.../records/impl/pb/QueueStatisticsPBImpl.java | 36 +
.../hadoop/yarn/event/AsyncDispatcher.java | 9 +-
.../ContainerLogsRetentionPolicy.java | 29 -
.../nodelabels/CommonNodeLabelsManager.java | 22 +-
.../nodelabels/FileSystemNodeLabelsStore.java | 8 +-
.../hadoop/yarn/util/resource/Resources.java | 14 +-
.../src/main/resources/yarn-default.xml | 99 ++-
.../hadoop/yarn/api/TestApplicatonReport.java | 4 +-
.../hadoop/yarn/event/TestAsyncDispatcher.java | 25 +-
.../nodelabels/TestCommonNodeLabelsManager.java | 17 +-
...pplicationHistoryManagerOnTimelineStore.java | 28 +-
.../ApplicationHistoryServer.java | 14 +-
...pplicationHistoryManagerOnTimelineStore.java | 4 +
.../yarn/server/api/records/NodeStatus.java | 18 +-
.../api/records/impl/pb/NodeStatusPBImpl.java | 22 +
.../metrics/ApplicationMetricsConstants.java | 6 +
.../hadoop/yarn/server/utils/BuilderUtils.java | 4 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 24 +-
.../hadoop/yarn/server/webapp/AppsBlock.java | 12 +-
.../hadoop/yarn/server/webapp/WebPageUtils.java | 4 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 15 +
.../main/proto/yarn_server_common_protos.proto | 1 +
.../hadoop/yarn/server/nodemanager/Context.java | 2 +
.../yarn/server/nodemanager/NodeManager.java | 73 +-
.../server/nodemanager/NodeResourceMonitor.java | 10 +-
.../nodemanager/NodeResourceMonitorImpl.java | 140 ++++
.../nodemanager/NodeStatusUpdaterImpl.java | 280 +++++--
.../application/ApplicationImpl.java | 5 +-
.../AMOnlyLogAggregationPolicy.java | 31 +
...AMOrFailedContainerLogAggregationPolicy.java | 35 +
.../AbstractContainerLogAggregationPolicy.java | 31 +
.../AllContainerLogAggregationPolicy.java | 30 +
.../logaggregation/AppLogAggregator.java | 5 +-
.../logaggregation/AppLogAggregatorImpl.java | 131 ++--
.../FailedContainerLogAggregationPolicy.java | 33 +
...edOrKilledContainerLogAggregationPolicy.java | 30 +
.../logaggregation/LogAggregationService.java | 19 +-
.../NoneContainerLogAggregationPolicy.java | 30 +
.../SampleContainerLogAggregationPolicy.java | 124 +++
.../event/LogHandlerAppStartedEvent.java | 15 +-
.../monitor/ContainersMonitorImpl.java | 18 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 13 +-
.../nodelabels/AbstractNodeLabelsProvider.java | 146 ++++
.../ConfigurationNodeLabelsProvider.java | 81 ++
.../server/nodemanager/webapp/NodePage.java | 8 +-
.../server/nodemanager/webapp/dao/NodeInfo.java | 6 +
.../impl/container-executor.c | 63 +-
.../server/nodemanager/TestNodeManager.java | 50 +-
.../nodemanager/TestNodeResourceMonitor.java | 35 +
.../nodemanager/TestNodeStatusUpdater.java | 34 +
.../TestNodeStatusUpdaterForLabels.java | 76 +-
.../containermanager/TestAuxServices.java | 1 +
.../launcher/TestContainerLaunch.java | 2 +-
.../TestLogAggregationService.java | 677 ++++++++++++++--
.../TestNonAggregatingLogHandler.java | 12 +-
.../monitor/TestContainersMonitor.java | 2 +-
.../TestConfigurationNodeLabelsProvider.java | 146 ++++
.../nodemanager/webapp/TestNMWebServices.java | 2 +-
.../ApplicationMasterService.java | 3 +-
.../server/resourcemanager/ClientRMService.java | 84 +-
.../resourcemanager/NodesListManager.java | 28 +-
.../resourcemanager/RMActiveServiceContext.java | 30 -
.../server/resourcemanager/RMAuditLogger.java | 2 +
.../server/resourcemanager/RMContextImpl.java | 23 +-
.../server/resourcemanager/ResourceManager.java | 43 +-
.../resourcemanager/ResourceTrackerService.java | 2 +
.../metrics/ApplicationCreatedEvent.java | 17 +-
.../metrics/SystemMetricsPublisher.java | 9 +-
.../nodelabels/RMNodeLabelsManager.java | 14 +-
.../recovery/FileSystemRMStateStore.java | 209 +++--
.../recovery/LeveldbRMStateStore.java | 114 ++-
.../recovery/MemoryRMStateStore.java | 57 ++
.../recovery/NullRMStateStore.java | 23 +
.../resourcemanager/recovery/RMStateStore.java | 166 +++-
.../recovery/RMStateStoreEventType.java | 5 +-
.../RMStateStoreStoreReservationEvent.java | 56 ++
.../recovery/RMStateUpdateAppEvent.java | 13 +
.../recovery/ZKRMStateStore.java | 124 ++-
.../resourcemanager/reservation/Plan.java | 3 +-
.../resourcemanager/reservation/PlanView.java | 11 +-
.../ReservationSchedulerConfiguration.java | 14 +-
.../reservation/ReservationSystem.java | 15 +-
.../reservation/ReservationSystemUtil.java | 102 ++-
.../server/resourcemanager/rmapp/RMAppImpl.java | 12 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 22 +-
.../rmcontainer/RMContainerImpl.java | 70 +-
.../server/resourcemanager/rmnode/RMNode.java | 7 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 23 +-
.../scheduler/AbstractYarnScheduler.java | 17 +-
.../scheduler/SchedulerApplicationAttempt.java | 2 +-
.../scheduler/YarnScheduler.java | 22 +
.../scheduler/capacity/AbstractCSQueue.java | 11 +-
.../scheduler/capacity/CSAssignment.java | 11 +-
.../scheduler/capacity/CapacityScheduler.java | 115 ++-
.../capacity/CapacitySchedulerContext.java | 2 -
.../scheduler/capacity/LeafQueue.java | 90 ++-
.../scheduler/capacity/ParentQueue.java | 18 +-
.../capacity/allocator/AllocationState.java | 28 +
.../capacity/allocator/ContainerAllocation.java | 89 +++
.../capacity/allocator/ContainerAllocator.java | 150 ++++
.../allocator/RegularContainerAllocator.java | 698 +++++++++++++++++
.../scheduler/common/fica/FiCaSchedulerApp.java | 639 +--------------
.../scheduler/fair/FSAppAttempt.java | 4 +-
.../AbstractComparatorOrderingPolicy.java | 6 +
.../ClientToAMTokenSecretManagerInRM.java | 7 +
.../resourcemanager/webapp/RMAppsBlock.java | 7 +-
.../resourcemanager/webapp/dao/AppInfo.java | 19 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 18 +
.../webapp/dao/FairSchedulerQueueInfo.java | 24 +-
.../yarn_server_resourcemanager_recovery.proto | 18 +-
.../yarn/server/resourcemanager/MockAM.java | 14 +-
.../yarn/server/resourcemanager/MockNodes.java | 4 +
.../server/resourcemanager/TestAppManager.java | 5 +-
.../resourcemanager/TestClientRMService.java | 64 ++
.../resourcemanager/TestRMNodeTransitions.java | 62 +-
.../resourcemanager/TestResourceManager.java | 1 +
.../TestWorkPreservingRMRestart.java | 2 +-
.../TestRMAppLogAggregationStatus.java | 4 +-
.../metrics/TestSystemMetricsPublisher.java | 16 +
...pacityPreemptionPolicyForNodePartitions.java | 10 +-
.../nodelabels/TestRMNodeLabelsManager.java | 48 +-
.../recovery/RMStateStoreTestBase.java | 186 +++++
.../recovery/TestFSRMStateStore.java | 1 +
.../recovery/TestLeveldbRMStateStore.java | 6 +
.../recovery/TestZKRMStateStore.java | 1 +
.../reservation/ReservationSystemTestUtil.java | 193 ++---
.../TestCapacityReservationSystem.java | 94 ---
.../TestCapacitySchedulerPlanFollower.java | 11 +-
.../reservation/TestFairReservationSystem.java | 127 ---
.../TestFairSchedulerPlanFollower.java | 71 +-
.../TestInMemoryReservationAllocation.java | 36 +-
.../reservation/TestReservationSystem.java | 213 +++++
.../resourcetracker/TestNMReconnect.java | 39 +
.../rmapp/TestNodesListManager.java | 162 ++++
.../rmapp/TestRMAppTransitions.java | 6 +-
.../attempt/TestRMAppAttemptTransitions.java | 40 +-
.../rmcontainer/TestRMContainerImpl.java | 79 +-
.../capacity/TestApplicationLimits.java | 14 +-
.../capacity/TestApplicationPriority.java | 329 ++++++--
.../capacity/TestCapacityScheduler.java | 83 +-
.../scheduler/capacity/TestChildQueueOrder.java | 3 +-
.../capacity/TestContainerAllocation.java | 12 +-
.../scheduler/capacity/TestLeafQueue.java | 356 +++++----
.../scheduler/capacity/TestParentQueue.java | 2 -
.../scheduler/capacity/TestReservations.java | 8 +-
.../scheduler/capacity/TestUtils.java | 9 +-
.../scheduler/fifo/TestFifoScheduler.java | 19 +-
.../webapp/TestRMWebServicesApps.java | 36 +-
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
.../yarn/server/webproxy/WebAppProxyServer.java | 32 +-
.../src/site/markdown/ResourceManagerRest.md | 20 +-
.../src/site/markdown/TimelineServer.md | 32 +-
614 files changed, 24687 insertions(+), 10842 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index 7131c7d,42934c3..786dd1a
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@@ -19,10 -19,7 +19,9 @@@ package org.apache.hadoop.hdfs.server.b
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
- import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import java.util.LinkedList;
+
/**
* Subclass of {@link BlockInfo}, used for a block with replication scheme.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 144d8fb,7f02612..4dd2205
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@@ -91,14 -99,6 +99,7 @@@ import com.google.common.base.Precondit
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import java.io.IOException;
- import java.io.PrintWriter;
- import java.util.*;
- import java.util.concurrent.ThreadLocalRandom;
- import java.util.concurrent.atomic.AtomicLong;
-
- import static org.apache.hadoop.util.ExitUtil.terminate;
+
/**
* Keeps information related to the blocks stored in the Hadoop cluster.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index de30e08,7e3c59b..fd1c0f0
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@@ -17,10 -17,22 +17,8 @@@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-
import com.google.common.annotations.VisibleForTesting;
-
import com.google.common.collect.ImmutableList;
- import org.apache.commons.logging.Log;
- import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.StorageType;
@@@ -36,9 -49,9 +35,11 @@@ import org.apache.hadoop.hdfs.util.Enum
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
import org.apache.hadoop.util.IntrusiveCollection;
import org.apache.hadoop.util.Time;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+import java.util.*;
+
/**
* This class extends the DatanodeInfo class with ephemeral information (eg
* health, capacity, what blocks are associated with the Datanode) that is
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index a325a19,216d6d2..8bd8c51
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@@ -17,9 -17,13 +17,8 @@@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
import com.google.common.annotations.VisibleForTesting;
-
import org.apache.hadoop.fs.StorageType;
- import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index f393aa0,1cb308f..2468453
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@@ -791,15 -791,15 +794,15 @@@ class BlockReceiver implements Closeabl
//
if (syncBehindWrites) {
if (syncBehindWritesInBackground) {
- this.datanode.getFSDataset().submitBackgroundSyncFileRangeRequest(
+ dataset.submitBackgroundSyncFileRangeRequest(
block, outFd, lastCacheManagementOffset,
offsetInBlock - lastCacheManagementOffset,
- NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
+ SYNC_FILE_RANGE_WRITE);
} else {
NativeIO.POSIX.syncFileRangeIfPossible(outFd,
- lastCacheManagementOffset, offsetInBlock
- - lastCacheManagementOffset,
- NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
+ lastCacheManagementOffset,
+ offsetInBlock - lastCacheManagementOffset,
+ SYNC_FILE_RANGE_WRITE);
}
}
//
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 75b0d26,dfaa525..11ad30a
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@@ -643,14 -628,9 +642,16 @@@ class DataXceiver extends Receiver impl
final boolean isClient = !isDatanode;
final boolean isTransfer = stage == BlockConstructionStage.TRANSFER_RBW
|| stage == BlockConstructionStage.TRANSFER_FINALIZED;
+ allowLazyPersist = allowLazyPersist &&
+ (dnConf.getAllowNonLocalLazyPersist() || peer.isLocal());
long size = 0;
+ final FsDatasetSpi<?> dataset =
+ (FsDatasetSpi<?>) datanode.getDataset(block.getBlockPoolId());
+ if (dataset == null) {
+ throw new IOException(
+ "Unknown or unitialized blockpool " + block.getBlockPoolId());
+ }
+
// check single target for transfer-RBW/Finalized
if (isTransfer && targets.length > 0) {
throw new IOException(stage + " does not support multiple targets "
@@@ -1208,10 -1158,10 +1205,10 @@@
DataChecksum remoteChecksum = DataTransferProtoUtil.fromProto(
checksumInfo.getChecksum());
// open a block receiver and check if the block does not exist
- blockReceiver = new BlockReceiver(block, storageType,
+ blockReceiver = getBlockReceiver(block, storageType,
proxyReply, proxySock.getRemoteSocketAddress().toString(),
proxySock.getLocalSocketAddress().toString(),
- null, 0, 0, 0, "", null, datanode, remoteChecksum,
+ null, 0, 0, 0, "", null, datanode, dataset, remoteChecksum,
CachingStrategy.newDropBehind(), false, false);
// receive a block
@@@ -1263,6 -1213,39 +1260,40 @@@
datanode.metrics.addReplaceBlockOp(elapsed());
}
+
+ /**
+ * Separated for testing.
+ */
+ @VisibleForTesting
+ BlockReceiver getBlockReceiver(
+ final ExtendedBlock block, final StorageType storageType,
+ final DataInputStream in,
+ final String inAddr, final String myAddr,
+ final BlockConstructionStage stage,
+ final long newGs, final long minBytesRcvd, final long maxBytesRcvd,
+ final String clientname, final DatanodeInfo srcDataNode,
- final DataNode dn, DataChecksum requestedChecksum,
++ final DataNode dn, final FsDatasetSpi<?> dataset,
++ DataChecksum requestedChecksum,
+ CachingStrategy cachingStrategy,
+ final boolean allowLazyPersist,
+ final boolean pinning) throws IOException {
+ return new BlockReceiver(block, storageType, in,
+ inAddr, myAddr, stage, newGs, minBytesRcvd, maxBytesRcvd,
- clientname, srcDataNode, dn, requestedChecksum,
++ clientname, srcDataNode, dn, dataset, requestedChecksum,
+ cachingStrategy, allowLazyPersist, pinning);
+ }
+
+ /**
+ * Separated for testing.
+ * @return
+ */
+ @VisibleForTesting
+ DataOutputStream getBufferedOutputStream() {
+ return new DataOutputStream(
+ new BufferedOutputStream(getOutputStream(), smallBufferSize));
+ }
+
+
private long elapsed() {
return monotonicNow() - opStartTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 3c39ace,63f0377..9b9e5c4
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@@ -30,14 -30,19 +30,13 @@@ import java.util.ArrayList
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
- import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataStorage;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.Replica;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
@@@ -398,21 -560,37 +397,18 @@@ public interface FsDatasetSpi<V extend
) throws IOException;
/**
- * Get a {@link HdfsBlocksMetadata} corresponding to the list of blocks in
- * <code>blocks</code>.
- *
- * @param bpid pool to query
- * @param blockIds List of block ids for which to return metadata
- * @return metadata Metadata for the list of blocks
- * @throws IOException
++ /**
++ * submit a sync_file_range request to AsyncDiskService
+ * Enable 'trash' for the given dataset. When trash is enabled, files are
+ * moved to a separate trash directory instead of being deleted immediately.
+ * This can be useful for example during rolling upgrades.
*/
- public HdfsBlocksMetadata getHdfsBlocksMetadata(String bpid,
- long[] blockIds) throws IOException;
+ void enableTrash(String bpid);
/**
- * submit a sync_file_range request to AsyncDiskService
+ * Clear trash
*/
- void clearTrash(String bpid);
-
- /**
- * @return true when trash is enabled
- */
- boolean trashEnabled(String bpid);
-
- /**
- * Create a marker file indicating that a rolling upgrade is in progress.
- */
- void setRollingUpgradeMarker(String bpid) throws IOException;
-
- /**
- * Delete the rolling upgrade marker file if it exists.
- * @param bpid
- */
- void clearRollingUpgradeMarker(String bpid) throws IOException;
-
- /**
- * submit a sync_file_range request to AsyncDiskService.
- */
- void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block,
+ public void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block,
final FileDescriptor fd, final long offset, final long nbytes,
final int flags);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
index 2b7e3ff,0000000..7d1eb66
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java
@@@ -1,163 -1,0 +1,162 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.storagecontainer;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
- import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.namenode.CacheManager;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.Namesystem;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.AccessControlException;
+
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Namesystem implementation to be used by StorageContainerManager.
+ */
+public class StorageContainerNameService implements Namesystem {
+
+ private ReentrantReadWriteLock coarseLock = new ReentrantReadWriteLock();
+ private String blockPoolId;
+ private volatile boolean serviceRunning = true;
+
+ public void shutdown() {
+ serviceRunning = false;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return serviceRunning;
+ }
+
+ @Override
+ public void checkSuperuserPrivilege() throws AccessControlException {
+ // TBD
+ }
+
+ @Override
+ public String getBlockPoolId() {
+ return blockPoolId;
+ }
+
+ public void setBlockPoolId(String id) {
+ this.blockPoolId = id;
+ }
+
+ @Override
+ public boolean isInStandbyState() {
+ // HA mode is not supported
+ return false;
+ }
+
+ @Override
+ public boolean isGenStampInFuture(Block block) {
+ // HA mode is not supported
+ return false;
+ }
+
+ @Override
+ public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal) {
+ // TBD
+ }
+
+ @Override
+ public void checkOperation(NameNode.OperationCategory read)
+ throws StandbyException {
+ // HA mode is not supported
+ }
+
+ @Override
- public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
++ public boolean isInSnapshot(BlockInfo blockInfo) {
+ // Snapshots not supported
+ return false;
+ }
+
+ @Override
+ public CacheManager getCacheManager() {
+ // Cache Management is not supported
+ return null;
+ }
+
+ @Override
+ public void readLock() {
+ coarseLock.readLock().lock();
+ }
+
+ @Override
+ public void readUnlock() {
+ coarseLock.readLock().unlock();
+ }
+
+ @Override
+ public boolean hasReadLock() {
+ return coarseLock.getReadHoldCount() > 0 || hasWriteLock();
+ }
+
+ @Override
+ public void writeLock() {
+ coarseLock.writeLock().lock();
+ }
+
+ @Override
+ public void writeLockInterruptibly() throws InterruptedException {
+ coarseLock.writeLock().lockInterruptibly();
+ }
+
+ @Override
+ public void writeUnlock() {
+ coarseLock.writeLock().unlock();
+ }
+
+ @Override
+ public boolean hasWriteLock() {
+ return coarseLock.isWriteLockedByCurrentThread();
+ }
+
+ @Override
+ public void checkSafeMode() {
+ // TBD
+ }
+
+ @Override
+ public boolean isInSafeMode() {
+ return false;
+ }
+
+ @Override
+ public boolean isInStartupSafeMode() {
+ return false;
+ }
+
+ @Override
+ public boolean isPopulatingReplQueues() {
+ return false;
+ }
+
+ @Override
+ public void incrementSafeBlockCount(int replication) {
+ // Do nothing
+ }
+
+ @Override
+ public void decrementSafeBlockCount(BlockInfo b) {
+ // Do nothing
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index b05be39,5d1b31a..4788fcc
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@@ -42,12 -42,7 +42,10 @@@ import org.apache.hadoop.hdfs.protocol.
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
- import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
- import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.datanode.dataset.DatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.*;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
index 0000000,d8a7188..48f8cef
mode 000000,100644..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
@@@ -1,0 -1,178 +1,180 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.hdfs.server.datanode;
+
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.StorageType;
+ import org.apache.hadoop.hdfs.*;
+ import org.apache.hadoop.hdfs.net.*;
+ import org.apache.hadoop.hdfs.protocol.*;
+ import org.apache.hadoop.hdfs.protocol.datatransfer.*;
+ import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
+ import org.apache.hadoop.util.DataChecksum;
+ import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
++import org.apache.hadoop.hdfs.server.datanode.fsdataset.*;
+
+ import org.junit.Rule;
+ import org.junit.Test;
+ import org.junit.rules.Timeout;
+ import org.mockito.ArgumentCaptor;
+
+ import java.io.DataInputStream;
+ import java.io.DataOutputStream;
+ import java.io.IOException;
+ import java.util.Arrays;
+
+ import static org.hamcrest.MatcherAssert.assertThat;
+ import static org.hamcrest.core.Is.is;
+ import static org.mockito.Mockito.*;
+
+
+ /**
+ * Mock-based unit test to verify that the DataXceiver correctly handles the
+ * LazyPersist hint from clients.
+ */
+ public class TestDataXceiverLazyPersistHint {
+ @Rule
+ public Timeout timeout = new Timeout(300000);
+
+ private enum PeerLocality {
+ LOCAL,
+ REMOTE
+ }
+
+ private enum NonLocalLazyPersist {
+ ALLOWED,
+ NOT_ALLOWED
+ }
+
+ /**
+ * Ensure that the correct hint is passed to the block receiver when
+ * the client is local.
+ */
+ @Test
+ public void testWithLocalClient() throws IOException {
+ ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class);
+ DataXceiver xceiver = makeStubDataXceiver(
+ PeerLocality.LOCAL, NonLocalLazyPersist.NOT_ALLOWED, captor);
+
+ for (Boolean lazyPersistSetting : Arrays.asList(true, false)) {
+ issueWriteBlockCall(xceiver, lazyPersistSetting);
+ assertThat(captor.getValue(), is(lazyPersistSetting));
+ }
+ }
+
+ /**
+ * Ensure that hint is always false when the client is remote.
+ */
+ @Test
+ public void testWithRemoteClient() throws IOException {
+ ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class);
+ DataXceiver xceiver = makeStubDataXceiver(
+ PeerLocality.REMOTE, NonLocalLazyPersist.NOT_ALLOWED, captor);
+
+ for (Boolean lazyPersistSetting : Arrays.asList(true, false)) {
+ issueWriteBlockCall(xceiver, lazyPersistSetting);
+ assertThat(captor.getValue(), is(false));
+ }
+ }
+
+ /**
+ * Ensure that the correct hint is passed to the block receiver when
+ * the client is remote AND dfs.datanode.allow.non.local.lazy.persist
+ * is set to true.
+ */
+ @Test
+ public void testOverrideWithRemoteClient() throws IOException {
+ ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class);
+ DataXceiver xceiver = makeStubDataXceiver(
+ PeerLocality.REMOTE, NonLocalLazyPersist.ALLOWED, captor);
+
+ for (Boolean lazyPersistSetting : Arrays.asList(true, false)) {
+ issueWriteBlockCall(xceiver, lazyPersistSetting);
+ assertThat(captor.getValue(), is(lazyPersistSetting));
+ }
+ }
+
+ /**
+ * Issue a write block call with dummy parameters. The only parameter useful
+ * for this test is the value of lazyPersist.
+ */
+ private void issueWriteBlockCall(DataXceiver xceiver, boolean lazyPersist)
+ throws IOException {
+ xceiver.writeBlock(
+ new ExtendedBlock("Dummy-pool", 0L),
+ StorageType.RAM_DISK,
+ null,
+ "Dummy-Client",
+ new DatanodeInfo[0],
+ new StorageType[0],
+ mock(DatanodeInfo.class),
+ BlockConstructionStage.PIPELINE_SETUP_CREATE,
+ 0, 0, 0, 0,
+ DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 0),
+ CachingStrategy.newDefaultStrategy(),
+ lazyPersist,
+ false, null);
+ }
+
+ // Helper functions to setup the mock objects.
+
+ private static DataXceiver makeStubDataXceiver(
+ PeerLocality locality,
+ NonLocalLazyPersist nonLocalLazyPersist,
+ final ArgumentCaptor<Boolean> captor) throws IOException {
+ DataXceiver xceiverSpy = spy(DataXceiver.create(
+ getMockPeer(locality),
+ getMockDn(nonLocalLazyPersist),
+ mock(DataXceiverServer.class)));
+
+ doReturn(mock(BlockReceiver.class)).when(xceiverSpy).getBlockReceiver(
+ any(ExtendedBlock.class), any(StorageType.class),
+ any(DataInputStream.class), anyString(), anyString(),
+ any(BlockConstructionStage.class), anyLong(), anyLong(), anyLong(),
+ anyString(), any(DatanodeInfo.class), any(DataNode.class),
++ any(FsDatasetSpi.class),
+ any(DataChecksum.class), any(CachingStrategy.class),
+ captor.capture(), anyBoolean());
+ doReturn(mock(DataOutputStream.class)).when(xceiverSpy)
+ .getBufferedOutputStream();
+ return xceiverSpy;
+ }
+
+ private static Peer getMockPeer(PeerLocality locality) {
+ Peer peer = mock(Peer.class);
+ when(peer.isLocal()).thenReturn(locality == PeerLocality.LOCAL);
+ when(peer.getRemoteAddressString()).thenReturn("1.1.1.1:1000");
+ when(peer.getLocalAddressString()).thenReturn("2.2.2.2:2000");
+ return peer;
+ }
+
+ private static DataNode getMockDn(NonLocalLazyPersist nonLocalLazyPersist) {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(
+ DFS_DATANODE_NON_LOCAL_LAZY_PERSIST,
+ nonLocalLazyPersist == NonLocalLazyPersist.ALLOWED);
+ DNConf dnConf = new DNConf(conf);
+ DataNodeMetrics mockMetrics = mock(DataNodeMetrics.class);
+ DataNode mockDn = mock(DataNode.class);
+ when(mockDn.getDnConf()).thenReturn(dnConf);
+ when(mockDn.getConf()).thenReturn(conf);
+ when(mockDn.getMetrics()).thenReturn(mockMetrics);
+ return mockDn;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index 29bb04b,e3be5fd..c5a1bd9
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@@ -27,9 -27,7 +27,8 @@@ import org.apache.hadoop.hdfs.protocol.
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
- import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.datanode.*;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;