You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/09/18 22:12:33 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19f7f875
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19f7f875
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19f7f875
Branch: refs/heads/HDFS-7240
Commit: 19f7f87517d89cfe095f688163b867aaee24f04b
Parents: 44d0867 1ee2527
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Sep 18 15:11:06 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Sep 18 15:11:06 2017 -0700
----------------------------------------------------------------------
LICENSE.txt | 2 +-
NOTICE.txt | 65 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 27 +
.../hadoop-client-check-invariants/pom.xml | 74 +-
.../ensure-jars-have-correct-contents.sh | 82 +
.../hadoop-client-check-test-invariants/pom.xml | 76 +-
.../ensure-jars-have-correct-contents.sh | 70 +
.../apache/hadoop/example/ITUseMiniCluster.java | 4 +-
.../hadoop-client-minicluster/pom.xml | 105 +-
.../hadoop-client-runtime/pom.xml | 50 +
hadoop-client-modules/hadoop-client/pom.xml | 80 +-
.../JWTRedirectAuthenticationHandler.java | 4 +-
.../TestJWTRedirectAuthenticationHandler.java | 476 +++++
.../TestJWTRedirectAuthentictionHandler.java | 481 -----
.../src/main/conf/log4j.properties | 2 -
.../crypto/key/KeyProviderCryptoExtension.java | 10 +
.../crypto/key/kms/KMSClientProvider.java | 39 +-
.../key/kms/LoadBalancingKMSClientProvider.java | 78 +-
.../apache/hadoop/fs/AbstractFileSystem.java | 5 +
.../org/apache/hadoop/fs/BlockLocation.java | 28 +
.../fs/CommonConfigurationKeysPublic.java | 29 -
.../org/apache/hadoop/fs/FSDataInputStream.java | 4 +-
.../java/org/apache/hadoop/fs/FileContext.java | 33 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 32 +-
.../org/apache/hadoop/fs/LocatedFileStatus.java | 7 +
.../org/apache/hadoop/http/HttpServer2.java | 31 +-
.../file/tfile/BoundedRangeFileInputStream.java | 2 +-
.../hadoop/io/file/tfile/Compression.java | 6 +-
.../file/tfile/SimpleBufferedOutputStream.java | 2 +-
.../org/apache/hadoop/util/CpuTimeTracker.java | 4 +-
.../src/main/resources/core-default.xml | 302 +--
.../src/site/markdown/Compatibility.md | 645 +++++--
.../site/markdown/InterfaceClassification.md | 227 ++-
.../src/site/markdown/filesystem/filesystem.md | 6 +-
.../site/markdown/filesystem/introduction.md | 7 +-
.../src/site/markdown/filesystem/testing.md | 10 +-
.../conf/TestCommonConfigurationFields.java | 6 +-
.../apache/hadoop/conf/TestConfigRedactor.java | 1 -
.../kms/TestLoadBalancingKMSClientProvider.java | 315 +---
.../hadoop/fs/FileSystemContractBaseTest.java | 52 +-
.../fs/contract/AbstractContractOpenTest.java | 4 +-
.../fs/contract/AbstractContractSeekTest.java | 2 +-
.../hadoop/fs/permission/TestFsPermission.java | 57 +
.../hadoop/fs/sftp/TestSFTPFileSystem.java | 16 +-
.../org/apache/hadoop/test/LambdaTestUtils.java | 68 +-
.../apache/hadoop/test/TestLambdaTestUtils.java | 127 +-
.../apache/hadoop/util/TestCpuTimeTracker.java | 52 +
.../src/test/resources/core-site.xml | 6 -
.../src/test/resources/jets3t.properties | 16 -
.../main/java/org/apache/hadoop/fs/Hdfs.java | 15 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +
.../hadoop/hdfs/DFSStripedOutputStream.java | 17 +-
.../hadoop/hdfs/DistributedFileSystem.java | 14 +
.../hadoop/hdfs/protocol/BlocksStats.java | 90 -
.../hadoop/hdfs/protocol/ClientProtocol.java | 4 +-
.../hadoop/hdfs/protocol/ECBlockGroupStats.java | 83 +
.../hdfs/protocol/ECBlockGroupsStats.java | 83 -
.../hdfs/protocol/HdfsLocatedFileStatus.java | 12 +-
.../hdfs/protocol/ReplicatedBlockStats.java | 90 +
.../ClientNamenodeProtocolTranslatorPB.java | 32 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 58 +-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 53 -
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 62 +-
.../hadoop/hdfs/web/resources/GetOpParam.java | 12 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 16 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 42 -
.../hadoop/fs/http/server/FSOperations.java | 38 -
.../http/server/HttpFSParametersProvider.java | 3 +-
.../hadoop/fs/http/server/HttpFSServer.java | 21 +-
.../fs/http/client/BaseTestHttpFSWith.java | 89 +-
.../org/apache/hadoop/test/TestHdfsHelper.java | 3 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 20 +-
.../hadoop/hdfs/qjournal/client/QuorumCall.java | 65 +-
.../hdfs/qjournal/server/JournalNode.java | 47 +-
.../server/blockmanagement/BlockManager.java | 48 +-
.../blockmanagement/CorruptReplicasMap.java | 6 +-
.../blockmanagement/InvalidateBlocks.java | 52 +-
.../blockmanagement/LowRedundancyBlocks.java | 16 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 12 +-
.../erasurecode/ErasureCodingWorker.java | 18 +-
.../erasurecode/StripedBlockReconstructor.java | 11 +
.../datanode/metrics/DataNodeMetrics.java | 18 +
.../namenode/ErasureCodingPolicyManager.java | 64 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 5 +-
.../server/namenode/FSDirErasureCodingOp.java | 61 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 24 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 37 +
.../hdfs/server/namenode/FSEditLogLoader.java | 43 +
.../hdfs/server/namenode/FSEditLogOp.java | 325 ++++
.../hdfs/server/namenode/FSEditLogOpCodes.java | 5 +
.../server/namenode/FSImageFormatPBINode.java | 6 +-
.../server/namenode/FSImageFormatProtobuf.java | 48 +-
.../server/namenode/FSImageSerialization.java | 45 +
.../hdfs/server/namenode/FSNamesystem.java | 67 +-
.../hdfs/server/namenode/LeaseManager.java | 6 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 70 +-
.../namenode/metrics/ECBlockGroupsMBean.java | 4 +-
.../namenode/startupprogress/StepType.java | 7 +-
.../web/resources/NamenodeWebHdfsMethods.java | 16 -
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 33 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 33 -
.../hadoop-hdfs/src/main/proto/fsimage.proto | 4 +
.../src/main/resources/hdfs-default.xml | 25 +-
.../src/main/webapps/hdfs/dfshealth.js | 12 +-
.../hadoop-hdfs/src/site/markdown/Federation.md | 4 +-
.../src/site/markdown/HDFSErasureCoding.md | 12 +-
.../src/site/markdown/HdfsEditsViewer.md | 4 +
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 188 +-
.../apache/hadoop/cli/TestErasureCodingCLI.java | 11 +-
.../java/org/apache/hadoop/fs/TestUnbuffer.java | 21 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 80 +-
.../hdfs/ReadStripedFileWithDecodingHelper.java | 4 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 2 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 3 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 5 +-
.../TestDFSStripedOutputStreamWithFailure.java | 37 +-
.../hdfs/TestDecommissionWithStriped.java | 4 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 1 -
.../TestDistributedFileSystemWithECFile.java | 186 ++
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 19 +-
.../TestErasureCodeBenchmarkThroughput.java | 4 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 11 +-
.../TestErasureCodingPolicyWithSnapshot.java | 3 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 5 +-
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 4 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 3 +-
.../TestReadStripedFileWithMissingBlocks.java | 3 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 6 +-
.../hdfs/TestSafeModeWithStripedFile.java | 4 +-
.../hadoop/hdfs/TestSetrepIncreasing.java | 4 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 4 +-
.../datatransfer/sasl/TestSaslDataTransfer.java | 4 +-
.../hdfs/qjournal/client/TestQuorumCall.java | 31 +-
.../hdfs/qjournal/server/TestJournalNode.java | 21 +
.../hdfs/server/balancer/TestBalancer.java | 4 +-
.../blockmanagement/TestBlockManager.java | 7 +-
.../blockmanagement/TestBlockStatsMXBean.java | 7 +-
.../TestBlockTokenWithDFSStriped.java | 6 +-
.../TestComputeInvalidateWork.java | 8 +-
.../blockmanagement/TestCorruptReplicaInfo.java | 8 +-
.../TestLowRedundancyBlockQueues.java | 10 +-
...constructStripedBlocksWithRackAwareness.java | 13 +-
.../TestSequentialBlockGroupId.java | 8 +-
.../TestDataNodeErasureCodingMetrics.java | 11 +-
.../command/TestDiskBalancerCommand.java | 5 +-
.../hadoop/hdfs/server/mover/TestMover.java | 4 +-
.../TestAddOverReplicatedStripedBlocks.java | 3 +-
.../namenode/TestAddStripedBlockInFBR.java | 5 +-
.../server/namenode/TestAddStripedBlocks.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 4 +-
.../hdfs/server/namenode/TestDeleteRace.java | 84 +
.../server/namenode/TestEnabledECPolicies.java | 23 +-
.../server/namenode/TestFSEditLogLoader.java | 89 +-
.../hdfs/server/namenode/TestFSImage.java | 159 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 17 +-
.../server/namenode/TestNameNodeMXBean.java | 4 +-
.../server/namenode/TestNamenodeRetryCache.java | 4 +-
.../namenode/TestQuotaWithStripedBlocks.java | 3 +-
.../namenode/TestReconstructStripedBlocks.java | 17 +-
.../hdfs/server/namenode/TestReencryption.java | 103 +-
.../namenode/TestReencryptionWithKMS.java | 5 +
.../hdfs/server/namenode/TestStartup.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 8 +-
.../namenode/ha/TestRetryCacheWithHA.java | 4 +-
.../namenode/metrics/TestNameNodeMetrics.java | 8 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 7 +-
.../TestOfflineEditsViewer.java | 2 +-
.../TestOfflineImageViewer.java | 3 +-
...TestOfflineImageViewerWithStripedBlocks.java | 4 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 281 +--
.../hadoop-hdfs/src/test/resources/editsStored | Bin 5850 -> 6293 bytes
.../src/test/resources/editsStored.xml | 90 +-
.../hadoop-mapreduce-client-core/pom.xml | 4 +
.../java/org/apache/hadoop/mapred/Task.java | 2 +-
.../hadoop/mapred/lib/MultipleOutputs.java | 4 +-
.../hadoop/mapreduce/JobResourceUploader.java | 17 +
.../apache/hadoop/mapreduce/MRJobConfig.java | 5 +
.../lib/output/FileOutputCommitter.java | 25 +-
.../mapreduce/lib/output/FileOutputFormat.java | 57 +-
.../lib/output/PathOutputCommitter.java | 91 +
.../hadoop/mapreduce/task/JobContextImpl.java | 10 +-
.../mapreduce/task/TaskAttemptContextImpl.java | 13 +-
.../hadoop/mapreduce/task/reduce/Fetcher.java | 18 +-
.../task/reduce/ShuffleSchedulerImpl.java | 3 +
.../src/main/resources/mapred-default.xml | 9 +
.../mapreduce/TestJobResourceUploader.java | 46 +
.../lib/output/TestPathOutputCommitter.java | 377 ++++
.../mapreduce/task/reduce/TestFetcher.java | 9 +-
.../task/reduce/TestShuffleScheduler.java | 80 +
.../hadoop/mapred/ResourceMgrDelegate.java | 20 +
.../hadoop/mapred/TestClientRedirect.java | 26 +
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 7 +-
.../hadoop/mapred/nativetask/NativeRuntime.java | 2 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 7 +
.../apache/hadoop/examples/QuasiMonteCarlo.java | 2 +-
hadoop-project/pom.xml | 23 +-
hadoop-project/src/site/markdown/index.md.vm | 10 +
hadoop-project/src/site/site.xml | 1 +
.../hadoop-aws/dev-support/findbugs-exclude.xml | 4 -
hadoop-tools/hadoop-aws/pom.xml | 7 -
.../org/apache/hadoop/fs/s3a/Constants.java | 12 +
.../fs/s3a/InconsistentAmazonS3Client.java | 143 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 22 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 162 +-
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 143 --
.../org/apache/hadoop/fs/s3a/S3ListRequest.java | 69 +
.../org/apache/hadoop/fs/s3a/S3ListResult.java | 97 +
.../apache/hadoop/fs/s3native/FileMetadata.java | 59 -
.../s3native/Jets3tNativeFileSystemStore.java | 481 -----
.../fs/s3native/NativeFileSystemStore.java | 67 -
.../hadoop/fs/s3native/NativeS3FileSystem.java | 799 +-------
.../hadoop/fs/s3native/PartialListing.java | 64 -
.../hadoop/fs/s3native/S3Credentials.java | 100 -
.../apache/hadoop/fs/s3native/S3Exception.java | 39 -
.../s3native/S3NativeFileSystemConfigKeys.java | 66 -
.../org/apache/hadoop/fs/s3native/package.html | 5 +-
.../markdown/tools/hadoop-aws/encryption.md | 427 +++++
.../src/site/markdown/tools/hadoop-aws/index.md | 1746 ++++++------------
.../site/markdown/tools/hadoop-aws/s3guard.md | 19 +-
.../src/site/markdown/tools/hadoop-aws/s3n.md | 52 +
.../site/markdown/tools/hadoop-aws/testing.md | 91 +-
.../tools/hadoop-aws/troubleshooting_s3a.md | 701 ++++++-
.../fs/contract/s3a/ITestS3AContractDistCp.java | 1 -
.../fs/contract/s3n/ITestS3NContractCreate.java | 41 -
.../fs/contract/s3n/ITestS3NContractDelete.java | 34 -
.../fs/contract/s3n/ITestS3NContractMkdir.java | 34 -
.../fs/contract/s3n/ITestS3NContractOpen.java | 34 -
.../fs/contract/s3n/ITestS3NContractRename.java | 35 -
.../contract/s3n/ITestS3NContractRootDir.java | 35 -
.../fs/contract/s3n/ITestS3NContractSeek.java | 34 -
.../fs/contract/s3n/NativeS3Contract.java | 50 -
.../hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 1 -
.../ITestS3AContractGetFileStatusV1List.java | 59 +
...ITestS3AEncryptionSSECBlockOutputStream.java | 1 -
...onSSEKMSUserDefinedKeyBlockOutputStream.java | 4 +-
...TestS3AEncryptionSSES3BlockOutputStream.java | 1 -
.../fs/s3a/ITestS3GuardListConsistency.java | 22 +-
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 41 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 1 -
.../scale/ITestS3AHugeFilesClassicOutput.java | 41 -
...ITestInMemoryNativeS3FileSystemContract.java | 33 -
.../ITestJets3tNativeFileSystemStore.java | 133 --
.../ITestJets3tNativeS3FileSystemContract.java | 33 -
.../s3native/InMemoryNativeFileSystemStore.java | 213 ---
.../NativeS3FileSystemContractBaseTest.java | 266 ---
.../fs/s3native/S3NInMemoryFileSystem.java | 32 -
.../hadoop/fs/s3native/TestS3Credentials.java | 129 --
.../fs/s3native/TestS3NInMemoryFileSystem.java | 69 -
.../src/test/resources/contract/s3n.xml | 110 --
.../src/test/resources/log4j.properties | 3 +
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 -
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +-
.../src/site/markdown/index.md | 21 +-
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 4 -
hadoop-tools/hadoop-azure/pom.xml | 251 +++
.../fs/azure/AzureNativeFileSystemStore.java | 2 +-
.../hadoop-azure/src/site/markdown/index.md | 94 +-
.../src/site/markdown/testing_azure.md | 576 ++++++
.../hadoop/fs/azure/AbstractWasbTestBase.java | 136 +-
.../fs/azure/AbstractWasbTestWithTimeout.java | 73 +
.../fs/azure/AzureBlobStorageTestAccount.java | 42 +-
.../azure/ITestAzureConcurrentOutOfBandIo.java | 179 ++
...zureConcurrentOutOfBandIoWithSecureMode.java | 33 +
.../ITestAzureFileSystemErrorConditions.java | 243 +++
.../fs/azure/ITestBlobDataValidation.java | 244 +++
.../fs/azure/ITestBlobTypeSpeedDifference.java | 163 ++
.../fs/azure/ITestBlockBlobInputStream.java | 874 +++++++++
.../hadoop/fs/azure/ITestContainerChecks.java | 194 ++
...estFileSystemOperationExceptionHandling.java | 283 +++
...TestFileSystemOperationExceptionMessage.java | 79 +
...perationsExceptionHandlingMultiThreaded.java | 366 ++++
.../ITestFileSystemOperationsWithThreads.java | 821 ++++++++
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 40 +
.../ITestNativeAzureFSAuthorizationCaching.java | 53 +
.../azure/ITestNativeAzureFSPageBlobLive.java | 43 +
.../azure/ITestNativeAzureFileSystemAppend.java | 350 ++++
...ativeAzureFileSystemAtomicRenameDirList.java | 55 +
...veAzureFileSystemAuthorizationWithOwner.java | 122 ++
...ITestNativeAzureFileSystemClientLogging.java | 136 ++
...estNativeAzureFileSystemConcurrencyLive.java | 185 ++
...stNativeAzureFileSystemContractEmulator.java | 65 +
.../ITestNativeAzureFileSystemContractLive.java | 108 ++
...tiveAzureFileSystemContractPageBlobLive.java | 114 ++
.../azure/ITestNativeAzureFileSystemLive.java | 236 +++
.../ITestOutOfBandAzureBlobOperationsLive.java | 185 ++
.../ITestReadAndSeekPageBlobAfterWrite.java | 341 ++++
.../fs/azure/ITestWasbRemoteCallHelper.java | 568 ++++++
.../fs/azure/ITestWasbUriAndConfiguration.java | 610 ++++++
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 103 +-
.../fs/azure/NativeAzureFileSystemBaseTest.java | 115 +-
.../hadoop/fs/azure/RunningLiveWasbTests.txt | 22 -
.../azure/TestAzureConcurrentOutOfBandIo.java | 195 --
...zureConcurrentOutOfBandIoWithSecureMode.java | 50 -
.../TestAzureFileSystemErrorConditions.java | 244 ---
.../hadoop/fs/azure/TestBlobDataValidation.java | 237 ---
.../hadoop/fs/azure/TestBlobMetadata.java | 7 +-
.../fs/azure/TestBlobOperationDescriptor.java | 3 -
.../fs/azure/TestBlobTypeSpeedDifference.java | 160 --
.../fs/azure/TestBlockBlobInputStream.java | 875 ---------
.../fs/azure/TestClientThrottlingAnalyzer.java | 5 +-
.../hadoop/fs/azure/TestContainerChecks.java | 185 --
...estFileSystemOperationExceptionHandling.java | 269 ---
...TestFileSystemOperationExceptionMessage.java | 79 -
...perationsExceptionHandlingMultiThreaded.java | 330 ----
.../TestFileSystemOperationsWithThreads.java | 821 --------
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 44 -
.../TestNativeAzureFSAuthorizationCaching.java | 60 -
.../fs/azure/TestNativeAzureFSPageBlobLive.java | 43 -
.../azure/TestNativeAzureFileSystemAppend.java | 362 ----
...ativeAzureFileSystemAtomicRenameDirList.java | 50 -
.../TestNativeAzureFileSystemAuthorization.java | 53 +-
...veAzureFileSystemAuthorizationWithOwner.java | 122 --
...TestNativeAzureFileSystemBlockLocations.java | 8 +-
.../TestNativeAzureFileSystemClientLogging.java | 140 --
.../TestNativeAzureFileSystemConcurrency.java | 29 +-
...estNativeAzureFileSystemConcurrencyLive.java | 184 --
...stNativeAzureFileSystemContractEmulator.java | 48 -
.../TestNativeAzureFileSystemContractLive.java | 80 -
...TestNativeAzureFileSystemContractMocked.java | 3 +
...tiveAzureFileSystemContractPageBlobLive.java | 93 -
.../TestNativeAzureFileSystemFileNameCheck.java | 28 +-
.../fs/azure/TestNativeAzureFileSystemLive.java | 242 ---
.../azure/TestNativeAzureFileSystemMocked.java | 4 +
.../TestNativeAzureFileSystemUploadLogic.java | 78 +-
.../azure/TestOutOfBandAzureBlobOperations.java | 8 +-
.../TestOutOfBandAzureBlobOperationsLive.java | 203 --
.../TestReadAndSeekPageBlobAfterWrite.java | 355 ----
.../azure/TestShellDecryptionKeyProvider.java | 15 +-
.../apache/hadoop/fs/azure/TestWasbFsck.java | 9 +-
.../fs/azure/TestWasbRemoteCallHelper.java | 569 ------
.../fs/azure/TestWasbUriAndConfiguration.java | 617 -------
.../ITestAzureNativeContractAppend.java | 41 +
.../ITestAzureNativeContractCreate.java | 34 +
.../ITestAzureNativeContractDelete.java | 33 +
.../ITestAzureNativeContractDistCp.java | 47 +
.../ITestAzureNativeContractGetFileStatus.java | 35 +
.../contract/ITestAzureNativeContractMkdir.java | 33 +
.../contract/ITestAzureNativeContractOpen.java | 34 +
.../ITestAzureNativeContractRename.java | 34 +
.../contract/ITestAzureNativeContractSeek.java | 34 +
.../contract/NativeAzureFileSystemContract.java | 19 +-
.../contract/TestAzureNativeContractAppend.java | 37 -
.../contract/TestAzureNativeContractCreate.java | 30 -
.../contract/TestAzureNativeContractDelete.java | 30 -
.../contract/TestAzureNativeContractDistCp.java | 33 -
.../TestAzureNativeContractGetFileStatus.java | 30 -
.../contract/TestAzureNativeContractMkdir.java | 30 -
.../contract/TestAzureNativeContractOpen.java | 30 -
.../contract/TestAzureNativeContractRename.java | 30 -
.../contract/TestAzureNativeContractSeek.java | 30 -
.../integration/AbstractAzureScaleTest.java | 66 +
.../azure/integration/AzureTestConstants.java | 180 ++
.../fs/azure/integration/AzureTestUtils.java | 479 +++++
.../integration/CleanupTestContainers.java | 87 +
.../azure/integration/ITestAzureHugeFiles.java | 456 +++++
.../hadoop/fs/azure/integration/Sizes.java | 43 +
.../ITestAzureFileSystemInstrumentation.java | 586 ++++++
.../TestAzureFileSystemInstrumentation.java | 579 ------
.../hadoop/tools/util/TestDistCpUtils.java | 6 +-
.../dev-support/findbugs-exclude.xml | 22 +
.../ams/ApplicationMasterServiceProcessor.java | 8 +-
.../yarn/api/ApplicationClientProtocol.java | 55 +-
.../GetAllResourceProfilesRequest.java | 35 +
.../GetAllResourceProfilesResponse.java | 60 +
.../GetAllResourceTypeInfoRequest.java | 35 +
.../GetAllResourceTypeInfoResponse.java | 60 +
.../GetResourceProfileRequest.java | 59 +
.../GetResourceProfileResponse.java | 68 +
.../RegisterApplicationMasterResponse.java | 8 +
.../yarn/api/protocolrecords/ResourceTypes.java | 27 +
.../UpdateApplicationTimeoutsResponse.java | 19 +
.../records/ApplicationResourceUsageReport.java | 57 +-
.../hadoop/yarn/api/records/CollectorInfo.java | 8 +-
.../hadoop/yarn/api/records/ContainerState.java | 7 +-
.../yarn/api/records/ProfileCapability.java | 174 ++
.../hadoop/yarn/api/records/Resource.java | 402 +++-
.../yarn/api/records/ResourceInformation.java | 291 +++
.../yarn/api/records/ResourceRequest.java | 44 +-
.../yarn/api/records/ResourceTypeInfo.java | 197 ++
.../api/records/impl/LightWeightResource.java | 163 ++
.../yarn/api/records/impl/package-info.java | 26 +
.../hadoop/yarn/conf/YarnConfiguration.java | 92 +-
.../exceptions/ResourceNotFoundException.java | 45 +
.../ResourceProfilesNotEnabledException.java | 43 +
.../YARNFeatureNotEnabledException.java | 45 +
.../hadoop/yarn/util/UnitsConversionUtil.java | 221 +++
.../yarn/util/resource/ResourceUtils.java | 576 ++++++
.../hadoop/yarn/util/resource/package-info.java | 22 +
.../main/proto/applicationclient_protocol.proto | 3 +
.../src/main/proto/yarn_protos.proto | 41 +
.../src/main/proto/yarn_service_protos.proto | 23 +
.../yarn/conf/TestResourceInformation.java | 73 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../yarn/util/TestUnitsConversionUtil.java | 139 ++
.../distributedshell/ApplicationMaster.java | 61 +-
.../applications/distributedshell/Client.java | 176 +-
.../distributedshell/TestDistributedShell.java | 29 +
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 1 +
.../hadoop/yarn/client/api/AMRMClient.java | 132 +-
.../hadoop/yarn/client/api/YarnClient.java | 45 +
.../yarn/client/api/async/AMRMClientAsync.java | 8 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 153 +-
.../client/api/impl/RemoteRequestsTable.java | 109 +-
.../yarn/client/api/impl/YarnClientImpl.java | 29 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 51 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 200 +-
.../api/async/impl/TestAMRMClientAsync.java | 2 +-
.../yarn/client/api/impl/TestAMRMClient.java | 145 +-
.../impl/TestAMRMClientContainerRequest.java | 8 +-
.../api/impl/TestDistributedScheduling.java | 12 +-
.../yarn/client/api/impl/TestNMClient.java | 5 +-
.../TestOpportunisticContainerAllocation.java | 31 +-
.../yarn/client/api/impl/TestYarnClient.java | 10 +-
.../hadoop/yarn/client/cli/TestLogsCLI.java | 205 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 38 +-
.../src/test/resources/resource-profiles.json | 18 +
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 8 +
.../FileSystemBasedConfigurationProvider.java | 3 +-
.../hadoop/yarn/LocalConfigurationProvider.java | 3 +-
.../ApplicationClientProtocolPBClientImpl.java | 54 +
.../ApplicationClientProtocolPBServiceImpl.java | 63 +
.../pb/GetAllResourceProfilesRequestPBImpl.java | 55 +
.../GetAllResourceProfilesResponsePBImpl.java | 137 ++
.../pb/GetAllResourceTypeInfoRequestPBImpl.java | 71 +
.../GetAllResourceTypeInfoResponsePBImpl.java | 184 ++
.../pb/GetResourceProfileRequestPBImpl.java | 96 +
.../pb/GetResourceProfileResponsePBImpl.java | 107 ++
...RegisterApplicationMasterResponsePBImpl.java | 58 +
...UpdateApplicationTimeoutsResponsePBImpl.java | 108 ++
.../ApplicationResourceUsageReportPBImpl.java | 150 +-
.../impl/pb/ProfileCapabilityPBImpl.java | 129 ++
.../yarn/api/records/impl/pb/ProtoUtils.java | 46 +
.../api/records/impl/pb/ResourcePBImpl.java | 152 +-
.../records/impl/pb/ResourceRequestPBImpl.java | 41 +-
.../records/impl/pb/ResourceTypeInfoPBImpl.java | 152 ++
.../logaggregation/ContainerLogFileInfo.java | 93 +
.../yarn/logaggregation/ContainerLogMeta.java | 8 +-
.../logaggregation/LogAggregationUtils.java | 27 +
.../yarn/logaggregation/LogCLIHelpers.java | 20 +-
.../yarn/logaggregation/LogToolUtils.java | 26 +
.../logaggregation/PerContainerLogFileInfo.java | 93 -
.../LogAggregationFileController.java | 45 +-
.../ifile/IndexedFileAggregatedLogsBlock.java | 275 +++
.../LogAggregationIndexedFileController.java | 1057 +++++++++++
.../filecontroller/ifile/package-info.java | 21 +
.../tfile/LogAggregationTFileController.java | 10 +-
.../state/MultiStateTransitionListener.java | 61 +
.../hadoop/yarn/state/StateMachineFactory.java | 40 +
.../yarn/state/StateTransitionListener.java | 50 +
.../apache/hadoop/yarn/util/StringHelper.java | 36 +
.../resource/DefaultResourceCalculator.java | 3 +-
.../resource/DominantResourceCalculator.java | 577 ++++--
.../yarn/util/resource/ResourceCalculator.java | 3 +-
.../hadoop/yarn/util/resource/Resources.java | 285 ++-
.../src/main/resources/yarn-default.xml | 59 +
.../hadoop/yarn/api/BasePBImplRecordsTest.java | 12 +
.../hadoop/yarn/api/TestPBImplRecords.java | 64 +
.../hadoop/yarn/api/TestResourcePBImpl.java | 61 +
.../TestLogAggregationIndexFileController.java | 314 ++++
.../util/resource/TestResourceCalculator.java | 225 ++-
.../yarn/util/resource/TestResourceUtils.java | 306 +++
.../yarn/util/resource/TestResources.java | 220 ++-
.../resource-types/node-resources-1.xml | 29 +
.../resource-types/node-resources-2.xml | 39 +
.../resource-types/resource-types-1.xml | 18 +
.../resource-types/resource-types-2.xml | 29 +
.../resource-types/resource-types-3.xml | 24 +
.../resource-types/resource-types-4.xml | 34 +
.../resource-types/resource-types-error-1.xml | 29 +
.../resource-types/resource-types-error-2.xml | 33 +
.../resource-types/resource-types-error-3.xml | 29 +
.../resource-types/resource-types-error-4.xml | 24 +
...pplicationHistoryManagerOnTimelineStore.java | 18 +-
.../webapp/TestAHSWebServices.java | 8 +-
.../api/protocolrecords/NMContainerStatus.java | 17 +-
.../protocolrecords/NodeHeartbeatResponse.java | 5 +
.../impl/pb/NMContainerStatusPBImpl.java | 30 +
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 65 +
.../hadoop/yarn/server/utils/BuilderUtils.java | 16 +-
.../yarn/server/webapp/AppAttemptBlock.java | 26 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 34 +-
.../hadoop/yarn/server/webapp/AppsBlock.java | 11 +-
.../yarn/server/webapp/ContainerBlock.java | 14 +-
.../hadoop/yarn/server/webapp/WebServices.java | 64 +-
.../server/webapp/dao/ContainerLogsInfo.java | 10 +-
.../yarn_server_common_service_protos.proto | 1 +
.../yarn/server/MockResourceManagerFacade.java | 24 +
.../server/nodemanager/ContainerExecutor.java | 26 +-
.../ContainerStateTransitionListener.java | 48 +
.../hadoop/yarn/server/nodemanager/Context.java | 2 +
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../yarn/server/nodemanager/NodeManager.java | 48 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 13 +-
.../containermanager/container/Container.java | 9 +
.../container/ContainerEventType.java | 6 +-
.../container/ContainerImpl.java | 236 ++-
.../container/ContainerPauseEvent.java | 40 +
.../container/ContainerResumeEvent.java | 39 +
.../container/ContainerState.java | 3 +-
.../container/ResourceMappings.java | 124 ++
.../launcher/ContainerLaunch.java | 111 +-
.../launcher/ContainersLauncher.java | 42 +
.../launcher/ContainersLauncherEventType.java | 4 +
.../launcher/RecoverPausedContainerLaunch.java | 124 ++
.../launcher/RecoveredContainerLaunch.java | 3 +-
.../linux/resources/CGroupsHandlerImpl.java | 11 -
.../DelegatingLinuxContainerRuntime.java | 79 +-
.../runtime/LinuxContainerRuntimeConstants.java | 9 +
.../monitor/ContainersMonitorImpl.java | 4 +-
.../runtime/ContainerRuntime.java | 6 +-
.../scheduler/ContainerScheduler.java | 129 +-
.../scheduler/ContainerSchedulerEventType.java | 1 +
.../recovery/NMLeveldbStateStoreService.java | 84 +-
.../recovery/NMNullStateStoreService.java | 16 +
.../recovery/NMStateStoreService.java | 49 +-
.../util/NodeManagerHardwareUtils.java | 84 +-
.../webapp/dao/NMContainerLogsInfo.java | 8 +-
.../yarn/server/nodemanager/TestEventFlow.java | 3 +-
.../server/nodemanager/TestNodeManager.java | 68 +
.../amrmproxy/BaseAMRMProxyTest.java | 8 +
.../containermanager/TestContainerManager.java | 2 +-
.../TestContainerManagerRecovery.java | 160 +-
.../container/TestContainer.java | 107 ++
.../launcher/TestContainerLaunch.java | 32 +-
.../TestDelegatingLinuxContainerRuntime.java | 137 ++
...ContainerSchedulerBehaviorCompatibility.java | 100 +
.../TestContainerSchedulerQueuing.java | 103 ++
.../recovery/NMMemoryStateStoreService.java | 27 +
.../TestNMLeveldbStateStoreService.java | 138 +-
.../util/TestNodeManagerHardwareUtils.java | 2 +-
.../nodemanager/webapp/MockContainer.java | 11 +
.../nodemanager/webapp/TestNMWebServices.java | 8 +-
.../conf/capacity-scheduler.xml | 35 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 5 +
.../resourcemanager/AMSProcessingChain.java | 2 +-
.../server/resourcemanager/ClientRMService.java | 62 +-
.../resourcemanager/DefaultAMSProcessor.java | 12 +-
...pportunisticContainerAllocatorAMService.java | 3 +-
.../server/resourcemanager/RMAppManager.java | 38 +-
.../yarn/server/resourcemanager/RMContext.java | 5 +
.../server/resourcemanager/RMContextImpl.java | 12 +
.../server/resourcemanager/RMServerUtils.java | 49 +-
.../server/resourcemanager/ResourceManager.java | 9 +
.../resourcemanager/ResourceTrackerService.java | 9 +-
.../CapacitySchedulerPreemptionUtils.java | 4 +-
...QueuePriorityContainerCandidateSelector.java | 5 +-
.../ReservedContainerCandidatesSelector.java | 34 +-
.../resourcemanager/recovery/RMStateStore.java | 7 +-
.../records/ApplicationAttemptStateData.java | 89 +-
.../pb/ApplicationAttemptStateDataPBImpl.java | 50 +
.../reservation/CapacityOverTimePolicy.java | 32 +-
.../reservation/NoOverCommitPolicy.java | 8 +-
.../resource/ResourceProfilesManager.java | 86 +
.../resource/ResourceProfilesManagerImpl.java | 257 +++
.../resource/ResourceWeights.java | 72 -
.../server/resourcemanager/rmapp/RMAppImpl.java | 45 +-
.../resourcemanager/rmapp/RMAppMetrics.java | 41 +-
.../attempt/AggregateAppResourceUsage.java | 34 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 32 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 105 +-
.../rmcontainer/RMContainerImpl.java | 20 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 23 +-
.../rmnode/RMNodeUpdateContainerEvent.java | 9 +-
.../scheduler/AbstractYarnScheduler.java | 160 +-
.../scheduler/AppSchedulingInfo.java | 4 +
.../scheduler/ClusterNodeTracker.java | 13 +-
.../scheduler/SchedulerApplicationAttempt.java | 57 +-
.../scheduler/SchedulerNode.java | 8 -
.../scheduler/SchedulerUtils.java | 10 +
.../scheduler/YarnScheduler.java | 20 +
.../scheduler/capacity/AbstractCSQueue.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 47 +-
.../CapacitySchedulerConfiguration.java | 26 +
.../scheduler/capacity/LeafQueue.java | 34 +-
.../scheduler/capacity/ParentQueue.java | 4 +
.../scheduler/capacity/UsersManager.java | 4 +-
.../allocator/RegularContainerAllocator.java | 8 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 21 +-
.../common/fica/FiCaSchedulerNode.java | 13 +
.../scheduler/fair/AllocationConfiguration.java | 17 +-
.../fair/AllocationFileLoaderService.java | 7 +-
.../scheduler/fair/FSAppAttempt.java | 28 +-
.../scheduler/fair/FSLeafQueue.java | 3 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 9 +-
.../scheduler/fair/FSSchedulerNode.java | 10 +
.../scheduler/fair/FairScheduler.java | 75 +-
.../scheduler/fair/Schedulable.java | 12 +-
.../fair/policies/ComputeFairShares.java | 81 +-
.../DominantResourceFairnessPolicy.java | 240 ++-
.../fair/policies/FairSharePolicy.java | 15 +-
.../scheduler/fifo/FifoScheduler.java | 13 +-
.../RMDelegationTokenSecretManager.java | 16 +-
.../resourcemanager/webapp/ContainerPage.java | 3 +-
.../webapp/MetricsOverviewTable.java | 11 +-
.../webapp/RMAppAttemptBlock.java | 20 +-
.../resourcemanager/webapp/RMAppBlock.java | 49 +-
.../resourcemanager/webapp/RMAppsBlock.java | 17 +-
.../webapp/RMContainerBlock.java | 47 +
.../server/resourcemanager/webapp/RMWebApp.java | 2 -
.../resourcemanager/webapp/RMWebServices.java | 52 +-
.../resourcemanager/webapp/dao/AppInfo.java | 28 +
.../webapp/dao/ResourceInfo.java | 36 +-
.../webapp/dao/SchedulerInfo.java | 3 +-
.../yarn_server_resourcemanager_recovery.proto | 2 +
.../yarn/server/resourcemanager/MockRM.java | 11 +
.../ParameterizedSchedulerTestBase.java | 63 +-
.../server/resourcemanager/RMHATestBase.java | 2 +
.../server/resourcemanager/TestAppManager.java | 9 +-
.../TestApplicationMasterService.java | 46 +-
.../resourcemanager/TestClientRMService.java | 120 ++
.../resourcemanager/TestClientRMTokens.java | 3 +
.../TestContainerResourceUsage.java | 7 +-
.../TestNodeBlacklistingOnAMFailures.java | 12 +
.../yarn/server/resourcemanager/TestRM.java | 6 +
.../resourcemanager/TestRMAdminService.java | 5 +
.../server/resourcemanager/TestRMRestart.java | 9 +-
.../TestResourceTrackerService.java | 108 ++
.../TestTokenClientRMService.java | 3 +
.../TestWorkPreservingRMRestart.java | 4 +
.../applicationsmanager/MockAsm.java | 9 +-
.../metrics/TestSystemMetricsPublisher.java | 15 +-
.../TestSystemMetricsPublisherForV2.java | 22 +-
.../recovery/RMStateStoreTestBase.java | 8 +-
.../recovery/TestZKRMStateStore.java | 23 +-
.../reservation/BaseSharingPolicyTest.java | 189 ++
.../reservation/ReservationSystemTestUtil.java | 28 +-
.../reservation/TestCapacityOverTimePolicy.java | 339 +---
.../TestFairSchedulerPlanFollower.java | 6 +-
.../reservation/TestNoOverCommitPolicy.java | 185 +-
.../reservation/TestReservationSystem.java | 4 +
.../resource/TestResourceProfiles.java | 152 ++
.../resource/TestResourceWeights.java | 55 -
.../resourcemanager/resource/TestResources.java | 43 -
.../resourcetracker/TestNMReconnect.java | 5 +
.../rmapp/TestApplicationLifetimeMonitor.java | 56 +-
.../scheduler/TestAbstractYarnScheduler.java | 42 +-
.../TestSchedulingWithAllocationRequestId.java | 155 +-
.../capacity/TestCapacityScheduler.java | 227 +--
.../capacity/TestCapacitySchedulerPerf.java | 265 +++
.../capacity/TestContainerAllocation.java | 50 +
.../scheduler/capacity/TestLeafQueue.java | 8 +-
.../scheduler/capacity/TestReservations.java | 20 +-
.../scheduler/fair/FakeSchedulable.java | 29 +-
.../scheduler/fair/TestComputeFairShares.java | 58 +-
.../fair/TestContinuousScheduling.java | 11 +-
.../scheduler/fair/TestFairScheduler.java | 18 +-
.../scheduler/fair/TestSchedulingPolicy.java | 13 +-
.../TestDominantResourceFairnessPolicy.java | 304 ++-
.../policy/TestFairOrderingPolicy.java | 6 +-
.../security/TestClientToAMTokens.java | 4 +
.../resourcemanager/webapp/TestAppPage.java | 13 +-
.../resourcemanager/webapp/TestRMWebApp.java | 3 -
.../webapp/TestRMWebAppFairScheduler.java | 11 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../webapp/TestRedirectionErrorPage.java | 3 -
.../resources/profiles/illegal-profiles-1.json | 10 +
.../resources/profiles/illegal-profiles-2.json | 10 +
.../resources/profiles/illegal-profiles-3.json | 10 +
.../resources/profiles/sample-profiles-1.json | 14 +
.../resources/profiles/sample-profiles-2.json | 26 +
.../DefaultClientRequestInterceptor.java | 24 +
.../clientrm/FederationClientInterceptor.java | 23 +
.../router/clientrm/RouterClientRMService.java | 27 +
.../PassThroughClientRequestInterceptor.java | 24 +
.../hadoop/yarn/server/MiniYARNCluster.java | 7 +
.../storage/TimelineSchemaCreator.java | 2 +-
.../src/site/markdown/CapacityScheduler.md | 10 +
.../src/site/markdown/DockerContainers.md | 145 +-
.../src/site/markdown/ResourceProfiles.md | 116 ++
.../src/site/markdown/TimelineServiceV2.md | 4 +-
.../app/components/base-chart-component.js | 2 +-
.../main/webapp/app/components/nodes-heatmap.js | 30 +-
.../src/main/webapp/app/styles/app.css | 19 +-
pom.xml | 2 +-
677 files changed, 33887 insertions(+), 18480 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/LICENSE.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --cc hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 8c8507c,a11e7c3..538df97
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@@ -1428,20 -1344,16 +1344,16 @@@
<description>The implementation class of the S3A AbstractFileSystem.</description>
</property>
- <!-- Ozone file system properties -->
- <property>
- <name>fs.ozfs.impl</name>
- <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
- <description>The implementation class of the Ozone FileSystem.</description>
- </property>
-
--<property>
- <name>fs.AbstractFileSystem.ozfs.impl</name>
- <value>org.apache.hadoop.fs.ozone.OzFs</value>
- <description>The implementation class of the OzFs AbstractFileSystem.</description>
- <name>fs.s3a.list.version</name>
- <value>2</value>
- <description>
- Select which version of the S3 SDK's List Objects API to use. Currently
- support 2 (default) and 1 (older API).
- </description>
--</property>
++ <property>
++ <name>fs.s3a.list.version</name>
++ <value>2</value>
++ <description>
++ Select which version of the S3 SDK's List Objects API to use. Currently
++ support 2 (default) and 1 (older API).
++ </description>
++ </property>
--<!-- Azure file system properties -->
++ <!-- Azure file system properties -->
<property>
<name>fs.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
@@@ -1547,7 -1459,7 +1459,21 @@@
</property>
--<!-- ipc properties -->
++ <!-- Ozone file system properties -->
++ <property>
++ <name>fs.ozfs.impl</name>
++ <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
++ <description>The implementation class of the Ozone FileSystem.</description>
++ </property>
++
++ <property>
++ <name>fs.AbstractFileSystem.ozfs.impl</name>
++ <value>org.apache.hadoop.fs.ozone.OzFs</value>
++ <description>The implementation class of the OzFs AbstractFileSystem.</description>
++ </property>
++
++
++ <!-- ipc properties -->
<property>
<name>ipc.client.idlethreshold</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19f7f875/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --cc hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 51806c9,742a43a..aa75ce8
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@@ -130,15 -130,17 +130,17 @@@ public class TestAMRMClient
@Before
public void setup() throws Exception {
conf = new YarnConfiguration();
- createClusterAndStartApplication();
+ createClusterAndStartApplication(conf);
}
- private void createClusterAndStartApplication() throws Exception {
+ private void createClusterAndStartApplication(Configuration conf)
+ throws Exception {
// start minicluster
+ this.conf = conf;
conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName);
conf.setLong(
- YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
- rolling_interval_sec);
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+ rolling_interval_sec);
conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
// set the minimum allocation so that resource decrease can go under 1024
@@@ -527,16 -529,17 +529,17 @@@
}
}
}
-
+
private void verifyMatches(
- List<? extends Collection<ContainerRequest>> matches,
- int matchSize) {
+ List<? extends Collection<ContainerRequest>> matches,
+ int matchSize) {
assertEquals(1, matches.size());
- assertEquals(matches.get(0).size(), matchSize);
+ assertEquals(matchSize, matches.get(0).size());
}
-
+
@Test (timeout=60000)
- public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException {
+ public void testAMRMClientMatchingFitInferredRack()
+ throws YarnException, IOException {
AMRMClientImpl<ContainerRequest> amClient = null;
try {
// start am rm client
@@@ -605,6 -609,9 +609,9 @@@
amClient.addContainerRequest(storedContainer2);
amClient.addContainerRequest(storedContainer3);
+ ProfileCapability profileCapability =
+ ProfileCapability.newInstance(capability);
-
++
// test addition and storage
RemoteRequestsTable<ContainerRequest> remoteRequestsTable =
amClient.getTable(0);
@@@ -613,10 -620,10 +620,10 @@@
.remoteRequest.getNumContainers();
assertEquals(2, containersRequestedAny);
containersRequestedAny = remoteRequestsTable.get(priority1,
- ResourceRequest.ANY, ExecutionType.GUARANTEED, capability)
+ ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
.remoteRequest.getNumContainers();
- assertEquals(1, containersRequestedAny);
- List<? extends Collection<ContainerRequest>> matches =
+ assertEquals(1, containersRequestedAny);
+ List<? extends Collection<ContainerRequest>> matches =
amClient.getMatchingRequests(priority, node, capability);
verifyMatches(matches, 2);
matches = amClient.getMatchingRequests(priority, rack, capability);
@@@ -1183,9 -1190,11 +1190,11 @@@
true, null, ExecutionTypeRequest
.newInstance(ExecutionType.OPPORTUNISTIC, true)));
+ ProfileCapability profileCapability =
- ProfileCapability.newInstance(capability);
++ ProfileCapability.newInstance(capability);
int oppContainersRequestedAny =
amClient.getTable(0).get(priority2, ResourceRequest.ANY,
- ExecutionType.OPPORTUNISTIC, capability).remoteRequest
+ ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
.getNumContainers();
assertEquals(1, oppContainersRequestedAny);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org