You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/06/10 03:36:01 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23923086
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23923086
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23923086
Branch: refs/heads/HDFS-7240
Commit: 23923086d1aff381fcf3a0d0e1dd19ed029bda18
Parents: 54f7975 9581fb7
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Jun 9 20:34:39 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Jun 9 20:34:39 2016 -0700
----------------------------------------------------------------------
LICENSE.txt | 1017 +++++++++++++++-
NOTICE.txt | 266 +++++
dev-support/bin/qbt | 18 +
dev-support/bin/yetus-wrapper | 2 +-
dev-support/checkstyle/suppressions.xml | 21 +
hadoop-build-tools/pom.xml | 41 +
hadoop-common-project/hadoop-auth/pom.xml | 27 +-
.../authentication/util/KerberosUtil.java | 18 +-
.../TestKerberosAuthenticationHandler.java | 1 -
.../authentication/util/TestKerberosUtil.java | 26 +-
.../hadoop-common/HadoopCommon.cmake | 9 +-
.../dev-support/findbugsExcludeFile.xml | 8 +-
hadoop-common-project/hadoop-common/pom.xml | 4 +
.../KeyProviderDelegationTokenExtension.java | 30 +-
.../org/apache/hadoop/crypto/key/KeyShell.java | 182 +--
.../crypto/key/kms/KMSClientProvider.java | 158 ++-
.../key/kms/LoadBalancingKMSClientProvider.java | 41 +-
.../hadoop/crypto/key/kms/ValueQueue.java | 2 +-
.../hadoop/fs/CommonConfigurationKeys.java | 3 +
.../hadoop/fs/InvalidRequestException.java | 4 +
.../org/apache/hadoop/fs/LocalDirAllocator.java | 153 ++-
.../hadoop/fs/PathAccessDeniedException.java | 12 +-
.../apache/hadoop/fs/PathNotFoundException.java | 20 +-
.../hadoop/fs/PathPermissionException.java | 16 +-
.../apache/hadoop/io/FastByteComparisons.java | 2 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 113 +-
.../io/erasurecode/ErasureCoderOptions.java | 89 ++
.../erasurecode/coder/HHXORErasureDecoder.java | 18 +-
.../erasurecode/coder/HHXORErasureEncoder.java | 15 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 6 +-
.../io/erasurecode/coder/RSErasureEncoder.java | 6 +-
.../io/erasurecode/coder/XORErasureDecoder.java | 6 +-
.../io/erasurecode/coder/XORErasureEncoder.java | 6 +-
.../rawcoder/AbstractRawErasureCoder.java | 220 ----
.../rawcoder/AbstractRawErasureDecoder.java | 181 ---
.../rawcoder/AbstractRawErasureEncoder.java | 146 ---
.../rawcoder/ByteArrayDecodingState.java | 111 ++
.../rawcoder/ByteArrayEncodingState.java | 81 ++
.../rawcoder/ByteBufferDecodingState.java | 134 +++
.../rawcoder/ByteBufferEncodingState.java | 98 ++
.../io/erasurecode/rawcoder/CoderOption.java | 43 -
.../io/erasurecode/rawcoder/CoderUtil.java | 199 ++++
.../io/erasurecode/rawcoder/DecodingState.java | 55 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 16 +-
.../erasurecode/rawcoder/DummyRawEncoder.java | 15 +-
.../rawcoder/DummyRawErasureCoderFactory.java | 10 +-
.../io/erasurecode/rawcoder/EncodingState.java | 44 +
.../io/erasurecode/rawcoder/RSRawDecoder.java | 48 +-
.../rawcoder/RSRawDecoderLegacy.java | 66 +-
.../io/erasurecode/rawcoder/RSRawEncoder.java | 45 +-
.../rawcoder/RSRawEncoderLegacy.java | 82 +-
.../rawcoder/RSRawErasureCoderFactory.java | 9 +-
.../RSRawErasureCoderFactoryLegacy.java | 9 +-
.../erasurecode/rawcoder/RawErasureCoder.java | 73 --
.../rawcoder/RawErasureCoderFactory.java | 11 +-
.../erasurecode/rawcoder/RawErasureDecoder.java | 137 ++-
.../erasurecode/rawcoder/RawErasureEncoder.java | 135 ++-
.../io/erasurecode/rawcoder/XORRawDecoder.java | 51 +-
.../io/erasurecode/rawcoder/XORRawEncoder.java | 57 +-
.../rawcoder/XORRawErasureCoderFactory.java | 9 +-
.../io/erasurecode/rawcoder/package-info.java | 38 +
.../io/erasurecode/rawcoder/util/CoderUtil.java | 83 --
.../erasurecode/rawcoder/util/GaloisField.java | 4 +-
.../hadoop/io/retry/AsyncCallHandler.java | 321 +++++
.../org/apache/hadoop/io/retry/CallReturn.java | 75 ++
.../hadoop/io/retry/RetryInvocationHandler.java | 134 ++-
.../apache/hadoop/io/retry/RetryPolicies.java | 4 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 124 +-
.../apache/hadoop/ipc/DecayRpcScheduler.java | 130 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 65 +-
.../java/org/apache/hadoop/log/LogLevel.java | 285 ++++-
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 1 +
.../metrics2/lib/DefaultMetricsSystem.java | 9 +
.../metrics2/sink/RollingFileSystemSink.java | 353 +++++-
.../java/org/apache/hadoop/security/KDiag.java | 34 +-
.../hadoop/security/LdapGroupsMapping.java | 12 +
.../hadoop/security/ShellBasedIdMapping.java | 2 +-
.../hadoop/security/alias/CredentialShell.java | 203 ++--
.../hadoop/security/token/DtFileOperations.java | 23 +
.../hadoop/security/token/DtUtilShell.java | 47 +-
.../DelegationTokenAuthenticationHandler.java | 7 +
.../web/DelegationTokenAuthenticator.java | 19 +
.../org/apache/hadoop/tools/CommandShell.java | 4 +-
.../org/apache/hadoop/tracing/TraceAdmin.java | 16 +-
.../org/apache/hadoop/util/HostsFileReader.java | 159 ++-
.../java/org/apache/hadoop/util/KMSUtil.java | 76 ++
.../org/apache/hadoop/util/NativeCrc32.java | 6 +-
.../apache/hadoop/util/concurrent/AsyncGet.java | 65 +
.../hadoop/util/concurrent/AsyncGetFuture.java | 73 ++
...rg.apache.hadoop.security.token.TokenRenewer | 14 +
.../src/main/resources/core-default.xml | 109 +-
.../src/site/markdown/CommandsManual.md | 32 +-
.../hadoop-common/src/site/markdown/Metrics.md | 1 +
.../src/site/markdown/SingleCluster.md.vm | 15 +-
.../hadoop-common/src/site/markdown/Tracing.md | 9 +
.../conf/TestCommonConfigurationFields.java | 6 +
.../kms/TestLoadBalancingKMSClientProvider.java | 94 ++
.../fs/FileContextCreateMkdirBaseTest.java | 73 +-
.../hadoop/fs/contract/ContractTestUtils.java | 420 +++++++
.../erasurecode/TestCodecRawCoderMapping.java | 29 +-
.../hadoop/io/erasurecode/TestCoderBase.java | 14 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 2 +-
.../erasurecode/rawcoder/TestRawCoderBase.java | 50 +-
.../org/apache/hadoop/ipc/TestAsyncIPC.java | 130 +-
.../java/org/apache/hadoop/ipc/TestRPC.java | 31 +-
.../org/apache/hadoop/log/TestLogLevel.java | 479 ++++++--
.../sink/RollingFileSystemSinkTestBase.java | 24 +-
.../sink/TestRollingFileSystemSink.java | 285 +++--
.../TestRollingFileSystemSinkWithLocal.java | 157 +++
.../org/apache/hadoop/net/ServerSocketUtil.java | 39 +
.../hadoop/security/TestLdapGroupsMapping.java | 140 +++
.../security/TestShellBasedIdMapping.java | 16 +-
.../hadoop/security/token/TestDtUtilShell.java | 24 +
.../delegation/web/TestWebDelegationToken.java | 114 +-
.../apache/hadoop/util/TestHostsFileReader.java | 26 +
.../hadoop-common/src/test/resources/krb5.conf | 23 +-
.../key/kms/server/KMSAuthenticationFilter.java | 2 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 134 ++-
.../dev-support/findbugsExcludeFile.xml | 28 +
hadoop-common-project/hadoop-minikdc/pom.xml | 119 +-
.../java/org/apache/hadoop/minikdc/MiniKdc.java | 381 ++----
.../src/main/resources/minikdc-krb5.conf | 25 -
.../src/main/resources/minikdc.ldiff | 47 -
.../org/apache/hadoop/minikdc/TestMiniKdc.java | 19 +-
.../hadoop/hdfs/AsyncDistributedFileSystem.java | 86 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 39 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 7 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 7 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 38 +-
.../hadoop/hdfs/DistributedFileSystem.java | 3 +
.../apache/hadoop/hdfs/FileChecksumHelper.java | 3 +-
.../hadoop/hdfs/protocol/StripedBlockInfo.java | 10 +-
.../hdfs/protocol/datatransfer/Sender.java | 2 +
.../ClientNamenodeProtocolTranslatorPB.java | 73 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 16 +
.../hdfs/web/resources/NoRedirectParam.java | 49 +
.../src/main/proto/datatransfer.proto | 1 +
.../hadoop/fs/http/server/HttpFSServer.java | 218 ++--
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 10 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 21 +-
.../apache/hadoop/hdfs/HDFSPolicyProvider.java | 6 +-
.../hdfs/protocol/datatransfer/Receiver.java | 1 +
.../qjournal/server/JournalNodeRpcServer.java | 6 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 2 +-
.../server/blockmanagement/BlockManager.java | 72 +-
.../blockmanagement/DecommissionManager.java | 8 +-
.../PendingReconstructionBlocks.java | 16 +-
.../hdfs/server/common/HdfsServerConstants.java | 1 -
.../server/datanode/BlockChecksumHelper.java | 172 ++-
.../hadoop/hdfs/server/datanode/DataNode.java | 3 +-
.../server/datanode/ShortCircuitRegistry.java | 6 +-
.../erasurecode/ErasureCodingWorker.java | 15 +-
.../StripedBlockChecksumReconstructor.java | 129 ++
.../erasurecode/StripedBlockReconstructor.java | 119 ++
.../datanode/erasurecode/StripedReader.java | 22 +-
.../erasurecode/StripedReconstructionInfo.java | 99 ++
.../erasurecode/StripedReconstructor.java | 176 +--
.../datanode/erasurecode/StripedWriter.java | 29 +-
.../datanode/web/webhdfs/ParameterParser.java | 5 +
.../datanode/web/webhdfs/WebHdfsHandler.java | 26 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 2 +-
.../server/namenode/EncryptionZoneManager.java | 35 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 2 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 11 +-
.../hdfs/server/namenode/FSNamesystem.java | 49 +-
.../server/namenode/FSPermissionChecker.java | 24 +-
.../hdfs/server/namenode/LeaseManager.java | 21 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 5 +-
.../hdfs/server/namenode/NamenodeFsck.java | 2 +-
.../web/resources/NamenodeWebHdfsMethods.java | 82 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 42 +-
.../hadoop/hdfs/tools/NNHAServiceTarget.java | 2 +-
.../src/main/resources/hdfs-default.xml | 18 +
.../src/main/webapps/hdfs/dfshealth.html | 2 +-
.../src/main/webapps/hdfs/explorer.html | 25 +-
.../src/main/webapps/hdfs/explorer.js | 53 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 2 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 57 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 5 +
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 8 +-
.../org/apache/hadoop/hdfs/TestAsyncDFS.java | 454 +++++++
.../apache/hadoop/hdfs/TestAsyncDFSRename.java | 572 ++-------
.../apache/hadoop/hdfs/TestAsyncHDFSWithHA.java | 181 +++
.../apache/hadoop/hdfs/TestDFSPermission.java | 13 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 4 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 23 +-
.../hadoop/hdfs/TestDFSUpgradeFromImage.java | 16 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 47 +-
.../hdfs/TestDecommissionWithStriped.java | 47 +
.../hadoop/hdfs/TestDistributedFileSystem.java | 1 -
.../apache/hadoop/hdfs/TestFileChecksum.java | 41 +-
.../hadoop/hdfs/TestHDFSPolicyProvider.java | 126 ++
.../apache/hadoop/hdfs/TestMiniDFSCluster.java | 31 -
.../java/org/apache/hadoop/hdfs/TestRead.java | 87 ++
.../hdfs/server/balancer/TestBalancer.java | 17 +-
.../balancer/TestBalancerWithHANameNodes.java | 2 +-
.../TestBalancerWithMultipleNameNodes.java | 2 +-
.../balancer/TestBalancerWithNodeGroup.java | 4 +-
.../TestPendingReconstruction.java | 20 +-
.../server/datanode/SimulatedFSDataset.java | 4 +-
.../TestDataNodeErasureCodingMetrics.java | 16 +
.../hadoop/hdfs/server/mover/TestMover.java | 12 +-
.../hdfs/server/mover/TestStorageMover.java | 2 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 12 +-
.../hdfs/server/namenode/TestLeaseManager.java | 24 +-
.../hdfs/server/namenode/ha/HATestUtil.java | 9 +-
.../server/namenode/ha/TestNNHealthCheck.java | 12 +-
.../shortcircuit/TestShortCircuitCache.java | 34 +-
.../shortcircuit/TestShortCircuitLocalRead.java | 6 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 84 +-
.../sink/TestRollingFileSystemSinkWithHdfs.java | 106 +-
...TestRollingFileSystemSinkWithSecureHdfs.java | 201 ++--
.../apache/hadoop/tracing/TestTraceAdmin.java | 69 +-
.../hadoop-hdfs/src/test/resources/krb5.conf | 19 +-
.../src/test/resources/testHDFSConf.xml | 2 +-
.../hadoop/mapred/LocalContainerLauncher.java | 28 +
.../jobhistory/JobHistoryEventHandler.java | 4 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 32 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 7 +-
.../v2/app/launcher/ContainerLauncherEvent.java | 21 +-
.../v2/app/launcher/ContainerLauncherImpl.java | 19 +-
.../v2/app/rm/RMContainerAllocator.java | 192 +--
.../v2/app/rm/RMContainerRequestor.java | 19 +-
.../v2/app/rm/ResourceCalculatorUtils.java | 17 +-
.../CheckpointAMPreemptionPolicy.java | 8 +-
.../v2/app/TestCheckpointPreemptionPolicy.java | 16 +-
.../hadoop/mapreduce/v2/app/TestRecovery.java | 2 +-
.../mapreduce/v2/app/TestStagingCleanup.java | 120 ++
.../v2/app/job/impl/TestTaskAttempt.java | 29 +-
.../v2/app/launcher/TestContainerLauncher.java | 10 +-
.../app/launcher/TestContainerLauncherImpl.java | 8 +
.../app/local/TestLocalContainerAllocator.java | 2 +-
.../v2/app/rm/TestRMContainerAllocator.java | 2 +-
.../v2/app/rm/TestResourceCalculatorUtils.java | 4 +-
.../apache/hadoop/mapreduce/TypeConverter.java | 6 +-
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 8 +
.../java/org/apache/hadoop/mapred/JobConf.java | 6 -
.../org/apache/hadoop/mapreduce/Cluster.java | 15 +-
.../org/apache/hadoop/mapreduce/JobStatus.java | 18 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 7 +
.../apache/hadoop/mapreduce/MRJobConfig.java | 18 +
.../jobhistory/NormalizedResourceEvent.java | 6 +-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 6 +-
.../src/main/resources/mapred-default.xml | 16 +
.../mapreduce/v2/hs/HistoryClientService.java | 1 +
.../hadoop/mapred/ResourceMgrDelegate.java | 12 +-
.../apache/hadoop/mapred/JobClientUnitTest.java | 6 +-
.../hadoop/mapred/TestClientRedirect.java | 10 +-
.../hadoop/mapred/TestLocalJobSubmission.java | 24 +-
.../hadoop/mapred/TestMROpportunisticMaps.java | 281 +++++
.../hadoop/mapred/TestMiniMRChildTask.java | 233 +---
.../TestClientProtocolProviderImpls.java | 26 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 119 ++
hadoop-project-dist/pom.xml | 2 +
hadoop-project/pom.xml | 48 +-
hadoop-project/src/site/site.xml | 2 +
hadoop-tools/hadoop-ant/pom.xml | 56 -
.../java/org/apache/hadoop/ant/DfsTask.java | 220 ----
.../ant/condition/DfsBaseConditional.java | 68 --
.../apache/hadoop/ant/condition/DfsExists.java | 24 -
.../apache/hadoop/ant/condition/DfsIsDir.java | 24 -
.../apache/hadoop/ant/condition/DfsZeroLen.java | 24 -
.../resources/org/apache/hadoop/ant/antlib.xml | 29 -
hadoop-tools/hadoop-aws/pom.xml | 16 +
.../hadoop/fs/s3a/AWSClientIOException.java | 50 +
.../apache/hadoop/fs/s3a/AWSS3IOException.java | 61 +
.../hadoop/fs/s3a/AWSServiceIOException.java | 72 ++
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 11 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 11 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 18 +-
.../s3a/CredentialInitializationException.java | 46 +
.../fs/s3a/ProgressableProgressListener.java | 94 ++
.../hadoop/fs/s3a/S3AFastOutputStream.java | 152 ++-
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 7 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1032 +++++++++++-----
.../apache/hadoop/fs/s3a/S3AInputStream.java | 33 +-
.../hadoop/fs/s3a/S3AInstrumentation.java | 218 ++--
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 116 +-
.../hadoop/fs/s3a/S3AStorageStatistics.java | 104 ++
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 237 ++++
.../org/apache/hadoop/fs/s3a/Statistic.java | 143 +++
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 70 ++
.../org/apache/hadoop/fs/s3a/package-info.java | 28 +
.../src/site/markdown/tools/hadoop-aws/index.md | 164 ++-
.../fs/contract/s3a/TestS3AContractDistCp.java | 46 +
.../hadoop/fs/s3a/AbstractS3ATestBase.java | 103 ++
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 76 ++
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 293 ++++-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 171 +++
.../hadoop/fs/s3a/TestS3AConfiguration.java | 23 +-
.../apache/hadoop/fs/s3a/TestS3AEncryption.java | 104 ++
.../TestS3AEncryptionAlgorithmPropagation.java | 82 ++
.../s3a/TestS3AEncryptionFastOutputStream.java | 35 +
.../hadoop/fs/s3a/TestS3AFailureHandling.java | 194 +++
.../hadoop/fs/s3a/TestS3AFileOperationCost.java | 191 +++
.../fs/s3a/TestS3AFileSystemContract.java | 1 -
.../fs/s3a/TestS3ATemporaryCredentials.java | 150 +++
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 130 +-
.../fs/s3a/scale/TestS3ADeleteManyFiles.java | 10 +-
.../s3a/scale/TestS3ADirectoryPerformance.java | 189 +++
.../scale/TestS3AInputStreamPerformance.java | 6 +-
.../src/test/resources/log4j.properties | 4 +-
.../dev-support/findbugs-exclude.xml | 24 +
hadoop-tools/hadoop-azure-datalake/pom.xml | 180 +++
.../main/java/org/apache/hadoop/fs/adl/Adl.java | 52 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 41 +
...hedRefreshTokenBasedAccessTokenProvider.java | 135 +++
.../hadoop/fs/adl/oauth2/package-info.java | 23 +
.../org/apache/hadoop/fs/adl/package-info.java | 23 +
.../org/apache/hadoop/hdfs/web/ADLConfKeys.java | 61 +
.../apache/hadoop/hdfs/web/BufferManager.java | 180 +++
.../web/PrivateAzureDataLakeFileSystem.java | 1108 ++++++++++++++++++
...hedRefreshTokenBasedAccessTokenProvider.java | 37 +
.../hadoop/hdfs/web/oauth2/package-info.java | 24 +
.../apache/hadoop/hdfs/web/package-info.java | 25 +
.../hadoop/hdfs/web/resources/ADLFlush.java | 49 +
.../hdfs/web/resources/ADLGetOpParam.java | 96 ++
.../hdfs/web/resources/ADLPostOpParam.java | 97 ++
.../hdfs/web/resources/ADLPutOpParam.java | 94 ++
.../hdfs/web/resources/ADLVersionInfo.java | 51 +
.../web/resources/AppendADLNoRedirectParam.java | 45 +
.../web/resources/CreateADLNoRedirectParam.java | 44 +
.../hadoop/hdfs/web/resources/LeaseParam.java | 53 +
.../web/resources/ReadADLNoRedirectParam.java | 44 +
.../hadoop/hdfs/web/resources/package-info.java | 27 +
.../src/site/markdown/index.md | 219 ++++
...hedRefreshTokenBasedAccessTokenProvider.java | 147 +++
hadoop-tools/hadoop-azure/pom.xml | 19 +
.../contract/TestAzureNativeContractDistCp.java | 33 +
hadoop-tools/hadoop-distcp/pom.xml | 16 +
.../apache/hadoop/tools/util/DistCpUtils.java | 10 +-
.../src/site/markdown/DistCp.md.vm | 1 +
.../contract/AbstractContractDistCpTest.java | 204 ++++
.../sls/scheduler/FairSchedulerMetrics.java | 60 +-
.../sls/scheduler/ResourceSchedulerWrapper.java | 36 +-
.../sls/scheduler/SLSCapacityScheduler.java | 36 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 4 +-
hadoop-tools/pom.xml | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +
.../dev-support/findbugs-exclude.xml | 5 +
.../yarn/api/ApplicationClientProtocol.java | 27 +-
.../yarn/api/ContainerManagementProtocol.java | 5 +
.../GetNewReservationRequest.java | 41 +
.../GetNewReservationResponse.java | 71 ++
.../ReservationSubmissionRequest.java | 25 +-
.../ReservationSubmissionResponse.java | 46 +-
.../SignalContainerResponse.java | 2 +-
.../yarn/api/records/AMBlackListingRequest.java | 67 --
.../records/ApplicationSubmissionContext.java | 23 -
.../hadoop/yarn/api/records/Container.java | 26 +
.../yarn/api/records/ExecutionTypeRequest.java | 124 ++
.../hadoop/yarn/api/records/Resource.java | 34 +-
.../yarn/api/records/ResourceRequest.java | 34 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 67 +-
.../main/proto/applicationclient_protocol.proto | 3 +-
.../proto/containermanagement_protocol.proto | 1 +
.../src/main/proto/yarn_protos.proto | 17 +-
.../src/main/proto/yarn_service_protos.proto | 9 +-
.../yarn/conf/TestYarnConfigurationFields.java | 7 +
.../distributedshell/ApplicationMaster.java | 10 +-
.../applications/distributedshell/Client.java | 4 +-
.../hadoop/yarn/client/api/YarnClient.java | 25 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 16 +-
.../yarn/client/api/impl/YarnClientImpl.java | 14 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 6 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 376 +++---
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 4 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 4 +-
.../api/impl/TestDistributedScheduling.java | 9 +-
.../yarn/client/api/impl/TestYarnClient.java | 374 ++++--
.../hadoop/yarn/client/cli/TestLogsCLI.java | 179 ++-
.../yarn/api/ContainerManagementProtocolPB.java | 7 +
.../ApplicationClientProtocolPBClientImpl.java | 23 +-
...ContainerManagementProtocolPBClientImpl.java | 19 +
.../ApplicationClientProtocolPBServiceImpl.java | 26 +-
...ontainerManagementProtocolPBServiceImpl.java | 20 +
.../impl/pb/GetNewReservationRequestPBImpl.java | 77 ++
.../pb/GetNewReservationResponsePBImpl.java | 144 +++
.../pb/ReservationSubmissionRequestPBImpl.java | 30 +
.../pb/ReservationSubmissionResponsePBImpl.java | 59 -
.../impl/pb/AMBlackListingRequestPBImpl.java | 104 --
.../pb/ApplicationSubmissionContextPBImpl.java | 40 -
.../api/records/impl/pb/ContainerPBImpl.java | 26 +-
.../impl/pb/ExecutionTypeRequestPBImpl.java | 93 ++
.../yarn/api/records/impl/pb/ProtoUtils.java | 15 +
.../api/records/impl/pb/ResourcePBImpl.java | 30 +-
.../records/impl/pb/ResourceRequestPBImpl.java | 52 +-
.../logaggregation/AggregatedLogFormat.java | 175 ++-
.../logaggregation/ContainerLogsRequest.java | 135 +++
.../yarn/logaggregation/LogCLIHelpers.java | 160 ++-
.../nodelabels/CommonNodeLabelsManager.java | 18 +-
.../yarn/security/SchedulerSecurityInfo.java | 2 +-
.../apache/hadoop/yarn/util/AdHocLogDumper.java | 9 +-
.../yarn/util/ProcfsBasedProcessTree.java | 19 +-
.../util/ResourceCalculatorProcessTree.java | 4 +-
.../yarn/util/WindowsBasedProcessTree.java | 8 +
.../resource/DefaultResourceCalculator.java | 36 +-
.../resource/DominantResourceCalculator.java | 46 +-
.../yarn/util/resource/ResourceCalculator.java | 10 +-
.../hadoop/yarn/util/resource/Resources.java | 62 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 53 +-
.../hadoop/yarn/webapp/util/WebAppUtils.java | 18 +
.../src/main/resources/yarn-default.xml | 73 +-
.../hadoop/yarn/TestContainerLaunchRPC.java | 10 +
.../yarn/TestContainerResourceIncreaseRPC.java | 8 +
.../java/org/apache/hadoop/yarn/TestRPC.java | 10 +
.../hadoop/yarn/api/TestPBImplRecords.java | 12 +-
.../yarn/util/TestProcfsBasedProcessTree.java | 66 +-
.../yarn/util/resource/TestResources.java | 8 +-
.../client/impl/zk/RegistrySecurity.java | 2 +-
.../ApplicationHistoryServer.java | 1 +
.../webapp/AHSWebServices.java | 187 +--
.../webapp/TestAHSWebServices.java | 66 ++
.../api/DistributedSchedulerProtocolPB.java | 10 +-
.../pb/DistSchedRegisterResponsePBImpl.java | 6 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 12 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 10 +-
.../yarn/server/webapp/dao/ContainerInfo.java | 10 +-
.../proto/distributed_scheduler_protocol.proto | 1 +
.../hadoop/yarn/TestYarnServerApiClasses.java | 6 +-
.../server/nodemanager/ContainerExecutor.java | 2 +-
.../nodemanager/DefaultContainerExecutor.java | 22 +-
.../containermanager/ContainerManagerImpl.java | 40 +-
.../application/ApplicationImpl.java | 80 +-
.../container/ContainerImpl.java | 20 +-
.../launcher/RecoveredContainerLaunch.java | 7 +-
.../CGroupsMemoryResourceHandlerImpl.java | 4 +-
.../runtime/DockerLinuxContainerRuntime.java | 46 +-
.../localizer/ContainerLocalizer.java | 40 +-
.../logaggregation/AppLogAggregatorImpl.java | 101 +-
.../logaggregation/LogAggregationService.java | 60 +-
.../event/LogHandlerAppStartedEvent.java | 21 +-
.../monitor/ContainerMetrics.java | 18 +-
.../monitor/ContainersMonitorImpl.java | 41 +-
.../queuing/QueuingContainerManagerImpl.java | 41 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 12 +-
.../nodemanager/scheduler/LocalScheduler.java | 3 +-
.../OpportunisticContainerAllocator.java | 3 +-
.../nodemanager/webapp/NMWebServices.java | 89 +-
.../server/nodemanager/webapp/WebServer.java | 5 +-
.../nodemanager/webapp/dao/ContainerInfo.java | 2 +-
.../impl/container-executor.c | 19 +-
.../test/test-container-executor.c | 49 -
.../yarn_server_nodemanager_recovery.proto | 1 +
.../TestDefaultContainerExecutor.java | 89 +-
.../yarn/server/nodemanager/TestEventFlow.java | 3 +
.../nodemanager/TestNodeStatusUpdater.java | 4 +-
.../TestNodeStatusUpdaterForLabels.java | 7 +
.../amrmproxy/MockResourceManagerFacade.java | 12 +-
.../BaseContainerManagerTest.java | 67 ++
.../containermanager/TestContainerManager.java | 76 +-
.../application/TestApplication.java | 45 +-
.../container/TestContainer.java | 34 +
.../runtime/TestDockerContainerRuntime.java | 298 ++++-
.../MockLocalizerHeartbeatResponse.java | 7 +-
.../localizer/TestContainerLocalizer.java | 41 +-
.../TestAppLogAggregatorImpl.java | 436 +++++++
.../TestLogAggregationService.java | 7 +-
.../MockCPUResourceCalculatorProcessTree.java | 70 ++
.../MockResourceCalculatorProcessTree.java | 5 +
.../monitor/TestContainerMetrics.java | 4 +-
.../monitor/TestContainersMonitor.java | 2 +-
.../TestContainersMonitorResourceChange.java | 62 +-
.../queuing/TestQueuingContainerManager.java | 551 +++++++--
.../scheduler/TestLocalScheduler.java | 11 +-
.../nodemanager/webapp/TestNMWebServices.java | 63 +-
.../server/resourcemanager/ClientRMService.java | 38 +-
.../resourcemanager/NodesListManager.java | 69 +-
.../server/resourcemanager/RMAuditLogger.java | 2 +
.../yarn/server/resourcemanager/RMNMInfo.java | 4 +-
.../server/resourcemanager/RMServerUtils.java | 14 +-
.../server/resourcemanager/ResourceManager.java | 1 +
.../resourcemanager/ResourceTrackerService.java | 2 +-
.../blacklist/BlacklistManager.java | 9 +-
.../blacklist/BlacklistUpdates.java | 47 -
.../blacklist/DisabledBlacklistManager.java | 12 +-
.../blacklist/SimpleBlacklistManager.java | 17 +-
.../metrics/SystemMetricsPublisher.java | 2 +-
.../monitor/capacity/TempQueuePerPartition.java | 12 +-
.../nodelabels/RMNodeLabelsManager.java | 9 +-
.../AbstractSchedulerPlanFollower.java | 2 +-
.../reservation/CapacityOverTimePolicy.java | 6 +-
.../reservation/NoOverCommitPolicy.java | 38 +-
.../RLESparseResourceAllocation.java | 6 +-
.../reservation/ReservationInputValidator.java | 7 +
.../reservation/ReservationSystemUtil.java | 2 +-
.../planning/StageAllocatorGreedyRLE.java | 16 +-
.../planning/StageEarliestStartByDemand.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 79 +-
.../rmapp/attempt/RMAppAttempt.java | 2 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 113 +-
.../rmcontainer/RMContainerImpl.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 10 +-
.../scheduler/AppSchedulingInfo.java | 74 +-
.../scheduler/ClusterNodeTracker.java | 10 +-
.../resourcemanager/scheduler/QueueMetrics.java | 61 +-
.../scheduler/SchedulerAppUtils.java | 16 +-
.../scheduler/SchedulerApplicationAttempt.java | 39 +-
.../scheduler/SchedulerUtils.java | 8 +-
.../scheduler/capacity/CSQueueMetrics.java | 24 +-
.../capacity/CapacityHeadroomProvider.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 11 +-
.../CapacitySchedulerConfiguration.java | 6 +-
.../scheduler/capacity/LeafQueue.java | 6 +-
.../allocator/RegularContainerAllocator.java | 4 +-
.../scheduler/fair/FSAppAttempt.java | 5 +-
.../scheduler/fair/FSLeafQueue.java | 25 +-
.../scheduler/fair/FSParentQueue.java | 2 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 35 +-
.../scheduler/fair/FSQueueMetrics.java | 66 +-
.../scheduler/fair/FairScheduler.java | 47 +-
.../fair/FairSchedulerConfiguration.java | 12 +
.../scheduler/fair/QueueManager.java | 5 +-
.../fair/policies/ComputeFairShares.java | 18 +-
.../DominantResourceFairnessPolicy.java | 10 +-
.../fair/policies/FairSharePolicy.java | 20 +-
.../scheduler/fair/policies/FifoPolicy.java | 8 +-
.../scheduler/fifo/FifoScheduler.java | 27 +-
.../scheduler/policy/FairOrderingPolicy.java | 5 +-
.../security/DelegationTokenRenewer.java | 27 +-
.../webapp/DefaultSchedulerPage.java | 4 +-
.../webapp/FairSchedulerAppsBlock.java | 2 +-
.../webapp/RMAppAttemptBlock.java | 9 +-
.../resourcemanager/webapp/RMAppBlock.java | 13 +-
.../resourcemanager/webapp/RMWebServices.java | 128 +-
.../webapp/dao/AMBlackListingRequestInfo.java | 61 -
.../webapp/dao/AppAttemptInfo.java | 8 +-
.../resourcemanager/webapp/dao/AppInfo.java | 20 +-
.../dao/ApplicationSubmissionContextInfo.java | 13 -
.../webapp/dao/FairSchedulerInfo.java | 4 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 14 +-
.../webapp/dao/FifoSchedulerInfo.java | 18 +-
.../webapp/dao/NewReservation.java | 55 +
.../resourcemanager/webapp/dao/NodeInfo.java | 4 +-
.../dao/ReservationSubmissionRequestInfo.java | 11 +
.../dao/ReservationSubmissionResponseInfo.java | 54 -
.../webapp/dao/ResourceInfo.java | 10 +-
.../server/resourcemanager/Application.java | 2 +-
.../yarn/server/resourcemanager/MockNM.java | 8 +-
.../yarn/server/resourcemanager/MockNodes.java | 4 +-
.../yarn/server/resourcemanager/MockRM.java | 20 +-
.../server/resourcemanager/NodeManager.java | 17 +-
.../ReservationACLsTestBase.java | 63 +-
.../resourcemanager/TestAMAuthorization.java | 8 +
.../resourcemanager/TestApplicationACLs.java | 6 +-
.../resourcemanager/TestApplicationCleanup.java | 2 +-
.../TestApplicationMasterLauncher.java | 8 +
.../resourcemanager/TestClientRMService.java | 287 +++--
.../TestContainerResourceUsage.java | 28 +-
.../TestDistributedSchedulingService.java | 166 ++-
.../TestNodeBlacklistingOnAMFailures.java | 251 ++++
.../yarn/server/resourcemanager/TestRM.java | 8 +-
.../yarn/server/resourcemanager/TestRMHA.java | 4 +-
.../resourcemanager/TestRMNodeTransitions.java | 22 +-
.../TestReservationSystemWithRMHA.java | 94 +-
.../TestResourceTrackerService.java | 2 +-
.../resourcemanager/TestSignalContainer.java | 2 +-
.../TestWorkPreservingRMRestart.java | 121 +-
.../applicationsmanager/TestAMRestart.java | 177 +--
.../blacklist/TestBlacklistManager.java | 29 +-
.../metrics/TestSystemMetricsPublisher.java | 10 +-
...alCapacityPreemptionPolicyMockFramework.java | 4 +-
...estProportionalCapacityPreemptionPolicy.java | 39 +-
...alCapacityPreemptionPolicyMockFramework.java | 6 +-
.../nodelabels/TestRMNodeLabelsManager.java | 69 +-
.../reservation/ReservationSystemTestUtil.java | 7 +-
.../TestRLESparseResourceAllocation.java | 40 +-
.../planning/TestSimpleCapacityReplanner.java | 6 +-
.../rmapp/TestRMAppTransitions.java | 58 -
.../attempt/TestRMAppAttemptTransitions.java | 3 +-
.../scheduler/TestAbstractYarnScheduler.java | 16 +-
.../scheduler/TestAppSchedulingInfo.java | 12 +-
.../scheduler/TestQueueMetrics.java | 8 +-
.../scheduler/TestResourceUsage.java | 2 +-
.../scheduler/TestSchedulerUtils.java | 20 +-
.../capacity/TestApplicationLimits.java | 15 +-
.../capacity/TestApplicationPriority.java | 52 +-
.../capacity/TestCapacityScheduler.java | 129 +-
.../TestCapacitySchedulerLazyPreemption.java | 30 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 52 +-
...TestCapacitySchedulerSurgicalPreemption.java | 4 +-
.../scheduler/capacity/TestChildQueueOrder.java | 4 +-
.../capacity/TestContainerAllocation.java | 53 +-
.../capacity/TestContainerResizing.java | 150 +--
.../capacity/TestIncreaseAllocationExpirer.java | 36 +-
.../scheduler/capacity/TestLeafQueue.java | 492 ++++----
.../TestNodeLabelContainerAllocation.java | 15 +-
.../scheduler/capacity/TestParentQueue.java | 4 +-
.../scheduler/capacity/TestQueueCapacities.java | 2 +-
.../scheduler/capacity/TestReservations.java | 406 +++----
...TestWorkPreservingRMRestartForNodeLabel.java | 4 +-
.../scheduler/fair/FairSchedulerTestBase.java | 4 +-
.../scheduler/fair/TestComputeFairShares.java | 2 +-
.../fair/TestContinuousScheduling.java | 2 +-
.../scheduler/fair/TestFSAppAttempt.java | 44 +-
.../scheduler/fair/TestFSLeafQueue.java | 27 +-
.../scheduler/fair/TestFairScheduler.java | 403 ++++---
.../fair/TestFairSchedulerFairShare.java | 32 +-
.../fair/TestFairSchedulerPreemption.java | 62 +-
.../scheduler/fifo/TestFifoScheduler.java | 63 +-
.../security/TestDelegationTokenRenewer.java | 98 ++
.../webapp/TestRMWebServices.java | 17 +-
.../webapp/TestRMWebServicesApps.java | 46 +-
.../TestRMWebServicesAppsModification.java | 39 +-
.../webapp/TestRMWebServicesCapacitySched.java | 5 +-
.../webapp/TestRMWebServicesNodes.java | 4 +-
.../webapp/TestRMWebServicesReservation.java | 351 ++++--
.../hadoop/yarn/webapp/TestRMWithXFSFilter.java | 158 +++
.../src/test/resources/krb5.conf | 23 +-
.../src/test/resources/submit-reservation.json | 1 +
.../yarn/server/timeline/EntityCacheItem.java | 66 +-
.../timeline/EntityGroupFSTimelineStore.java | 82 +-
.../timeline/EntityGroupPlugInForTest.java | 19 +-
.../TestEntityGroupFSTimelineStore.java | 248 +++-
.../src/site/markdown/CapacityScheduler.md | 2 +-
.../src/site/markdown/FairScheduler.md | 7 +-
.../src/site/markdown/ReservationSystem.md | 4 +-
.../src/site/markdown/ResourceManagerRest.md | 90 +-
hadoop-yarn-project/hadoop-yarn/pom.xml | 7 +
pom.xml | 44 +
620 files changed, 27022 insertions(+), 9621 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23923086/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23923086/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23923086/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org