You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2019/10/04 12:22:27 UTC
[hadoop] 01/01: Merge remote-tracking branch 'origin/trunk' into
HDDS-1880-Decom
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit ec70207838d5b29fa0b534b13c103865e50a35e8
Merge: fd5e877 6171a41
Author: Márton Elek <el...@apache.org>
AuthorDate: Fri Oct 4 14:17:38 2019 +0200
Merge remote-tracking branch 'origin/trunk' into HDDS-1880-Decom
BUILDING.txt | 31 -
dev-support/docker/Dockerfile | 19 +-
.../hadoop-cos/dev-support/findbugs-exclude.xml | 18 +
hadoop-cloud-storage-project/hadoop-cos/pom.xml | 140 ++++
.../site/markdown/cloud-storage/index.md | 367 ++++++++++
.../hadoop-cos/site/resources/css/site.css | 29 +
.../java/org/apache/hadoop/fs/cosn/BufferPool.java | 245 +++++++
.../hadoop/fs/cosn/ByteBufferInputStream.java | 89 +++
.../hadoop/fs/cosn/ByteBufferOutputStream.java | 74 ++
.../apache/hadoop/fs/cosn/ByteBufferWrapper.java | 103 +++
.../java/org/apache/hadoop/fs/cosn/Constants.java | 35 +-
.../main/java/org/apache/hadoop/fs/cosn/CosN.java | 31 +-
.../org/apache/hadoop/fs/cosn/CosNConfigKeys.java | 86 +++
.../apache/hadoop/fs/cosn/CosNCopyFileContext.java | 66 ++
.../apache/hadoop/fs/cosn/CosNCopyFileTask.java | 68 ++
.../apache/hadoop/fs/cosn/CosNFileReadTask.java | 125 ++++
.../org/apache/hadoop/fs/cosn/CosNFileSystem.java | 814 +++++++++++++++++++++
.../org/apache/hadoop/fs/cosn/CosNInputStream.java | 365 +++++++++
.../apache/hadoop/fs/cosn/CosNOutputStream.java | 284 +++++++
.../java/org/apache/hadoop/fs/cosn/CosNUtils.java | 167 +++++
.../hadoop/fs/cosn/CosNativeFileSystemStore.java | 768 +++++++++++++++++++
.../org/apache/hadoop/fs/cosn/FileMetadata.java | 68 ++
.../hadoop/fs/cosn/NativeFileSystemStore.java | 99 +++
.../org/apache/hadoop/fs/cosn/PartialListing.java | 64 ++
.../main/java/org/apache/hadoop/fs/cosn/Unit.java | 27 +-
.../fs/cosn/auth/COSCredentialProviderList.java | 139 ++++
.../EnvironmentVariableCredentialProvider.java | 55 ++
.../fs/cosn/auth/NoAuthWithCOSException.java | 32 +-
.../fs/cosn/auth/SimpleCredentialProvider.java | 54 ++
.../apache/hadoop/fs/cosn/auth/package-info.java | 19 +-
.../org/apache/hadoop/fs/cosn/package-info.java | 19 +-
.../apache/hadoop/fs/cosn/CosNTestConfigKey.java | 30 +-
.../org/apache/hadoop/fs/cosn/CosNTestUtils.java | 78 ++
.../apache/hadoop/fs/cosn/TestCosNInputStream.java | 167 +++++
.../hadoop/fs/cosn/TestCosNOutputStream.java | 87 +++
.../hadoop/fs/cosn/contract/CosNContract.java | 36 +-
.../fs/cosn/contract/TestCosNContractCreate.java | 26 +-
.../fs/cosn/contract/TestCosNContractDelete.java | 26 +-
.../fs/cosn/contract/TestCosNContractDistCp.java | 54 ++
.../contract/TestCosNContractGetFileStatus.java | 27 +-
.../fs/cosn/contract/TestCosNContractMkdir.java | 26 +-
.../fs/cosn/contract/TestCosNContractOpen.java | 26 +-
.../fs/cosn/contract/TestCosNContractRename.java | 26 +-
.../fs/cosn/contract/TestCosNContractRootDir.java | 27 +-
.../fs/cosn/contract/TestCosNContractSeek.java | 26 +-
.../hadoop/fs/cosn/contract/package-info.java | 19 +-
.../src/test/resources/contract/cosn.xml | 120 +++
.../hadoop-cos/src/test/resources/core-site.xml | 107 +++
.../hadoop-cos/src/test/resources/log4j.properties | 18 +
hadoop-cloud-storage-project/pom.xml | 1 +
.../apache/hadoop/crypto/CryptoInputStream.java | 67 +-
.../org/apache/hadoop/fs/AbstractFileSystem.java | 16 +-
.../hadoop/fs/ByteBufferPositionedReadable.java | 24 +
.../org/apache/hadoop/fs/ChecksumFileSystem.java | 22 +
.../apache/hadoop/fs/CommonPathCapabilities.java | 126 ++++
.../org/apache/hadoop/fs/DelegateToFileSystem.java | 7 +
.../org/apache/hadoop/fs/FSDataInputStream.java | 23 +-
.../java/org/apache/hadoop/fs/FileContext.java | 23 +-
.../main/java/org/apache/hadoop/fs/FileSystem.java | 30 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 7 +
.../main/java/org/apache/hadoop/fs/FilterFs.java | 5 +
.../main/java/org/apache/hadoop/fs/Globber.java | 208 +++++-
.../java/org/apache/hadoop/fs/HarFileSystem.java | 19 +-
.../org/apache/hadoop/fs/PathCapabilities.java | 61 ++
.../org/apache/hadoop/fs/RawLocalFileSystem.java | 19 +
.../hadoop/fs/http/AbstractHttpFileSystem.java | 18 +
.../apache/hadoop/fs/impl/FsLinkResolution.java | 98 +++
.../hadoop/fs/impl/PathCapabilitiesSupport.java | 40 +-
.../java/org/apache/hadoop/fs/shell/Mkdir.java | 4 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 32 +
.../apache/hadoop/util/NodeHealthScriptRunner.java | 1 +
.../src/site/markdown/DeprecatedProperties.md | 4 +
.../src/site/markdown/filesystem/filesystem.md | 5 +-
.../src/site/markdown/filesystem/index.md | 1 +
.../site/markdown/filesystem/pathcapabilities.md | 158 ++++
.../site/markdown/release/3.1.3/CHANGES.3.1.3.md | 336 +++++++++
.../markdown/release/3.1.3/RELEASENOTES.3.1.3.md | 59 ++
.../hadoop/crypto/CryptoStreamsTestBase.java | 115 ++-
.../apache/hadoop/crypto/TestCryptoStreams.java | 24 +
.../hadoop/crypto/TestCryptoStreamsForLocalFS.java | 5 +
.../hadoop/crypto/TestCryptoStreamsNormal.java | 5 +
.../fs/contract/AbstractContractAppendTest.java | 10 +
.../fs/contract/AbstractContractConcatTest.java | 7 +
.../hadoop/fs/contract/ContractTestUtils.java | 50 +-
.../org/apache/hadoop/test/LambdaTestUtils.java | 3 +
.../hadoop/util/TestNodeHealthScriptRunner.java | 9 +
hadoop-hdds/common/pom.xml | 4 +-
.../function/FunctionWithServiceException.java | 29 +-
.../apache/hadoop/hdds/function/package-info.java | 15 +-
.../SCMSecurityProtocolClientSideTranslatorPB.java | 104 ++-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 132 ----
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +
...inerLocationProtocolClientSideTranslatorPB.java | 411 +++++------
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 24 +-
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 62 ++
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 9 +
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 7 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 63 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 166 ++++-
...inerLocationProtocolServerSideTranslatorPB.java | 389 ----------
.../src/main/proto/SCMSecurityProtocol.proto | 96 ++-
.../src/main/proto/ScmBlockLocationProtocol.proto | 2 +-
.../proto/StorageContainerLocationProtocol.proto | 185 +++--
.../common/src/main/resources/ozone-default.xml | 8 +
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 16 +-
.../apache/hadoop/ozone/lock/TestLockManager.java | 145 +++-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 5 +-
hadoop-hdds/container-service/pom.xml | 9 +-
.../server/ratis/ContainerStateMachine.java | 20 +-
.../transport/server/ratis/XceiverServerRatis.java | 8 +-
.../replication/GrpcReplicationClient.java | 6 +
...inerDatanodeProtocolClientSideTranslatorPB.java | 60 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 115 +--
.../proto/StorageContainerDatanodeProtocol.proto | 58 +-
.../ozone/container/common/SCMTestUtils.java | 4 +-
.../server/OzoneProtocolMessageDispatcher.java | 88 +++
.../apache/hadoop/hdds/server/ProfileServlet.java | 10 +-
.../hadoop/hdds/server/PrometheusMetricsSink.java | 16 +-
.../hdds/server/TestPrometheusMetricsSink.java | 77 +-
hadoop-hdds/pom.xml | 17 +-
hadoop-hdds/server-scm/pom.xml | 4 +-
.../container/AbstractContainerReportHandler.java | 8 +
.../hdds/scm/container/ContainerManager.java | 8 +
.../hdds/scm/container/ContainerReportHandler.java | 2 +
.../IncrementalContainerReportHandler.java | 10 +
.../hdds/scm/container/SCMContainerManager.java | 48 +-
.../metrics/SCMContainerManagerMetrics.java | 144 ++++
.../hdds/scm/container/metrics/package-info.java | 22 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 51 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 186 +++++
...lockLocationProtocolServerSideTranslatorPB.java | 56 +-
...inerLocationProtocolServerSideTranslatorPB.java | 393 ++++++++++
.../hadoop/hdds/scm/protocol/package-info.java | 22 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 26 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 9 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 102 ++-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 27 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 36 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 67 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 6 +-
.../scm/server/TestSCMBlockProtocolServer.java | 3 +-
.../testutils/ReplicationNodeManagerMock.java | 5 +-
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 22 +-
.../hdds/scm/cli/container/CloseSubcommand.java | 7 +-
...CloseSubcommand.java => ContainerCommands.java} | 37 +-
.../hdds/scm/cli/container/CreateSubcommand.java | 5 +-
.../hdds/scm/cli/container/DeleteSubcommand.java | 7 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 5 +-
.../hdds/scm/cli/container/ListSubcommand.java | 5 +-
.../cli/pipeline/ActivatePipelineSubcommand.java | 11 +-
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 11 +-
.../cli/pipeline/DeactivatePipelineSubcommand.java | 11 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 11 +-
.../PipelineCommands.java} | 38 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 14 +
.../apache/hadoop/hdfs/DistributedFileSystem.java | 22 +
.../hadoop/hdfs/client/DfsPathCapabilities.java | 62 ++
.../org/apache/hadoop/hdfs/protocol/Block.java | 141 +++-
.../namenode/ha/ObserverReadProxyProvider.java | 26 +-
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 30 +
.../hadoop/hdfs/TestDefaultNameNodePort.java | 38 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 30 +
.../fs/http/server/HttpFSAuthenticationFilter.java | 16 +-
.../fs/http/server/HttpFSServerWebServer.java | 22 +
.../service/hadoop/FileSystemAccessService.java | 21 +-
.../src/main/resources/httpfs-default.xml | 20 +-
.../main/native/libhdfs-tests/test_libhdfs_ops.c | 43 +-
.../src/main/native/libhdfs/hdfs.c | 117 +++
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 21 +-
.../src/main/native/libhdfspp/tests/hdfs_shim.c | 6 +
.../libhdfspp/tests/libhdfs_wrapper_defines.h | 1 +
.../libhdfspp/tests/libhdfs_wrapper_undefs.h | 1 +
.../libhdfspp/tests/libhdfspp_wrapper_defines.h | 1 +
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 48 ++
.../src/main/webapps/router/federationhealth.html | 10 +-
.../src/main/webapps/router/federationhealth.js | 7 +-
.../src/main/webapps/static/rbf.css | 26 +-
.../fs/contract/router/RouterHDFSContract.java | 17 +-
.../fs/contract/router/SecurityConfUtil.java | 15 +-
.../router/TestRouterHDFSContractAppendSecure.java | 4 +-
.../router/TestRouterHDFSContractConcatSecure.java | 5 +-
.../router/TestRouterHDFSContractCreateSecure.java | 5 +-
.../TestRouterHDFSContractDelegationToken.java | 2 +-
.../router/TestRouterHDFSContractDeleteSecure.java | 4 +-
.../TestRouterHDFSContractGetFileStatusSecure.java | 5 +-
.../router/TestRouterHDFSContractMkdirSecure.java | 5 +-
.../router/TestRouterHDFSContractOpenSecure.java | 5 +-
.../router/TestRouterHDFSContractRenameSecure.java | 5 +-
.../TestRouterHDFSContractRootDirectorySecure.java | 5 +-
.../router/TestRouterHDFSContractSeekSecure.java | 5 +-
.../TestRouterHDFSContractSetTimesSecure.java | 5 +-
.../server/federation/router/TestRouterQuota.java | 2 +-
.../security/TestRouterHttpDelegationToken.java | 1 +
.../dev-support/jdiff/Apache_Hadoop_HDFS_3.1.3.xml | 676 +++++++++++++++++
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 11 +-
.../hdfs/qjournal/client/IPCLoggerChannel.java | 14 +-
.../hadoop/hdfs/qjournal/client/QuorumCall.java | 18 +
.../hdfs/qjournal/client/QuorumJournalManager.java | 2 +
.../token/block/BlockTokenSecretManager.java | 31 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 6 +
.../blockmanagement/BlockInfoContiguous.java | 14 +
.../server/blockmanagement/BlockInfoStriped.java | 9 +
.../hdfs/server/blockmanagement/BlockManager.java | 94 ++-
.../blockmanagement/BlockPlacementStatus.java | 8 +
.../BlockPlacementStatusDefault.java | 8 +
.../BlockPlacementStatusWithNodeGroup.java | 11 +
.../BlockPlacementStatusWithUpgradeDomain.java | 22 +-
.../blockmanagement/BlockReportLeaseManager.java | 8 +-
.../hdfs/server/namenode/INodeDirectory.java | 6 +
.../hadoop/hdfs/server/namenode/INodeFile.java | 3 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 14 +-
.../namenode/snapshot/AbstractINodeDiffList.java | 4 +
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../src/main/resources/hdfs-default.xml | 19 +
.../src/main/webapps/hdfs/dfshealth.html | 4 +-
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 4 +-
.../src/site/markdown/ObserverNameNode.md | 14 +
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 17 +-
.../hadoop/hdfs/TestBlockMissingException.java | 2 -
.../apache/hadoop/hdfs/TestByteBufferPread.java | 27 +-
.../java/org/apache/hadoop/hdfs/TestClose.java | 1 -
.../hdfs/TestStateAlignmentContextWithHA.java | 1 +
.../hadoop/hdfs/qjournal/MiniJournalCluster.java | 6 +-
.../qjournal/client/TestQuorumJournalManager.java | 34 +-
.../hdfs/security/token/block/TestBlockToken.java | 23 +
.../blockmanagement/BlockManagerTestUtil.java | 30 +-
.../hdfs/server/blockmanagement/TestBlockInfo.java | 36 +
.../server/blockmanagement/TestBlockManager.java | 58 ++
.../TestBlockPlacementStatusDefault.java | 57 ++
.../TestBlockPlacementStatusWithUpgradeDomain.java | 59 +-
.../TestBlocksWithNotEnoughRacks.java | 111 ++-
.../namenode/ha/TestConsistentReadsObserver.java | 2 +
.../ha/TestFailoverWithBlockTokensEnabled.java | 36 +-
.../server/namenode/ha/TestMultiObserverNode.java | 2 +
.../hdfs/server/namenode/ha/TestObserverNode.java | 2 +
.../namenode/snapshot/TestRenameWithSnapshots.java | 15 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 2 +-
.../hadoop-mapreduce-client-common/pom.xml | 34 +-
.../src/main/proto/HSAdminRefreshProtocol.proto | 3 +-
.../src/main/proto/MRClientProtocol.proto | 1 +
.../src/main/proto/mr_protos.proto | 1 +
.../src/main/proto/mr_service_protos.proto | 1 +
.../org/apache/hadoop/mapred/FileInputFormat.java | 5 +-
.../hadoop/mapred/InvalidInputException.java | 4 +
.../hadoop/mapred/LocatedFileStatusFetcher.java | 66 +-
.../mapreduce/lib/input/FileInputFormat.java | 6 +-
.../mapreduce/lib/input/InvalidInputException.java | 4 +
.../hadoop-mapreduce-client-shuffle/pom.xml | 27 +-
.../src/main/proto/ShuffleHandlerRecovery.proto | 1 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +-
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 57 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +-
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 52 ++
...lumeListCodec.java => UserVolumeInfoCodec.java} | 13 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 100 ++-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 91 +++
.../hadoop/ozone/security/GDPRSymmetricKey.java | 8 +-
.../OzoneDelegationTokenSecretManager.java | 5 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 7 +-
.../src/main/proto/OzoneManagerProtocol.proto | 18 +-
.../ozone/security/TestGDPRSymmetricKey.java | 4 +-
hadoop-ozone/csi/pom.xml | 4 +-
.../dev-support/checks/_mvn_unit_report.sh | 16 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 12 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 39 +-
.../src/main/compose/ozonesecure-mr/docker-config | 31 +-
.../{test-single.sh => ozonesecure-mr/test.sh} | 37 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 4 +-
hadoop-ozone/dist/src/main/compose/test-single.sh | 2 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 27 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 89 +++
.../smoketest/{kinit.robot => kinit-hadoop.robot} | 2 +-
hadoop-ozone/dist/src/main/smoketest/kinit.robot | 5 +-
.../dist/src/main/smoketest/mapreduce.robot | 2 +-
hadoop-ozone/insight/pom.xml | 9 +-
.../ozone/insight/BaseInsightSubCommand.java | 7 +-
.../scm/ScmProtocolBlockLocationInsight.java | 6 +-
...va => ScmProtocolContainerLocationInsight.java} | 18 +-
...nsight.java => ScmProtocolDatanodeInsight.java} | 27 +-
...nsight.java => ScmProtocolSecurityInsight.java} | 18 +-
.../metrics/TestSCMContainerManagerMetrics.java | 167 +++++
.../scm/pipeline/TestRatisPipelineProvider.java | 26 +-
.../rpc/TestContainerStateMachineFailures.java | 65 ++
.../client/rpc/TestOzoneRpcClientAbstract.java | 76 +-
.../ozone/om/TestOzoneManagerConfiguration.java | 9 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 97 +++
hadoop-ozone/ozone-manager/pom.xml | 5 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 40 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 113 +--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 232 +-----
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 35 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 306 ++++++++
.../hadoop/ozone/om/{ => ha}/OMNodeDetails.java | 2 +-
.../apache/hadoop/ozone/om/ha/package-info.java | 14 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 3 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 8 +-
.../multipart/S3MultipartUploadAbortRequest.java | 5 +-
.../S3MultipartUploadCommitPartRequest.java | 4 +-
.../om/request/volume/OMVolumeCreateRequest.java | 12 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 5 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 32 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 10 +-
.../om/response/bucket/OMBucketCreateResponse.java | 8 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 4 +-
.../bucket/OMBucketSetPropertyResponse.java | 7 +-
.../response/file/OMDirectoryCreateResponse.java | 3 +-
.../om/response/file/OMFileCreateResponse.java | 3 +-
.../om/response/key/OMAllocateBlockResponse.java | 6 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 7 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 3 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 32 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 4 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 6 +-
.../multipart/S3MultipartUploadAbortResponse.java | 21 +-
.../S3MultipartUploadCommitPartResponse.java | 45 +-
.../S3MultipartUploadCompleteResponse.java | 3 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 3 +-
.../om/response/volume/OMVolumeCreateResponse.java | 13 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 10 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 12 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 4 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 41 +-
.../org/apache/hadoop/ozone/web/ozShell/Shell.java | 3 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 21 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/TestOMRequestUtils.java | 25 +-
.../hadoop/ozone/om/request/package-info.java} | 18 +-
.../request/volume/TestOMVolumeCreateRequest.java | 16 +-
.../volume/TestOMVolumeSetOwnerRequest.java | 4 +-
.../ozone/om/response/TestOMResponseUtils.java | 8 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 20 +-
.../hadoop/ozone/om/response/package-info.java} | 18 +-
.../s3/multipart/TestS3MultipartResponse.java | 3 +-
.../TestS3MultipartUploadAbortResponse.java | 19 +-
.../volume/TestOMVolumeCreateResponse.java | 7 +-
.../volume/TestOMVolumeDeleteResponse.java | 11 +-
.../volume/TestOMVolumeSetOwnerResponse.java | 17 +-
.../ozone/om/response/volume/package-info.java | 22 +-
.../TestOzoneDelegationTokenSecretManager.java | 29 +-
hadoop-ozone/ozonefs-lib-current/pom.xml | 4 +-
hadoop-ozone/ozonefs-lib-legacy/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 9 +-
hadoop-ozone/pom.xml | 14 +-
hadoop-ozone/recon/pom.xml | 4 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 35 +
.../recon/recovery/ReconOmMetadataManagerImpl.java | 21 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 28 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 33 +-
.../recon/spi/impl/ReconContainerDBProvider.java | 32 +-
.../ozone/recon/AbstractOMMetadataManagerTest.java | 2 +-
.../apache/hadoop/ozone/recon/TestReconUtils.java | 31 +
.../recovery/TestReconOmMetadataManagerImpl.java | 133 ++--
.../impl/TestOzoneManagerServiceProviderImpl.java | 29 +-
.../spi/impl/TestReconContainerDBProvider.java | 13 -
hadoop-ozone/s3gateway/pom.xml | 5 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 11 +-
.../hadoop/ozone/s3/OzoneServiceProvider.java | 50 +-
.../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 44 ++
.../hadoop/ozone/s3/util/TestOzoneS3Util.java | 130 ++++
hadoop-ozone/tools/pom.xml | 9 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 3 +-
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 +-
.../services/org.apache.hadoop.fs.FileSystem} | 11 +-
hadoop-ozone/upgrade/pom.xml | 5 +-
hadoop-project/pom.xml | 4 +-
hadoop-project/src/site/site.xml | 1 +
.../java/org/apache/hadoop/fs/s3a/Constants.java | 28 +-
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 3 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 76 +-
.../hadoop/fs/s3a/auth/AwsSignerInitializer.java | 53 ++
.../hadoop/fs/s3a/{ => auth}/SignerManager.java | 80 +-
.../auth/delegation/DelegationTokenProvider.java | 20 +-
.../hadoop/fs/s3a/commit/CommitConstants.java | 22 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 3 +-
.../hadoop/fs/s3a/select/SelectConstants.java | 2 +-
.../apache/hadoop/fs/s3a/select/SelectTool.java | 2 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 58 ++
.../fs/s3a/ITestLocatedFileStatusFetcher.java | 31 +-
.../hadoop/fs/s3a/ITestS3AFSMainOperations.java | 65 ++
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 7 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 7 +-
.../apache/hadoop/fs/s3a/TestSignerManager.java | 130 ----
.../hadoop/fs/s3a/auth/ITestCustomSigner.java | 237 ++++++
.../fs/s3a/auth/ITestRestrictedReadAccess.java | 707 ++++++++++++++++++
.../hadoop/fs/s3a/auth/TestSignerManager.java | 590 +++++++++++++++
.../fs/s3a/commit/ITestCommitOperations.java | 5 +-
.../s3a/s3guard/AbstractS3GuardToolTestBase.java | 2 +-
.../apache/hadoop/fs/s3a/select/ITestS3Select.java | 4 +-
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 18 +
.../fs/adl/live/TestAdlSdkConfiguration.java | 1 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 17 +
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 14 +
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 22 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +-
.../fs/azurebfs/constants/ConfigurationKeys.java | 8 +
.../constants/FileSystemConfigurations.java | 1 +
.../fs/azurebfs/services/AbfsOutputStream.java | 7 +-
.../hadoop-azure/src/site/markdown/abfs.md | 17 +
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 27 +-
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 6 +-
.../org/apache/hadoop/tools/DistCpConstants.java | 6 +
.../apache/hadoop/tools/mapred/CopyCommitter.java | 10 +-
.../org/apache/hadoop/tools/mapred/CopyMapper.java | 3 +-
.../tools/mapred/RetriableFileCopyCommand.java | 34 +-
.../org/apache/hadoop/tools/util/DistCpUtils.java | 26 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 5 +-
.../apache/hadoop/tools/mapred/TestCopyMapper.java | 56 ++
.../tools/mapred/TestRetriableFileCopyCommand.java | 25 +-
.../apache/hadoop/tools/util/TestDistCpUtils.java | 45 +-
.../tools/util/TestDistCpUtilsWithCombineMode.java | 4 +-
.../BlockPlacementPolicyAlwaysSatisfied.java | 5 +
.../java/org/apache/hadoop/yarn/sls/SLSRunner.java | 25 +
.../hadoop/yarn/sls/conf/SLSConfiguration.java | 3 +
.../apache/hadoop/yarn/sls/BaseSLSRunnerTest.java | 4 +-
.../org/apache/hadoop/yarn/sls/TestSLSRunner.java | 39 +
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 41 +-
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 22 +
.../src/main/proto/YarnCsiAdaptor.proto | 1 +
.../main/proto/application_history_client.proto | 1 +
.../main/proto/applicationclient_protocol.proto | 1 +
.../main/proto/applicationmaster_protocol.proto | 1 +
.../src/main/proto/client_SCM_protocol.proto | 1 +
.../main/proto/containermanagement_protocol.proto | 1 +
.../src/main/proto/server/SCM_Admin_protocol.proto | 1 +
.../proto/server/application_history_server.proto | 1 +
.../resourcemanager_administration_protocol.proto | 1 +
...arn_server_resourcemanager_service_protos.proto | 1 +
.../src/main/proto/yarn_csi_adaptor.proto | 4 +-
.../src/main/proto/yarn_protos.proto | 1 +
.../src/main/proto/yarn_service_protos.proto | 1 +
.../hadoop-yarn-services-core/pom.xml | 23 +-
.../src/main/proto/ClientAMProtocol.proto | 1 +
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 41 +-
.../hadoop/yarn/client/cli/SchedConfCLI.java | 147 +++-
.../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 59 +-
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 187 ++++-
.../src/test/proto/test_amrm_token.proto | 1 +
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 41 +-
.../hadoop/yarn/logaggregation/LogToolUtils.java | 60 +-
.../src/main/proto/yarn_security_token.proto | 1 +
.../src/main/resources/yarn-default.xml | 9 +
.../logaggregation/TestAggregatedLogFormat.java | 42 +-
.../TestLogAggregationIndexedFileController.java | 31 +-
.../pom.xml | 39 +-
.../yarn_server_timelineserver_recovery.proto | 1 +
.../hadoop-yarn-server-common/pom.xml | 39 +-
...DistributedOpportunisticContainerAllocator.java | 357 +++++++++
.../scheduler/OpportunisticContainerAllocator.java | 347 ++-------
.../src/main/proto/ResourceTracker.proto | 1 +
.../src/main/proto/SCMUploader.proto | 1 +
.../main/proto/collectornodemanager_protocol.proto | 1 +
.../proto/distributed_scheduling_am_protocol.proto | 2 +-
.../src/main/proto/yarn_server_common_protos.proto | 1 +
.../proto/yarn_server_common_service_protos.proto | 1 +
.../main/proto/yarn_server_federation_protos.proto | 1 +
.../TestOpportunisticContainerAllocator.java | 2 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 34 +-
.../yarn/server/nodemanager/NodeManager.java | 3 +-
.../src/main/proto/LocalizationProtocol.proto | 1 +
.../proto/yarn_server_nodemanager_recovery.proto | 1 +
.../yarn_server_nodemanager_service_protos.proto | 1 +
.../logaggregation/TestLogAggregationService.java | 106 +--
.../scheduler/TestDistributedScheduler.java | 4 +-
.../nodemanager/webapp/TestNMWebServices.java | 43 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 69 +-
.../server/resourcemanager/ClientRMService.java | 6 +-
.../resourcemanager/DefaultAMSProcessor.java | 8 +
.../OpportunisticContainerAllocatorAMService.java | 30 +-
.../yarn/server/resourcemanager/RMAppManager.java | 7 +
.../yarn/server/resourcemanager/RMAuditLogger.java | 53 +-
.../scheduler/MutableConfigurationProvider.java | 2 +
.../scheduler/SchedulerApplicationAttempt.java | 9 +
.../resourcemanager/scheduler/SchedulerUtils.java | 23 +
.../capacity/CapacitySchedulerConfiguration.java | 7 +
.../scheduler/capacity/LeafQueue.java | 8 +-
.../conf/FSSchedulerConfigurationStore.java | 5 +
.../capacity/conf/InMemoryConfigurationStore.java | 5 +
.../capacity/conf/LeveldbConfigurationStore.java | 7 +
.../conf/MutableCSConfigurationProvider.java | 46 +-
.../capacity/conf/YarnConfigurationStore.java | 7 +
.../capacity/conf/ZKConfigurationStore.java | 5 +
.../policy/AbstractComparatorOrderingPolicy.java | 8 +-
.../FifoOrderingPolicyWithExclusivePartitions.java | 144 ++++
.../scheduler/policy/IteratorSelector.java} | 37 +-
.../scheduler/policy/OrderingPolicy.java | 3 +-
.../scheduler/policy/SchedulableEntity.java | 5 +
.../server/resourcemanager/webapp/RMWSConsts.java | 3 +
.../resourcemanager/webapp/RMWebServices.java | 31 +
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/TestRMAuditLogger.java | 21 +-
.../scheduler/TestSchedulerUtils.java | 142 ++++
.../scheduler/capacity/TestCapacityScheduler.java | 11 +-
.../scheduler/capacity/TestLeafQueue.java | 145 +++-
.../conf/TestFSSchedulerConfigurationStore.java | 16 +
.../conf/TestMutableCSConfigurationProvider.java | 8 +
.../capacity/conf/TestZKConfigurationStore.java | 9 +
.../scheduler/policy/MockSchedulableEntity.java | 15 +-
.../scheduler/policy/TestFairOrderingPolicy.java | 12 +-
.../scheduler/policy/TestFifoOrderingPolicy.java | 2 +-
.../TestFifoOrderingPolicyForPendingApps.java | 5 +-
...tFifoOrderingPolicyWithExclusivePartitions.java | 244 ++++++
.../security/TestDelegationTokenRenewer.java | 4 +
.../TestRMWebServicesConfigurationMutation.java | 13 +
.../src/test/proto/test_client_tokens.proto | 1 +
.../hadoop-yarn-server-tests/pom.xml | 31 +-
.../src/test/proto/test_token.proto | 1 +
.../src/site/markdown/NodeManager.md | 8 +-
.../src/site/markdown/ResourceManagerRest.md | 81 ++
pom.ozone.xml | 29 +-
520 files changed, 18064 insertions(+), 4052 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org