You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/08/31 16:11:12 UTC
[46/47] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53201734
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53201734
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53201734
Branch: refs/heads/HDFS-12943
Commit: 53201734f5d888b892118a3f8d873ac01c209de4
Parents: 191faeb 8aa6c4f
Author: Erik Krogen <xk...@apache.org>
Authored: Fri Aug 31 09:06:54 2018 -0700
Committer: Erik Krogen <xk...@apache.org>
Committed: Fri Aug 31 09:06:54 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 6 +-
dev-support/bin/ozone-dist-tar-stitching | 9 +-
.../hadoop/fs/FileSystemMultipartUploader.java | 6 +-
.../org/apache/hadoop/fs/MultipartUploader.java | 11 +
.../apache/hadoop/fs/TrashPolicyDefault.java | 14 +
.../src/site/markdown/Compatibility.md | 2 +-
.../site/markdown/InterfaceClassification.md | 2 +-
.../java/org/apache/hadoop/fs/TestTrash.java | 54 +
.../AbstractContractMultipartUploaderTest.java | 43 +
.../crypto/key/kms/server/KMSConfiguration.java | 31 +
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 38 +-
.../crypto/key/kms/server/KMSWebServer.java | 1 +
.../apache/hadoop/hdds/scm/XceiverClient.java | 6 +-
.../hadoop/hdds/scm/XceiverClientGrpc.java | 8 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 8 +-
.../scm/client/ContainerOperationClient.java | 9 +
.../hdds/scm/storage/ChunkInputStream.java | 7 +-
.../hdds/scm/storage/ChunkOutputStream.java | 44 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 6 +
.../org/apache/hadoop/hdds/client/BlockID.java | 5 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 -
.../hadoop/hdds/scm/XceiverClientSpi.java | 2 -
.../hadoop/hdds/scm/client/ScmClient.java | 9 +-
.../common/helpers/AllocatedBlock.java | 4 +-
.../container/common/helpers/ContainerInfo.java | 12 +-
.../common/helpers/ContainerWithPipeline.java | 7 +-
.../scm/container/common/helpers/Pipeline.java | 11 +-
.../StorageContainerLocationProtocol.java | 6 +-
...rLocationProtocolClientSideTranslatorPB.java | 21 +-
.../scm/storage/ContainerProtocolCalls.java | 6 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 7 -
.../org/apache/hadoop/ozone/OzoneConsts.java | 8 +-
.../apache/hadoop/ozone/audit/AuditLogger.java | 66 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 64 +
.../apache/hadoop/ozone/audit/package-info.java | 19 +-
.../ozone/container/common/helpers/KeyData.java | 8 +-
.../apache/hadoop/utils/HddsVersionInfo.java | 6 +-
.../hadoop/utils/db/DBConfigFromFile.java | 134 +++
.../org/apache/hadoop/utils/db/DBProfile.java | 120 ++
.../apache/hadoop/utils/db/DBStoreBuilder.java | 201 ++++
.../org/apache/hadoop/utils/db/RDBStore.java | 32 +-
.../org/apache/hadoop/utils/db/TableConfig.java | 93 ++
.../common/src/main/resources/ozone-default.xml | 40 +-
.../ozone/audit/TestOzoneAuditLogger.java | 124 +-
.../apache/hadoop/utils/TestMetadataStore.java | 1 -
.../hadoop/utils/db/TestDBConfigFromFile.java | 116 ++
.../hadoop/utils/db/TestDBStoreBuilder.java | 174 +++
.../apache/hadoop/utils/db/TestRDBStore.java | 17 +-
.../hadoop/utils/db/TestRDBTableStore.java | 11 +-
.../common/src/test/resources/test.db.ini | 145 +++
.../hadoop/ozone/HddsDatanodeService.java | 3 +-
.../common/helpers/ContainerUtils.java | 22 +-
.../container/common/impl/ContainerData.java | 24 +-
.../common/impl/ContainerDataYaml.java | 5 +-
.../container/common/impl/ContainerSet.java | 2 +-
.../container/common/impl/HddsDispatcher.java | 6 +-
.../common/impl/OpenContainerBlockMap.java | 19 +-
.../transport/server/GrpcXceiverService.java | 8 +-
.../transport/server/XceiverServerGrpc.java | 2 +-
.../transport/server/ratis/CSMMetrics.java | 115 ++
.../server/ratis/ContainerStateMachine.java | 33 +
.../server/ratis/XceiverServerRatis.java | 6 +-
.../container/keyvalue/KeyValueContainer.java | 2 +-
.../keyvalue/KeyValueContainerData.java | 10 +-
.../container/keyvalue/KeyValueHandler.java | 15 +-
.../keyvalue/interfaces/KeyManager.java | 4 +-
.../container/ozoneimpl/OzoneContainer.java | 11 +-
.../ozone/protocol/commands/CommandStatus.java | 16 +-
.../ozone/container/common/ScmTestMock.java | 6 +-
.../common/TestKeyValueContainerData.java | 5 +-
.../common/impl/TestContainerDataYaml.java | 7 +-
.../container/common/impl/TestContainerSet.java | 7 +-
.../common/impl/TestHddsDispatcher.java | 3 +-
.../common/interfaces/TestHandler.java | 7 -
.../endpoint/TestHeartbeatEndpointTask.java | 2 -
.../TestRoundRobinVolumeChoosingPolicy.java | 5 +-
.../keyvalue/TestChunkManagerImpl.java | 4 +-
.../container/keyvalue/TestKeyManagerImpl.java | 4 +-
.../keyvalue/TestKeyValueBlockIterator.java | 4 +-
.../keyvalue/TestKeyValueContainer.java | 13 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 8 +-
.../test/resources/additionalfields.container | 4 +-
.../test/resources/incorrect.checksum.container | 2 +-
.../src/test/resources/incorrect.container | 2 +-
.../hadoop/hdds/server/events/EventWatcher.java | 6 +-
.../hdds/server/events/TestEventQueue.java | 3 -
hadoop-hdds/pom.xml | 1 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 18 +-
.../hdds/scm/block/DeletedBlockLogImpl.java | 3 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 4 +-
.../container/CloseContainerEventHandler.java | 4 +-
.../hdds/scm/container/ContainerMapping.java | 4 +-
.../scm/container/ContainerReportHandler.java | 4 +
.../scm/container/ContainerStateManager.java | 7 +-
.../replication/ReplicationManager.java | 2 +-
.../scm/container/states/ContainerStateMap.java | 2 +-
.../hdds/scm/node/states/Node2ContainerMap.java | 4 +-
.../scm/node/states/NodeNotFoundException.java | 2 -
.../hdds/scm/node/states/ReportResult.java | 3 +-
.../hdds/scm/pipelines/Node2PipelineMap.java | 50 +-
.../hdds/scm/pipelines/PipelineManager.java | 6 +-
.../hdds/scm/pipelines/PipelineSelector.java | 7 +-
.../scm/server/SCMClientProtocolServer.java | 3 +-
.../org/apache/hadoop/hdds/scm/TestUtils.java | 8 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 1 -
.../hdds/scm/block/TestDeletedBlockLog.java | 7 +-
.../command/TestCommandStatusReportHandler.java | 22 +-
.../TestCloseContainerEventHandler.java | 1 -
.../scm/container/TestContainerMapping.java | 7 +-
.../container/TestContainerReportHandler.java | 42 +-
.../TestSCMContainerPlacementCapacity.java | 8 +-
.../TestSCMContainerPlacementRandom.java | 4 +-
.../replication/TestReplicationManager.java | 11 +-
.../replication/TestReplicationQueue.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 5 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 3 +-
.../hdds/scm/node/TestNodeReportHandler.java | 3 +-
.../ozone/container/common/TestEndPoint.java | 9 +-
.../placement/TestContainerPlacement.java | 6 +-
.../main/native/fuse-dfs/test/TestFuseDFS.java | 6 +-
.../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 7 +-
.../hadoop/hdfs/nfs/nfs3/AsyncDataService.java | 6 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 4 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 13 +-
.../nfs/nfs3/PrivilegedNfsGatewayStarter.java | 7 +-
.../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 6 +-
.../hadoop/hdfs/nfs/nfs3/WriteManager.java | 6 +-
.../org/apache/hadoop/hdfs/nfs/TestMountd.java | 6 +-
.../hadoop/hdfs/nfs/TestOutOfOrderWrite.java | 9 +-
.../metrics/FederationRPCMetrics.java | 13 +-
.../FederationRPCPerformanceMonitor.java | 15 +-
.../federation/router/ConnectionManager.java | 18 +-
.../server/federation/router/RBFConfigKeys.java | 5 +
.../router/RouterPermissionChecker.java | 7 +-
.../server/federation/store/RecordStore.java | 6 +-
.../hdfs/tools/federation/RouterAdmin.java | 98 +-
.../src/main/resources/hdfs-rbf-default.xml | 8 +
.../federation/router/TestRouterAdminCLI.java | 82 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../hdfs/server/blockmanagement/BlockInfo.java | 2 +-
.../server/blockmanagement/BlockManager.java | 4 +
.../datanode/fsdataset/impl/FsDatasetImpl.java | 3 +
.../hdfs/server/namenode/FSNamesystem.java | 13 +-
.../hdfs/server/namenode/NamenodeFsck.java | 5 +-
.../server/namenode/ReencryptionHandler.java | 6 +-
.../server/namenode/ReencryptionUpdater.java | 52 +-
.../src/main/resources/hdfs-default.xml | 10 +
.../hadoop/hdfs/TestDistributedFileSystem.java | 5 +-
.../namenode/TestLargeDirectoryDelete.java | 2 +-
.../lib/output/PathOutputCommitter.java | 12 +-
.../hadoop/mapreduce/task/reduce/Shuffle.java | 24 +-
.../task/reduce/ShuffleClientMetrics.java | 43 +-
.../task/reduce/TestShuffleClientMetrics.java | 75 ++
.../dev-support/bin/robot-all.sh | 2 +-
.../dev-support/bin/robot-dnd-all.sh | 10 +-
.../acceptance-test/dev-support/bin/robot.sh | 7 +-
hadoop-ozone/acceptance-test/pom.xml | 7 +-
.../src/test/acceptance/basic/ozone-shell.robot | 1 -
.../src/test/acceptance/commonlib.robot | 2 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 7 +-
.../ozone/client/io/ChunkGroupOutputStream.java | 207 +++-
hadoop-ozone/common/pom.xml | 5 +-
.../hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
hadoop-ozone/docs/content/GettingStarted.md | 12 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 4 -
.../apache/hadoop/ozone/MiniOzoneCluster.java | 3 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 13 +-
.../apache/hadoop/ozone/RatisTestHelper.java | 16 +-
.../TestStorageContainerManagerHelper.java | 2 -
.../ozone/client/rest/TestOzoneRestClient.java | 7 +-
.../rpc/TestCloseContainerHandlingByClient.java | 405 +++++++
.../ozone/client/rpc/TestOzoneRpcClient.java | 9 +-
.../ozone/container/ContainerTestHelper.java | 37 +-
.../common/TestBlockDeletingService.java | 2 +-
.../TestContainerDeletionChoosingPolicy.java | 16 +-
.../common/impl/TestContainerPersistence.java | 120 +-
.../commandhandler/TestBlockDeletion.java | 17 +-
.../TestCloseContainerByPipeline.java | 35 +-
.../transport/server/ratis/TestCSMMetrics.java | 202 ++++
.../container/ozoneimpl/TestOzoneContainer.java | 53 +-
.../ozoneimpl/TestOzoneContainerRatis.java | 5 +-
.../container/ozoneimpl/TestRatisManager.java | 6 +-
.../hadoop/ozone/freon/TestDataValidate.java | 7 +-
.../apache/hadoop/ozone/freon/TestFreon.java | 3 +-
.../ozone/om/TestContainerReportWithKeys.java | 12 +-
.../om/TestMultipleContainerReadWrite.java | 5 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 23 +-
.../apache/hadoop/ozone/om/TestOmMetrics.java | 7 +-
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 6 +-
.../hadoop/ozone/om/TestOzoneManager.java | 5 +-
.../hadoop/ozone/ozShell/TestOzoneShell.java | 20 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 -
.../ozone/scm/TestXceiverClientManager.java | 25 -
.../ozone/web/TestDistributedOzoneVolumes.java | 188 ---
.../hadoop/ozone/web/TestLocalOzoneVolumes.java | 187 ---
.../hadoop/ozone/web/TestOzoneVolumes.java | 183 +++
.../hadoop/ozone/web/TestOzoneWebAccess.java | 11 +-
.../hadoop/ozone/web/client/TestBuckets.java | 14 +-
.../ozone/web/client/TestBucketsRatis.java | 2 +-
.../hadoop/ozone/web/client/TestKeys.java | 27 +-
.../hadoop/ozone/web/client/TestKeysRatis.java | 6 +-
.../ozone/web/client/TestOzoneClient.java | 3 -
.../hadoop/ozone/web/client/TestVolume.java | 11 +-
.../ozone/web/client/TestVolumeRatis.java | 3 -
.../server/datanode/ObjectStoreHandler.java | 182 ++-
.../web/handlers/StorageHandlerBuilder.java | 18 +-
.../web/localstorage/LocalStorageHandler.java | 385 ------
.../web/localstorage/OzoneMetadataManager.java | 1138 ------------------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 13 -
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 6 +-
hadoop-ozone/pom.xml | 5 +
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 -
hadoop-project/pom.xml | 2 +-
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 15 +-
.../hadoop/fs/s3a/S3AMultipartUploader.java | 10 +-
.../fs/s3a/commit/AbstractS3ACommitter.java | 16 +-
.../fs/s3a/commit/S3ACommitterFactory.java | 18 +-
.../s3a/commit/magic/MagicS3GuardCommitter.java | 7 +
.../staging/DirectoryStagingCommitter.java | 8 +-
.../staging/PartitionedStagingCommitter.java | 9 +-
.../hadoop/fs/s3a/commit/staging/Paths.java | 14 +-
.../fs/s3a/commit/staging/StagingCommitter.java | 50 +-
.../tools/hadoop-aws/committer_architecture.md | 94 +-
.../markdown/tools/hadoop-aws/committers.md | 2 +-
.../fs/s3a/commit/AbstractCommitITest.java | 19 +
.../fs/s3a/commit/AbstractITCommitMRJob.java | 5 +-
.../fs/s3a/commit/AbstractITCommitProtocol.java | 63 +-
.../fs/s3a/commit/ITestS3ACommitterFactory.java | 200 +++
.../fs/s3a/commit/magic/ITMagicCommitMRJob.java | 6 +-
.../commit/magic/ITestMagicCommitProtocol.java | 25 +-
.../ITStagingCommitMRJobBadDest.java | 62 +
.../integration/ITestStagingCommitProtocol.java | 13 +
...estNativeAzureFileSystemConcurrencyLive.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../hadoop/yarn/service/ServiceScheduler.java | 100 +-
.../service/api/records/ComponentState.java | 2 +-
.../service/api/records/ContainerState.java | 3 +-
.../yarn/service/api/records/ServiceState.java | 2 +-
.../component/instance/ComponentInstance.java | 144 ++-
.../timelineservice/ServiceTimelineEvent.java | 5 +-
.../ServiceTimelinePublisher.java | 33 +-
.../yarn/service/MockRunningServiceContext.java | 18 +-
.../hadoop/yarn/service/ServiceTestUtils.java | 9 +-
.../yarn/service/component/TestComponent.java | 55 +-
.../component/TestComponentRestartPolicy.java | 1 -
.../instance/TestComponentInstance.java | 35 +-
.../TestServiceTimelinePublisher.java | 4 +-
.../src/main/resources/yarn-default.xml | 7 +
.../yarn/server/AMHeartbeatRequestHandler.java | 227 ++++
.../LocalityMulticastAMRMProxyPolicy.java | 105 +-
.../server/uam/UnmanagedApplicationManager.java | 170 +--
.../TestLocalityMulticastAMRMProxyPolicy.java | 53 +-
.../amrmproxy/FederationInterceptor.java | 245 ++--
.../runtime/DockerLinuxContainerRuntime.java | 87 +-
.../linux/runtime/docker/DockerRunCommand.java | 5 +
.../container-executor/impl/utils/docker-util.c | 42 +
.../container-executor/impl/utils/docker-util.h | 3 +-
.../test/utils/test_docker_util.cc | 64 +
.../containermanager/TestContainerManager.java | 18 +-
.../runtime/TestDockerContainerRuntime.java | 149 +++
.../runtime/docker/TestDockerRunCommand.java | 5 +-
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../src/site/markdown/DockerContainers.md | 1 +
.../site/markdown/OpportunisticContainers.md.vm | 2 +-
266 files changed, 5288 insertions(+), 3716 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53201734/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53201734/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53201734/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org