You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/07/17 18:35:57 UTC

[31/32] hadoop git commit: Merge branch 'trunk' into HDFS-12943

Merge branch 'trunk' into HDFS-12943


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f9c3ca8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f9c3ca8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f9c3ca8

Branch: refs/heads/HDFS-12943
Commit: 5f9c3ca8ffa15d5845f4e0f04bf007ce99e399c0
Parents: 9e9e1e3 1af87df
Author: Erik Krogen <xk...@apache.org>
Authored: Tue Jul 17 11:33:15 2018 -0700
Committer: Erik Krogen <xk...@apache.org>
Committed: Tue Jul 17 11:33:15 2018 -0700

----------------------------------------------------------------------
 .../hadoop-client-minicluster/pom.xml           |   4 +
 hadoop-common-project/hadoop-common/pom.xml     |   5 +
 .../hadoop/conf/ReconfigurationServlet.java     |   2 +-
 .../crypto/key/kms/KMSClientProvider.java       |  11 +-
 .../org/apache/hadoop/security/token/Token.java |   5 +
 .../org/apache/hadoop/util/NativeCrc32.java     |   4 +-
 .../apache/hadoop/security/token/TestToken.java |  18 ++
 .../apache/hadoop/test/GenericTestUtils.java    |   2 +-
 .../org/apache/hadoop/hdds/HddsConfigKeys.java  |   8 +
 .../org/apache/hadoop/hdds/HddsIdFactory.java   |  53 +++++
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |   5 +-
 .../container/common/helpers/ContainerInfo.java |  11 +
 .../common/src/main/resources/ozone-default.xml |  13 +-
 .../apache/hadoop/utils/TestHddsIdFactory.java  |  77 ++++++
 .../container/common/impl/ContainerSet.java     |  16 +-
 .../container/common/interfaces/Container.java  |   2 +-
 .../ContainerDeletionChoosingPolicy.java        |   1 -
 .../report/CommandStatusReportPublisher.java    |  71 ++++++
 .../common/report/ReportPublisher.java          |   9 +
 .../common/report/ReportPublisherFactory.java   |   4 +
 .../statemachine/DatanodeStateMachine.java      |   2 +
 .../common/statemachine/StateContext.java       |  70 ++++++
 .../CloseContainerCommandHandler.java           |   5 +-
 .../commandhandler/CommandHandler.java          |  11 +
 .../DeleteBlocksCommandHandler.java             | 166 ++++++-------
 .../ReplicateContainerCommandHandler.java       |   7 +-
 .../container/keyvalue/KeyValueHandler.java     |  24 ++
 .../background/BlockDeletingService.java        |  11 +-
 .../commands/CloseContainerCommand.java         |  36 +--
 .../ozone/protocol/commands/CommandStatus.java  | 141 +++++++++++
 .../protocol/commands/DeleteBlocksCommand.java  |  13 +-
 .../commands/ReplicateContainerCommand.java     |  20 +-
 .../protocol/commands/ReregisterCommand.java    |  10 +
 .../ozone/protocol/commands/SCMCommand.java     |  19 ++
 .../StorageContainerDatanodeProtocol.proto      |  21 ++
 .../ozone/container/common/ScmTestMock.java     |  33 ++-
 .../common/report/TestReportPublisher.java      |  77 +++++-
 hadoop-hdds/pom.xml                             |  49 ++++
 .../hadoop/hdds/scm/block/BlockManagerImpl.java |  10 +-
 .../hdds/scm/block/SCMBlockDeletingService.java |  13 +-
 .../hdds/scm/container/ContainerMapping.java    |  11 +-
 .../scm/container/ContainerStateManager.java    |  40 ++++
 .../scm/container/states/ContainerStateMap.java | 122 +++++++++-
 .../hadoop/hdds/scm/events/SCMEvents.java       |  57 +++--
 .../hadoop/hdds/scm/node/NodeManager.java       |   9 +
 .../hadoop/hdds/scm/node/NodeReportHandler.java |  19 +-
 .../hadoop/hdds/scm/node/SCMNodeManager.java    |  11 +
 .../hdds/scm/pipelines/Node2PipelineMap.java    | 121 ++++++++++
 .../hdds/scm/pipelines/PipelineManager.java     |  22 +-
 .../hdds/scm/pipelines/PipelineSelector.java    |  24 +-
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  11 +-
 .../standalone/StandaloneManagerImpl.java       |   7 +-
 .../server/SCMDatanodeHeartbeatDispatcher.java  |  23 +-
 .../scm/server/StorageContainerManager.java     |   2 +-
 .../hadoop/hdds/scm/block/TestBlockManager.java |   2 +-
 .../hdds/scm/container/MockNodeManager.java     |  11 +
 .../hdds/scm/node/TestNodeReportHandler.java    |  95 ++++++++
 .../TestSCMDatanodeHeartbeatDispatcher.java     |  25 +-
 .../ozone/container/common/TestEndPoint.java    | 111 ++++++++-
 .../testutils/ReplicationNodeManagerMock.java   |  10 +
 .../hdfs/server/federation/router/Router.java   |   7 +
 .../federation/router/RouterAdminServer.java    |  32 ++-
 .../federation/router/RouterRpcServer.java      |  26 +--
 .../router/RouterSafemodeService.java           |  44 +++-
 .../src/site/markdown/HDFSRouterFederation.md   |  12 +
 .../federation/router/TestRouterAdminCLI.java   |   7 +-
 .../federation/router/TestRouterSafemode.java   |  58 ++++-
 .../qjournal/server/GetJournalEditServlet.java  |   2 +-
 .../server/datanode/BlockRecoveryWorker.java    |   4 +-
 .../server/diskbalancer/command/Command.java    |   6 +-
 .../diskbalancer/command/PlanCommand.java       |   4 +-
 .../diskbalancer/command/ReportCommand.java     |  10 +-
 .../server/namenode/AclEntryStatusFormat.java   |   6 +-
 .../hdfs/server/namenode/FSNamesystem.java      |   2 +-
 .../namenode/INodeWithAdditionalFields.java     |   4 +-
 .../server/namenode/ReencryptionHandler.java    |   4 +-
 .../hdfs/server/namenode/XAttrFormat.java       |  67 ++++--
 .../apache/hadoop/hdfs/tools/CacheAdmin.java    |   2 +-
 .../apache/hadoop/hdfs/TestDecommission.java    |   4 +-
 .../org/apache/hadoop/hdfs/TestLargeBlock.java  |   6 +-
 .../hdfs/server/namenode/TestReencryption.java  |  61 ++---
 .../apache/hadoop/hdfs/tools/TestDFSAdmin.java  |   4 +-
 .../mapreduce/v2/app/webapp/TaskPage.java       |   2 +-
 .../mapreduce/v2/app/webapp/TasksBlock.java     |   2 +-
 .../mapreduce/v2/hs/webapp/HsJobsBlock.java     |   2 +-
 .../mapreduce/v2/hs/webapp/HsTaskPage.java      |   2 +-
 .../container/TestContainerStateManager.java    |  79 +++++++
 .../hdds/scm/pipeline/TestNode2PipelineMap.java | 117 ++++++++++
 .../hadoop/ozone/MiniOzoneClusterImpl.java      |   2 +-
 .../common/TestBlockDeletingService.java        |  26 ++-
 .../TestContainerDeletionChoosingPolicy.java    |  62 ++---
 .../commandhandler/TestBlockDeletion.java       |  14 +-
 .../TestCloseContainerByPipeline.java           |  18 +-
 .../TestCloseContainerHandler.java              |  12 +-
 .../ozone/om/TestContainerReportWithKeys.java   |  11 +-
 .../hadoop/ozone/ozShell/TestOzoneShell.java    |  12 +
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |   3 +-
 .../hadoop/ozone/web/client/TestKeys.java       |   2 -
 .../ozone/web/ozShell/keys/GetKeyHandler.java   |   9 +-
 hadoop-ozone/pom.xml                            |  49 ++++
 hadoop-project/pom.xml                          |   5 +
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   |   3 +-
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |   6 +
 .../yarn/sls/scheduler/RMNodeWrapper.java       |   6 +
 .../hadoop/yarn/api/records/Resource.java       |  11 -
 .../api/records/impl/LightWeightResource.java   |  16 --
 .../yarn/service/client/ApiServiceClient.java   |  74 ++++--
 .../hadoop/yarn/service/webapp/ApiServer.java   |  67 ++++--
 .../hadoop/yarn/service/ClientAMProtocol.java   |   5 +
 .../hadoop/yarn/service/ClientAMService.java    |  14 ++
 .../yarn/service/client/ServiceClient.java      |  47 ++++
 .../component/instance/ComponentInstance.java   |  41 +++-
 .../yarn/service/conf/RestApiConstants.java     |   5 +-
 .../pb/client/ClientAMProtocolPBClientImpl.java |  13 ++
 .../service/ClientAMProtocolPBServiceImpl.java  |  13 ++
 .../hadoop/yarn/service/utils/FilterUtils.java  |  81 +++++++
 .../yarn/service/utils/ServiceApiUtil.java      |   9 +
 .../src/main/proto/ClientAMProtocol.proto       |  12 +
 .../yarn/service/MockRunningServiceContext.java | 154 ++++++++++++
 .../yarn/service/client/TestServiceCLI.java     |  25 +-
 .../yarn/service/client/TestServiceClient.java  |  54 ++++-
 .../yarn/service/component/TestComponent.java   | 133 +----------
 .../instance/TestComponentInstance.java         |  46 ++--
 .../yarn/service/utils/TestFilterUtils.java     | 102 ++++++++
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  68 +++++-
 .../apache/hadoop/yarn/client/cli/TopCLI.java   |   3 +-
 .../hadoop/yarn/client/cli/TestYarnCLI.java     |   6 +-
 .../hadoop/yarn/client/api/AppAdminClient.java  |   6 +
 .../org/apache/hadoop/yarn/state/Graph.java     |   2 +-
 .../hadoop/yarn/util/resource/Resources.java    |  23 +-
 .../hadoop/yarn/webapp/hamlet/HamletImpl.java   |   2 +-
 .../hadoop/yarn/webapp/hamlet2/HamletImpl.java  |   2 +-
 .../hadoop/yarn/webapp/view/JQueryUI.java       |   2 +-
 .../hadoop/yarn/webapp/view/TextView.java       |   2 +-
 .../yarn/util/resource/TestResources.java       |  36 +++
 .../yarn/server/webapp/AppAttemptBlock.java     |   2 +-
 .../hadoop/yarn/server/webapp/AppBlock.java     |   2 +-
 .../hadoop/yarn/server/webapp/AppsBlock.java    |   2 +-
 .../container-executor/impl/configuration.c     |   3 +
 .../main/native/container-executor/impl/main.c  |  10 +-
 .../impl/modules/cgroups/cgroups-operations.c   |   2 +
 .../container-executor/impl/utils/docker-util.c | 234 ++++++++++---------
 .../test/modules/cgroups/test-cgroups-module.cc |   8 +
 .../test/modules/fpga/test-fpga-module.cc       |  24 +-
 .../test/modules/gpu/test-gpu-module.cc         |  24 +-
 .../test/test-container-executor.c              |  14 +-
 .../test/test_configuration.cc                  |  34 ++-
 .../native/container-executor/test/test_util.cc |   5 +
 .../test/utils/test-string-utils.cc             |   6 +
 .../test/utils/test_docker_util.cc              | 128 ++++++----
 .../placement/AppNameMappingPlacementRule.java  |  18 +-
 .../rmcontainer/RMContainerImpl.java            |   5 -
 .../server/resourcemanager/rmnode/RMNode.java   |   6 +
 .../resourcemanager/rmnode/RMNodeImpl.java      |   5 +
 .../scheduler/AppSchedulingInfo.java            |   4 +-
 .../scheduler/SchedulerNode.java                |  15 ++
 .../scheduler/fair/ConfigurableResource.java    |   8 +-
 .../webapp/FairSchedulerAppsBlock.java          |   2 +-
 .../webapp/RMAppAttemptBlock.java               |   2 +-
 .../resourcemanager/webapp/RMAppBlock.java      |   2 +-
 .../resourcemanager/webapp/RMAppsBlock.java     |   2 +-
 .../yarn/server/resourcemanager/MockNodes.java  |   5 +
 .../TestAppNameMappingPlacementRule.java        |  43 ++--
 .../placement/TestPlacementManager.java         |   7 +-
 .../rmcontainer/TestRMContainerImpl.java        |  16 +-
 .../scheduler/TestAbstractYarnScheduler.java    | 104 +++++++++
 .../TestSchedulerApplicationAttempt.java        |  44 ++++
 .../yarn/server/router/webapp/AppsBlock.java    |   4 +-
 .../src/site/markdown/CapacityScheduler.md      |   6 +-
 .../src/site/markdown/Federation.md             |   1 -
 170 files changed, 3603 insertions(+), 876 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9c3ca8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org