You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/04/24 20:14:38 UTC
[83/83] [abbrv] hadoop git commit: Merge branch 'HDFS-7240' into trunk
Merge branch 'HDFS-7240' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d6befb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d6befb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d6befb2
Branch: refs/heads/trunk
Commit: 9d6befb2989a127b5dd7776522a12357b4eb3c61
Parents: 1b9ecc2 9197588
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Apr 24 12:08:43 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Apr 24 12:08:43 2018 -0700
----------------------------------------------------------------------
LICENSE.txt | 68 +
dev-support/bin/dist-layout-stitching | 22 +-
dev-support/docker/Dockerfile | 3 +
.../assemblies/hadoop-src-with-hdsl.xml | 56 +
.../main/resources/assemblies/hadoop-src.xml | 2 +
.../ensure-jars-have-correct-contents.sh | 6 +
.../hadoop-client-minicluster/pom.xml | 7 +
.../hadoop-client-runtime/pom.xml | 1 +
.../src/main/bin/hadoop-functions.sh | 5 +
.../hadoop-common/src/main/conf/hadoop-env.sh | 17 +
.../src/main/conf/log4j.properties | 34 +
.../src/main/resources/core-default.xml | 13 +
.../conf/TestCommonConfigurationFields.java | 3 +
hadoop-dist/pom.xml | 83 ++
hadoop-dist/src/main/compose/ozone/.env | 17 +
.../src/main/compose/ozone/docker-compose.yaml | 61 +
.../src/main/compose/ozone/docker-config | 35 +
hadoop-hdds/client/pom.xml | 49 +
.../apache/hadoop/hdds/scm/XceiverClient.java | 192 +++
.../hadoop/hdds/scm/XceiverClientHandler.java | 202 +++
.../hdds/scm/XceiverClientInitializer.java | 72 +
.../hadoop/hdds/scm/XceiverClientManager.java | 218 +++
.../hadoop/hdds/scm/XceiverClientMetrics.java | 92 ++
.../hadoop/hdds/scm/XceiverClientRatis.java | 266 ++++
.../scm/client/ContainerOperationClient.java | 407 ++++++
.../hadoop/hdds/scm/client/HddsClientUtils.java | 232 ++++
.../hadoop/hdds/scm/client/package-info.java | 23 +
.../apache/hadoop/hdds/scm/package-info.java | 23 +
.../hdds/scm/storage/ChunkInputStream.java | 261 ++++
.../hdds/scm/storage/ChunkOutputStream.java | 227 +++
.../hadoop/hdds/scm/storage/package-info.java | 23 +
.../common/dev-support/findbugsExcludeFile.xml | 21 +
hadoop-hdds/common/pom.xml | 128 ++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 23 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 318 +++++
.../apache/hadoop/hdds/client/OzoneQuota.java | 203 +++
.../hadoop/hdds/client/ReplicationFactor.java | 63 +
.../hadoop/hdds/client/ReplicationType.java | 28 +
.../apache/hadoop/hdds/client/package-info.java | 23 +
.../hadoop/hdds/conf/HddsConfServlet.java | 182 +++
.../hadoop/hdds/conf/OzoneConfiguration.java | 162 +++
.../apache/hadoop/hdds/conf/package-info.java | 18 +
.../org/apache/hadoop/hdds/package-info.java | 23 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 353 +++++
.../hadoop/hdds/protocol/package-info.java | 22 +
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 271 ++++
.../org/apache/hadoop/hdds/scm/ScmInfo.java | 81 ++
.../hadoop/hdds/scm/XceiverClientSpi.java | 129 ++
.../hadoop/hdds/scm/client/ScmClient.java | 139 ++
.../hadoop/hdds/scm/client/package-info.java | 24 +
.../hadoop/hdds/scm/container/ContainerID.java | 97 ++
.../common/helpers/AllocatedBlock.java | 77 ++
.../container/common/helpers/ContainerInfo.java | 333 +++++
.../common/helpers/DeleteBlockResult.java | 51 +
.../scm/container/common/helpers/Pipeline.java | 253 ++++
.../common/helpers/PipelineChannel.java | 122 ++
.../helpers/StorageContainerException.java | 104 ++
.../container/common/helpers/package-info.java | 22 +
.../hadoop/hdds/scm/container/package-info.java | 18 +
.../apache/hadoop/hdds/scm/package-info.java | 24 +
.../hdds/scm/protocol/LocatedContainer.java | 127 ++
.../scm/protocol/ScmBlockLocationProtocol.java | 72 +
.../hdds/scm/protocol/ScmLocatedBlock.java | 100 ++
.../StorageContainerLocationProtocol.java | 124 ++
.../hadoop/hdds/scm/protocol/package-info.java | 19 +
...kLocationProtocolClientSideTranslatorPB.java | 215 +++
.../protocolPB/ScmBlockLocationProtocolPB.java | 35 +
...rLocationProtocolClientSideTranslatorPB.java | 316 +++++
.../StorageContainerLocationProtocolPB.java | 36 +
.../hdds/scm/protocolPB/package-info.java | 24 +
.../scm/storage/ContainerProtocolCalls.java | 396 ++++++
.../hadoop/hdds/scm/storage/package-info.java | 23 +
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 231 ++++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 241 ++++
.../org/apache/hadoop/ozone/OzoneConsts.java | 167 +++
.../apache/hadoop/ozone/common/BlockGroup.java | 87 ++
.../ozone/common/DeleteBlockGroupResult.java | 96 ++
.../InconsistentStorageStateException.java | 51 +
.../org/apache/hadoop/ozone/common/Storage.java | 248 ++++
.../apache/hadoop/ozone/common/StorageInfo.java | 183 +++
.../hadoop/ozone/common/package-info.java | 18 +
.../InvalidStateTransitionException.java | 42 +
.../ozone/common/statemachine/StateMachine.java | 68 +
.../ozone/common/statemachine/package-info.java | 21 +
.../container/common/helpers/ChunkInfo.java | 185 +++
.../ozone/container/common/helpers/KeyData.java | 170 +++
.../container/common/helpers/package-info.java | 23 +
.../org/apache/hadoop/ozone/lease/Lease.java | 189 +++
.../ozone/lease/LeaseAlreadyExistException.java | 46 +
.../ozone/lease/LeaseCallbackExecutor.java | 65 +
.../hadoop/ozone/lease/LeaseException.java | 45 +
.../ozone/lease/LeaseExpiredException.java | 45 +
.../apache/hadoop/ozone/lease/LeaseManager.java | 247 ++++
.../lease/LeaseManagerNotRunningException.java | 45 +
.../ozone/lease/LeaseNotFoundException.java | 46 +
.../apache/hadoop/ozone/lease/package-info.java | 26 +
.../org/apache/hadoop/ozone/package-info.java | 35 +
...kLocationProtocolServerSideTranslatorPB.java | 170 +++
...rLocationProtocolServerSideTranslatorPB.java | 212 +++
.../hadoop/ozone/protocolPB/package-info.java | 24 +
.../hadoop/ozone/web/utils/JsonUtils.java | 71 +
.../hadoop/ozone/web/utils/package-info.java | 19 +
.../apache/hadoop/utils/BackgroundService.java | 161 +++
.../org/apache/hadoop/utils/BackgroundTask.java | 28 +
.../hadoop/utils/BackgroundTaskQueue.java | 64 +
.../hadoop/utils/BackgroundTaskResult.java | 44 +
.../org/apache/hadoop/utils/BatchOperation.java | 90 ++
.../org/apache/hadoop/utils/EntryConsumer.java | 38 +
.../org/apache/hadoop/utils/LevelDBStore.java | 380 ++++++
.../apache/hadoop/utils/MetadataKeyFilters.java | 118 ++
.../org/apache/hadoop/utils/MetadataStore.java | 172 +++
.../hadoop/utils/MetadataStoreBuilder.java | 126 ++
.../org/apache/hadoop/utils/RocksDBStore.java | 382 ++++++
.../apache/hadoop/utils/RocksDBStoreMBean.java | 144 ++
.../org/apache/hadoop/utils/package-info.java | 18 +
.../main/java/org/apache/ratis/RatisHelper.java | 129 ++
.../java/org/apache/ratis/package-info.java | 22 +
.../com/google/protobuf/ShadedProtoUtil.java | 38 +
.../com/google/protobuf/package-info.java | 22 +
.../main/proto/DatanodeContainerProtocol.proto | 415 ++++++
.../main/proto/ScmBlockLocationProtocol.proto | 166 +++
.../StorageContainerLocationProtocol.proto | 214 +++
hadoop-hdds/common/src/main/proto/hdds.proto | 170 +++
.../common/src/main/resources/ozone-default.xml | 1049 ++++++++++++++
.../apache/hadoop/hdds/scm/package-info.java | 21 +
.../apache/hadoop/ozone/TestMetadataStore.java | 414 ++++++
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 141 ++
.../hadoop/ozone/common/TestStateMachine.java | 106 ++
.../hadoop/ozone/lease/TestLeaseManager.java | 374 +++++
.../apache/hadoop/ozone/lease/package-info.java | 21 +
.../org/apache/hadoop/ozone/package-info.java | 21 +
.../hadoop/utils/TestRocksDBStoreMBean.java | 87 ++
hadoop-hdds/container-service/pom.xml | 103 ++
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 325 +++++
.../org/apache/hadoop/hdds/scm/VersionInfo.java | 81 ++
.../apache/hadoop/hdds/scm/package-info.java | 19 +
.../hadoop/ozone/HddsDatanodeService.java | 242 ++++
.../container/common/helpers/ChunkUtils.java | 346 +++++
.../container/common/helpers/ContainerData.java | 326 +++++
.../common/helpers/ContainerMetrics.java | 121 ++
.../common/helpers/ContainerReport.java | 218 +++
.../common/helpers/ContainerUtils.java | 442 ++++++
.../helpers/DeletedContainerBlocksSummary.java | 103 ++
.../container/common/helpers/FileUtils.java | 81 ++
.../container/common/helpers/KeyUtils.java | 148 ++
.../container/common/helpers/package-info.java | 22 +
.../container/common/impl/ChunkManagerImpl.java | 235 ++++
.../impl/ContainerLocationManagerImpl.java | 150 ++
.../common/impl/ContainerManagerImpl.java | 1113 +++++++++++++++
.../common/impl/ContainerReportManagerImpl.java | 90 ++
.../container/common/impl/ContainerStatus.java | 217 +++
.../common/impl/ContainerStorageLocation.java | 203 +++
.../ozone/container/common/impl/Dispatcher.java | 713 ++++++++++
.../container/common/impl/KeyManagerImpl.java | 202 +++
.../RandomContainerDeletionChoosingPolicy.java | 70 +
.../common/impl/StorageLocationReport.java | 63 +
...NOrderedContainerDeletionChoosingPolicy.java | 91 ++
.../container/common/impl/package-info.java | 22 +
.../common/interfaces/ChunkManager.java | 76 ++
.../ContainerDeletionChoosingPolicy.java | 46 +
.../common/interfaces/ContainerDispatcher.java | 51 +
.../interfaces/ContainerLocationManager.java | 58 +
.../ContainerLocationManagerMXBean.java | 36 +
.../common/interfaces/ContainerManager.java | 280 ++++
.../interfaces/ContainerReportManager.java | 32 +
.../container/common/interfaces/KeyManager.java | 76 ++
.../common/interfaces/package-info.java | 20 +
.../ozone/container/common/package-info.java | 28 +
.../statemachine/DatanodeStateMachine.java | 397 ++++++
.../statemachine/EndpointStateMachine.java | 294 ++++
.../statemachine/EndpointStateMachineMBean.java | 34 +
.../statemachine/SCMConnectionManager.java | 208 +++
.../SCMConnectionManagerMXBean.java | 27 +
.../common/statemachine/StateContext.java | 285 ++++
.../background/BlockDeletingService.java | 239 ++++
.../statemachine/background/package-info.java | 18 +
.../commandhandler/CloseContainerHandler.java | 112 ++
.../commandhandler/CommandDispatcher.java | 177 +++
.../commandhandler/CommandHandler.java | 59 +
.../commandhandler/ContainerReportHandler.java | 114 ++
.../DeleteBlocksCommandHandler.java | 211 +++
.../commandhandler/package-info.java | 18 +
.../common/statemachine/package-info.java | 28 +
.../container/common/states/DatanodeState.java | 55 +
.../states/datanode/InitDatanodeState.java | 164 +++
.../states/datanode/RunningDatanodeState.java | 175 +++
.../common/states/datanode/package-info.java | 21 +
.../states/endpoint/HeartbeatEndpointTask.java | 267 ++++
.../states/endpoint/RegisterEndpointTask.java | 208 +++
.../states/endpoint/VersionEndpointTask.java | 68 +
.../common/states/endpoint/package-info.java | 20 +
.../container/common/states/package-info.java | 18 +
.../common/transport/server/XceiverServer.java | 130 ++
.../transport/server/XceiverServerHandler.java | 82 ++
.../server/XceiverServerInitializer.java | 62 +
.../transport/server/XceiverServerSpi.java | 43 +
.../common/transport/server/package-info.java | 24 +
.../server/ratis/ContainerStateMachine.java | 293 ++++
.../server/ratis/XceiverServerRatis.java | 214 +++
.../transport/server/ratis/package-info.java | 23 +
.../container/common/utils/ContainerCache.java | 168 +++
.../container/common/utils/package-info.java | 18 +
.../container/ozoneimpl/OzoneContainer.java | 277 ++++
.../ozone/container/ozoneimpl/package-info.java | 21 +
.../org/apache/hadoop/ozone/package-info.java | 23 +
.../StorageContainerDatanodeProtocol.java | 96 ++
.../protocol/StorageContainerNodeProtocol.java | 68 +
.../hadoop/ozone/protocol/VersionResponse.java | 150 ++
.../commands/CloseContainerCommand.java | 76 ++
.../protocol/commands/DeleteBlocksCommand.java | 66 +
.../protocol/commands/RegisteredCommand.java | 229 ++++
.../protocol/commands/ReregisterCommand.java | 59 +
.../ozone/protocol/commands/SCMCommand.java | 41 +
.../protocol/commands/SendContainerCommand.java | 80 ++
.../ozone/protocol/commands/package-info.java | 21 +
.../hadoop/ozone/protocol/package-info.java | 23 +
...rDatanodeProtocolClientSideTranslatorPB.java | 204 +++
.../StorageContainerDatanodeProtocolPB.java | 34 +
...rDatanodeProtocolServerSideTranslatorPB.java | 119 ++
.../hadoop/ozone/protocolPB/package-info.java | 19 +
.../StorageContainerDatanodeProtocol.proto | 353 +++++
...m.sun.jersey.spi.container.ContainerProvider | 16 +
.../container/common/ContainerTestUtils.java | 68 +
.../ozone/container/common/SCMTestUtils.java | 120 ++
.../ozone/container/common/ScmTestMock.java | 274 ++++
.../common/TestDatanodeStateMachine.java | 379 +++++
.../testutils/BlockDeletingServiceTestImpl.java | 104 ++
.../ozone/container/testutils/package-info.java | 18 +
.../src/test/resources/log4j.properties | 23 +
hadoop-hdds/framework/README.md | 24 +
hadoop-hdds/framework/pom.xml | 70 +
.../hadoop/hdds/server/BaseHttpServer.java | 222 +++
.../apache/hadoop/hdds/server/ServerUtils.java | 139 ++
.../hadoop/hdds/server/ServiceRuntimeInfo.java | 64 +
.../hdds/server/ServiceRuntimeInfoImpl.java | 55 +
.../apache/hadoop/hdds/server/package-info.java | 23 +
.../src/main/resources/webapps/datanode/dn.js | 92 ++
.../webapps/static/angular-1.6.4.min.js | 332 +++++
.../webapps/static/angular-nvd3-1.0.9.min.js | 1 +
.../webapps/static/angular-route-1.6.4.min.js | 17 +
.../resources/webapps/static/d3-3.5.17.min.js | 5 +
.../main/resources/webapps/static/dfs-dust.js | 133 ++
.../resources/webapps/static/nvd3-1.8.5.min.css | 2 +
.../webapps/static/nvd3-1.8.5.min.css.map | 1 +
.../resources/webapps/static/nvd3-1.8.5.min.js | 11 +
.../webapps/static/nvd3-1.8.5.min.js.map | 1 +
.../src/main/resources/webapps/static/ozone.css | 60 +
.../src/main/resources/webapps/static/ozone.js | 387 ++++++
.../webapps/static/templates/config.html | 91 ++
.../resources/webapps/static/templates/jvm.html | 26 +
.../webapps/static/templates/menu.html | 60 +
.../webapps/static/templates/overview.html | 39 +
.../webapps/static/templates/rpc-metrics.html | 87 ++
.../hadoop/hdds/server/TestBaseHttpServer.java | 98 ++
.../framework/src/test/resources/ozone-site.xml | 24 +
hadoop-hdds/pom.xml | 117 ++
hadoop-hdds/server-scm/pom.xml | 159 +++
.../org/apache/hadoop/hdds/scm/SCMMXBean.java | 50 +
.../org/apache/hadoop/hdds/scm/SCMStorage.java | 73 +
.../hdds/scm/StorageContainerManager.java | 1290 ++++++++++++++++++
.../scm/StorageContainerManagerHttpServer.java | 76 ++
.../hadoop/hdds/scm/block/BlockManager.java | 86 ++
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 530 +++++++
.../hdds/scm/block/BlockmanagerMXBean.java | 30 +
.../block/DatanodeDeletedBlockTransactions.java | 132 ++
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 124 ++
.../hdds/scm/block/DeletedBlockLogImpl.java | 356 +++++
.../hdds/scm/block/SCMBlockDeletingService.java | 175 +++
.../hadoop/hdds/scm/block/package-info.java | 22 +
.../hdds/scm/container/ContainerMapping.java | 607 ++++++++
.../scm/container/ContainerStateManager.java | 456 +++++++
.../hadoop/hdds/scm/container/Mapping.java | 106 ++
.../scm/container/closer/ContainerCloser.java | 192 +++
.../hdds/scm/container/closer/package-info.java | 23 +
.../hadoop/hdds/scm/container/package-info.java | 22 +
.../algorithms/ContainerPlacementPolicy.java | 41 +
.../placement/algorithms/SCMCommonPolicy.java | 197 +++
.../SCMContainerPlacementCapacity.java | 133 ++
.../algorithms/SCMContainerPlacementRandom.java | 89 ++
.../placement/algorithms/package-info.java | 18 +
.../placement/metrics/ContainerStat.java | 165 +++
.../placement/metrics/DatanodeMetric.java | 91 ++
.../container/placement/metrics/LongMetric.java | 162 +++
.../container/placement/metrics/NodeStat.java | 67 +
.../container/placement/metrics/SCMMetrics.java | 154 +++
.../placement/metrics/SCMNodeMetric.java | 223 +++
.../placement/metrics/SCMNodeStat.java | 139 ++
.../placement/metrics/package-info.java | 20 +
.../scm/container/placement/package-info.java | 19 +
.../replication/ContainerSupervisor.java | 343 +++++
.../container/replication/InProgressPool.java | 313 +++++
.../scm/container/replication/PeriodicPool.java | 119 ++
.../scm/container/replication/package-info.java | 23 +
.../container/states/ContainerAttribute.java | 245 ++++
.../scm/container/states/ContainerState.java | 96 ++
.../scm/container/states/ContainerStateMap.java | 405 ++++++
.../hdds/scm/container/states/package-info.java | 22 +
.../hdds/scm/exceptions/SCMException.java | 119 ++
.../hdds/scm/exceptions/package-info.java | 18 +
.../hadoop/hdds/scm/node/CommandQueue.java | 190 +++
.../hdds/scm/node/HeartbeatQueueItem.java | 117 ++
.../hadoop/hdds/scm/node/NodeManager.java | 153 +++
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 57 +
.../hadoop/hdds/scm/node/NodePoolManager.java | 71 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 909 ++++++++++++
.../hdds/scm/node/SCMNodePoolManager.java | 269 ++++
.../hadoop/hdds/scm/node/package-info.java | 31 +
.../apache/hadoop/hdds/scm/package-info.java | 22 +
.../hdds/scm/pipelines/PipelineManager.java | 175 +++
.../hdds/scm/pipelines/PipelineSelector.java | 230 ++++
.../hadoop/hdds/scm/pipelines/package-info.java | 38 +
.../scm/pipelines/ratis/RatisManagerImpl.java | 158 +++
.../hdds/scm/pipelines/ratis/package-info.java | 18 +
.../standalone/StandaloneManagerImpl.java | 145 ++
.../scm/pipelines/standalone/package-info.java | 18 +
.../hadoop/hdds/scm/ratis/package-info.java | 22 +
.../server-scm/src/main/webapps/scm/index.html | 76 ++
.../server-scm/src/main/webapps/scm/main.html | 20 +
.../src/main/webapps/scm/scm-overview.html | 60 +
.../server-scm/src/main/webapps/scm/scm.js | 54 +
.../hadoop/hdds/scm/HddsServerUtilTest.java | 308 +++++
.../TestStorageContainerManagerHttpServer.java | 140 ++
.../org/apache/hadoop/hdds/scm/TestUtils.java | 109 ++
.../hadoop/hdds/scm/block/TestBlockManager.java | 160 +++
.../hdds/scm/block/TestDeletedBlockLog.java | 363 +++++
.../hdds/scm/container/MockNodeManager.java | 520 +++++++
.../scm/container/TestContainerMapping.java | 333 +++++
.../container/closer/TestContainerCloser.java | 228 ++++
.../states/TestContainerAttribute.java | 143 ++
.../hdds/scm/node/TestContainerPlacement.java | 176 +++
.../hadoop/hdds/scm/node/TestNodeManager.java | 1176 ++++++++++++++++
.../hdds/scm/node/TestSCMNodePoolManager.java | 160 +++
.../apache/hadoop/hdds/scm/package-info.java | 21 +
.../ozone/container/common/TestEndPoint.java | 458 +++++++
.../placement/TestContainerPlacement.java | 134 ++
.../placement/TestDatanodeMetrics.java | 59 +
.../replication/TestContainerSupervisor.java | 272 ++++
.../container/replication/package-info.java | 18 +
.../ReplicationDatanodeStateManager.java | 100 ++
.../testutils/ReplicationNodeManagerMock.java | 326 +++++
.../ReplicationNodePoolManagerMock.java | 133 ++
.../ozone/container/testutils/package-info.java | 18 +
hadoop-hdds/tools/pom.xml | 65 +
.../hadoop/hdds/scm/cli/OzoneBaseCLI.java | 43 +
.../hdds/scm/cli/OzoneCommandHandler.java | 87 ++
.../apache/hadoop/hdds/scm/cli/ResultCode.java | 31 +
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 234 ++++
.../cli/container/CloseContainerHandler.java | 83 ++
.../cli/container/ContainerCommandHandler.java | 129 ++
.../cli/container/CreateContainerHandler.java | 81 ++
.../cli/container/DeleteContainerHandler.java | 92 ++
.../scm/cli/container/InfoContainerHandler.java | 114 ++
.../scm/cli/container/ListContainerHandler.java | 121 ++
.../hdds/scm/cli/container/package-info.java | 19 +
.../hadoop/hdds/scm/cli/package-info.java | 19 +
.../src/main/webapps/datanode/datanode.html | 46 +
.../src/test/resources/log4j.properties | 24 +
hadoop-minicluster/pom.xml | 6 +
hadoop-ozone/acceptance-test/README.md | 38 +
hadoop-ozone/acceptance-test/pom.xml | 86 ++
.../acceptance-test/src/test/compose/.env | 17 +
.../src/test/compose/docker-compose.yaml | 61 +
.../src/test/compose/docker-config | 35 +
.../test/robotframework/acceptance/ozone.robot | 116 ++
hadoop-ozone/client/pom.xml | 43 +
.../apache/hadoop/ozone/client/BucketArgs.java | 123 ++
.../apache/hadoop/ozone/client/ObjectStore.java | 211 +++
.../apache/hadoop/ozone/client/OzoneBucket.java | 363 +++++
.../apache/hadoop/ozone/client/OzoneClient.java | 103 ++
.../hadoop/ozone/client/OzoneClientFactory.java | 307 +++++
.../client/OzoneClientInvocationHandler.java | 62 +
.../apache/hadoop/ozone/client/OzoneKey.java | 119 ++
.../apache/hadoop/ozone/client/OzoneVolume.java | 295 ++++
.../apache/hadoop/ozone/client/VolumeArgs.java | 128 ++
.../ozone/client/io/ChunkGroupInputStream.java | 319 +++++
.../ozone/client/io/ChunkGroupOutputStream.java | 495 +++++++
.../client/io/OzoneContainerTranslation.java | 51 +
.../ozone/client/io/OzoneInputStream.java | 60 +
.../ozone/client/io/OzoneOutputStream.java | 64 +
.../hadoop/ozone/client/io/package-info.java | 23 +
.../hadoop/ozone/client/package-info.java | 23 +
.../ozone/client/protocol/ClientProtocol.java | 323 +++++
.../ozone/client/protocol/package-info.java | 23 +
.../client/rest/DefaultRestServerSelector.java | 36 +
.../ozone/client/rest/OzoneExceptionMapper.java | 45 +
.../hadoop/ozone/client/rest/RestClient.java | 799 +++++++++++
.../ozone/client/rest/RestServerSelector.java | 40 +
.../client/rest/exceptions/package-info.java | 22 +
.../ozone/client/rest/headers/package-info.java | 22 +
.../hadoop/ozone/client/rest/package-info.java | 23 +
.../hadoop/ozone/client/rpc/RpcClient.java | 567 ++++++++
.../hadoop/ozone/client/rpc/package-info.java | 23 +
.../hadoop/ozone/web/client/OzoneBucket.java | 645 +++++++++
.../hadoop/ozone/web/client/OzoneKey.java | 44 +
.../ozone/web/client/OzoneRestClient.java | 803 +++++++++++
.../web/client/OzoneRestClientException.java | 54 +
.../hadoop/ozone/web/client/OzoneVolume.java | 583 ++++++++
.../hadoop/ozone/web/client/package-info.java | 34 +
.../ozone/client/TestHddsClientUtils.java | 105 ++
.../hadoop/ozone/client/package-info.java | 23 +
.../common/dev-support/findbugsExcludeFile.xml | 22 +
hadoop-ozone/common/pom.xml | 88 ++
hadoop-ozone/common/src/main/bin/ozone | 188 +++
hadoop-ozone/common/src/main/bin/start-ozone.sh | 105 ++
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 99 ++
.../common/src/main/conf/ozone-site.xml | 24 +
.../java/org/apache/hadoop/ozone/KsmUtils.java | 87 ++
.../ozone/client/io/LengthInputStream.java | 49 +
.../hadoop/ozone/client/io/package-info.java | 22 +
.../ozone/client/rest/OzoneException.java | 267 ++++
.../ozone/client/rest/headers/Header.java | 71 +
.../ozone/client/rest/headers/package-info.java | 22 +
.../hadoop/ozone/client/rest/package-info.java | 22 +
.../ozone/client/rest/response/BucketInfo.java | 231 ++++
.../ozone/client/rest/response/KeyInfo.java | 216 +++
.../ozone/client/rest/response/VolumeInfo.java | 215 +++
.../ozone/client/rest/response/VolumeOwner.java | 61 +
.../client/rest/response/package-info.java | 24 +
.../apache/hadoop/ozone/freon/OzoneGetConf.java | 269 ++++
.../apache/hadoop/ozone/freon/package-info.java | 21 +
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 ++
.../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 ++++
.../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 ++++
.../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 ++
.../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 243 ++++
.../ozone/ksm/helpers/KsmKeyLocationInfo.java | 136 ++
.../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 ++
.../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 ++
.../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 +++
.../ozone/ksm/helpers/OpenKeySession.java | 50 +
.../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 ++++
.../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 ++
.../hadoop/ozone/ksm/helpers/package-info.java | 18 +
.../apache/hadoop/ozone/ksm/package-info.java | 21 +
.../ksm/protocol/KeySpaceManagerProtocol.java | 245 ++++
.../hadoop/ozone/ksm/protocol/package-info.java | 19 +
...ceManagerProtocolClientSideTranslatorPB.java | 744 ++++++++++
.../protocolPB/KeySpaceManagerProtocolPB.java | 34 +
.../ozone/ksm/protocolPB/package-info.java | 19 +
.../org/apache/hadoop/ozone/package-info.java | 22 +
.../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 ++
.../hadoop/ozone/protocolPB/package-info.java | 24 +
.../hadoop/ozone/web/handlers/BucketArgs.java | 214 +++
.../hadoop/ozone/web/handlers/KeyArgs.java | 117 ++
.../hadoop/ozone/web/handlers/ListArgs.java | 142 ++
.../hadoop/ozone/web/handlers/UserArgs.java | 168 +++
.../hadoop/ozone/web/handlers/VolumeArgs.java | 142 ++
.../hadoop/ozone/web/handlers/package-info.java | 22 +
.../hadoop/ozone/web/request/OzoneQuota.java | 214 +++
.../hadoop/ozone/web/request/package-info.java | 23 +
.../hadoop/ozone/web/response/BucketInfo.java | 325 +++++
.../hadoop/ozone/web/response/KeyInfo.java | 290 ++++
.../hadoop/ozone/web/response/ListBuckets.java | 154 +++
.../hadoop/ozone/web/response/ListKeys.java | 209 +++
.../hadoop/ozone/web/response/ListVolumes.java | 152 +++
.../hadoop/ozone/web/response/VolumeInfo.java | 308 +++++
.../hadoop/ozone/web/response/VolumeOwner.java | 60 +
.../hadoop/ozone/web/response/package-info.java | 23 +
.../hadoop/ozone/web/utils/OzoneUtils.java | 227 +++
.../hadoop/ozone/web/utils/package-info.java | 18 +
.../main/proto/KeySpaceManagerProtocol.proto | 457 +++++++
.../src/main/shellprofile.d/hadoop-ozone.sh | 44 +
.../apache/hadoop/ozone/web/TestBucketInfo.java | 81 ++
.../org/apache/hadoop/ozone/web/TestQuota.java | 116 ++
.../org/apache/hadoop/ozone/web/TestUtils.java | 101 ++
.../hadoop/ozone/web/TestVolumeStructs.java | 73 +
.../apache/hadoop/ozone/web/package-info.java | 21 +
hadoop-ozone/integration-test/pom.xml | 86 ++
.../container/TestContainerStateManager.java | 300 ++++
.../apache/hadoop/ozone/MiniOzoneCluster.java | 322 +++++
.../hadoop/ozone/MiniOzoneClusterImpl.java | 425 ++++++
.../apache/hadoop/ozone/RatisTestHelper.java | 110 ++
.../hadoop/ozone/TestContainerOperations.java | 87 ++
.../hadoop/ozone/TestMiniOzoneCluster.java | 217 +++
.../ozone/TestOzoneConfigurationFields.java | 38 +
.../apache/hadoop/ozone/TestOzoneHelper.java | 413 ++++++
.../ozone/TestStorageContainerManager.java | 443 ++++++
.../TestStorageContainerManagerHelper.java | 179 +++
.../ozone/client/rest/TestOzoneRestClient.java | 404 ++++++
.../hadoop/ozone/client/rest/package-info.java | 23 +
.../ozone/client/rpc/TestOzoneRpcClient.java | 815 +++++++++++
.../hadoop/ozone/client/rpc/package-info.java | 23 +
.../ozone/container/ContainerTestHelper.java | 601 ++++++++
.../common/TestBlockDeletingService.java | 398 ++++++
.../TestContainerDeletionChoosingPolicy.java | 197 +++
.../common/impl/TestContainerPersistence.java | 905 ++++++++++++
.../TestCloseContainerHandler.java | 114 ++
.../container/metrics/TestContainerMetrics.java | 127 ++
.../container/ozoneimpl/TestOzoneContainer.java | 587 ++++++++
.../ozoneimpl/TestOzoneContainerRatis.java | 142 ++
.../container/ozoneimpl/TestRatisManager.java | 128 ++
.../container/server/TestContainerServer.java | 262 ++++
.../hadoop/ozone/freon/TestDataValidate.java | 145 ++
.../apache/hadoop/ozone/freon/TestFreon.java | 130 ++
.../apache/hadoop/ozone/freon/package-info.java | 21 +
.../ozone/ksm/TestContainerReportWithKeys.java | 143 ++
.../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 306 +++++
.../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 284 ++++
.../hadoop/ozone/ksm/TestKeySpaceManager.java | 1238 +++++++++++++++++
.../ksm/TestKeySpaceManagerRestInterface.java | 134 ++
.../ozone/ksm/TestKsmBlockVersioning.java | 253 ++++
.../ksm/TestMultipleContainerReadWrite.java | 215 +++
.../hadoop/ozone/ozShell/TestOzoneShell.java | 800 +++++++++++
.../hadoop/ozone/scm/TestAllocateContainer.java | 99 ++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 300 ++++
.../ozone/scm/TestContainerSmallFile.java | 147 ++
.../org/apache/hadoop/ozone/scm/TestSCMCli.java | 540 ++++++++
.../apache/hadoop/ozone/scm/TestSCMMXBean.java | 161 +++
.../apache/hadoop/ozone/scm/TestSCMMetrics.java | 258 ++++
.../ozone/scm/TestXceiverClientManager.java | 198 +++
.../ozone/scm/TestXceiverClientMetrics.java | 168 +++
.../hadoop/ozone/scm/node/TestQueryNode.java | 120 ++
.../ozone/web/TestDistributedOzoneVolumes.java | 186 +++
.../hadoop/ozone/web/TestLocalOzoneVolumes.java | 185 +++
.../ozone/web/TestOzoneRestWithMiniCluster.java | 263 ++++
.../hadoop/ozone/web/TestOzoneWebAccess.java | 125 ++
.../hadoop/ozone/web/client/TestBuckets.java | 248 ++++
.../ozone/web/client/TestBucketsRatis.java | 76 ++
.../hadoop/ozone/web/client/TestKeys.java | 688 ++++++++++
.../hadoop/ozone/web/client/TestKeysRatis.java | 127 ++
.../ozone/web/client/TestOzoneClient.java | 305 +++++
.../hadoop/ozone/web/client/TestVolume.java | 433 ++++++
.../ozone/web/client/TestVolumeRatis.java | 129 ++
.../src/test/resources/log4j.properties | 18 +
.../src/test/resources/webapps/ksm/.gitkeep | 15 +
.../src/test/resources/webapps/scm/.gitkeep | 15 +
hadoop-ozone/objectstore-service/pom.xml | 69 +
.../server/datanode/ObjectStoreHandler.java | 191 +++
.../hdfs/server/datanode/package-info.java | 22 +
.../org/apache/hadoop/ozone/OzoneRestUtils.java | 222 +++
.../org/apache/hadoop/ozone/package-info.java | 22 +
.../ozone/web/ObjectStoreApplication.java | 59 +
.../ozone/web/OzoneHddsDatanodeService.java | 84 ++
.../hadoop/ozone/web/exceptions/ErrorTable.java | 225 +++
.../ozone/web/exceptions/package-info.java | 22 +
.../ozone/web/handlers/BucketHandler.java | 197 +++
.../web/handlers/BucketProcessTemplate.java | 323 +++++
.../hadoop/ozone/web/handlers/KeyHandler.java | 245 ++++
.../ozone/web/handlers/KeyProcessTemplate.java | 235 ++++
.../ozone/web/handlers/ServiceFilter.java | 61 +
.../web/handlers/StorageHandlerBuilder.java | 76 ++
.../ozone/web/handlers/UserHandlerBuilder.java | 75 +
.../ozone/web/handlers/VolumeHandler.java | 272 ++++
.../web/handlers/VolumeProcessTemplate.java | 276 ++++
.../hadoop/ozone/web/handlers/package-info.java | 22 +
.../hadoop/ozone/web/interfaces/Accounting.java | 57 +
.../hadoop/ozone/web/interfaces/Bucket.java | 183 +++
.../hadoop/ozone/web/interfaces/Keys.java | 146 ++
.../ozone/web/interfaces/StorageHandler.java | 295 ++++
.../hadoop/ozone/web/interfaces/UserAuth.java | 101 ++
.../hadoop/ozone/web/interfaces/Volume.java | 182 +++
.../ozone/web/interfaces/package-info.java | 22 +
.../web/localstorage/LocalStorageHandler.java | 379 +++++
.../web/localstorage/OzoneMetadataManager.java | 1138 +++++++++++++++
.../ozone/web/localstorage/package-info.java | 18 +
.../LengthInputStreamMessageBodyWriter.java | 59 +
.../web/messages/StringMessageBodyWriter.java | 62 +
.../hadoop/ozone/web/messages/package-info.java | 18 +
.../web/netty/CloseableCleanupListener.java | 46 +
.../web/netty/ObjectStoreChannelHandler.java | 78 ++
.../web/netty/ObjectStoreJerseyContainer.java | 348 +++++
.../ObjectStoreJerseyContainerProvider.java | 40 +
.../web/netty/ObjectStoreRestHttpServer.java | 215 +++
.../web/netty/ObjectStoreURLDispatcher.java | 61 +
...RequestContentObjectStoreChannelHandler.java | 117 ++
...equestDispatchObjectStoreChannelHandler.java | 103 ++
.../hadoop/ozone/web/netty/package-info.java | 26 +
.../apache/hadoop/ozone/web/package-info.java | 22 +
.../web/storage/DistributedStorageHandler.java | 539 ++++++++
.../hadoop/ozone/web/storage/package-info.java | 27 +
.../hadoop/ozone/web/userauth/Simple.java | 169 +++
.../hadoop/ozone/web/userauth/package-info.java | 23 +
...m.sun.jersey.spi.container.ContainerProvider | 16 +
.../org/apache/hadoop/ozone/TestErrorCode.java | 53 +
hadoop-ozone/ozone-manager/pom.xml | 114 ++
.../apache/hadoop/ozone/ksm/BucketManager.java | 79 ++
.../hadoop/ozone/ksm/BucketManagerImpl.java | 315 +++++
.../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 +
.../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ++++
.../ozone/ksm/KSMMetadataManagerImpl.java | 525 +++++++
.../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 437 ++++++
.../org/apache/hadoop/ozone/ksm/KSMStorage.java | 90 ++
.../hadoop/ozone/ksm/KeyDeletingService.java | 141 ++
.../org/apache/hadoop/ozone/ksm/KeyManager.java | 165 +++
.../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 512 +++++++
.../hadoop/ozone/ksm/KeySpaceManager.java | 901 ++++++++++++
.../ozone/ksm/KeySpaceManagerHttpServer.java | 78 ++
.../hadoop/ozone/ksm/OpenKeyCleanupService.java | 116 ++
.../ozone/ksm/ServiceListJSONServlet.java | 103 ++
.../apache/hadoop/ozone/ksm/VolumeManager.java | 100 ++
.../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 ++++++
.../ozone/ksm/exceptions/KSMException.java | 116 ++
.../ozone/ksm/exceptions/package-info.java | 19 +
.../apache/hadoop/ozone/ksm/package-info.java | 21 +
...ceManagerProtocolServerSideTranslatorPB.java | 539 ++++++++
.../hadoop/ozone/protocolPB/package-info.java | 22 +
.../hadoop/ozone/web/ozShell/Handler.java | 79 ++
.../apache/hadoop/ozone/web/ozShell/Shell.java | 415 ++++++
.../web/ozShell/bucket/CreateBucketHandler.java | 96 ++
.../web/ozShell/bucket/DeleteBucketHandler.java | 87 ++
.../web/ozShell/bucket/InfoBucketHandler.java | 94 ++
.../web/ozShell/bucket/ListBucketHandler.java | 113 ++
.../web/ozShell/bucket/UpdateBucketHandler.java | 94 ++
.../ozone/web/ozShell/bucket/package-info.java | 23 +
.../web/ozShell/keys/DeleteKeyHandler.java | 96 ++
.../ozone/web/ozShell/keys/GetKeyHandler.java | 116 ++
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 98 ++
.../ozone/web/ozShell/keys/ListKeyHandler.java | 116 ++
.../ozone/web/ozShell/keys/PutKeyHandler.java | 106 ++
.../ozone/web/ozShell/keys/package-info.java | 23 +
.../hadoop/ozone/web/ozShell/package-info.java | 27 +
.../web/ozShell/volume/CreateVolumeHandler.java | 99 ++
.../web/ozShell/volume/DeleteVolumeHandler.java | 81 ++
.../web/ozShell/volume/InfoVolumeHandler.java | 95 ++
.../web/ozShell/volume/ListVolumeHandler.java | 114 ++
.../web/ozShell/volume/UpdateVolumeHandler.java | 92 ++
.../ozone/web/ozShell/volume/package-info.java | 23 +
.../apache/hadoop/ozone/web/package-info.java | 24 +
.../src/main/site/markdown/OzoneCommandShell.md | 150 ++
.../site/markdown/OzoneGettingStarted.md.vm | 347 +++++
.../src/main/site/markdown/OzoneMetrics.md | 166 +++
.../src/main/site/markdown/OzoneOverview.md | 88 ++
.../src/main/site/markdown/OzoneRest.md | 549 ++++++++
.../src/main/webapps/ksm/index.html | 70 +
.../src/main/webapps/ksm/ksm-metrics.html | 44 +
.../ozone-manager/src/main/webapps/ksm/ksm.js | 108 ++
.../ozone-manager/src/main/webapps/ksm/main.css | 23 +
.../src/main/webapps/ksm/main.html | 18 +
.../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 ++++++
.../hadoop/ozone/ksm/TestChunkStreams.java | 234 ++++
.../ksm/TestKeySpaceManagerHttpServer.java | 141 ++
.../apache/hadoop/ozone/ksm/package-info.java | 21 +
hadoop-ozone/pom.xml | 169 +++
hadoop-ozone/tools/pom.xml | 71 +
.../org/apache/hadoop/ozone/freon/Freon.java | 1149 ++++++++++++++++
.../apache/hadoop/ozone/freon/package-info.java | 22 +
.../genesis/BenchMarkContainerStateMap.java | 171 +++
.../genesis/BenchMarkDatanodeDispatcher.java | 280 ++++
.../genesis/BenchMarkMetadataStoreReads.java | 67 +
.../genesis/BenchMarkMetadataStoreWrites.java | 60 +
.../ozone/genesis/BenchMarkRocksDbStore.java | 115 ++
.../apache/hadoop/ozone/genesis/Genesis.java | 54 +
.../hadoop/ozone/genesis/GenesisUtil.java | 90 ++
.../hadoop/ozone/genesis/package-info.java | 25 +
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 709 ++++++++++
.../hadoop/ozone/scm/cli/package-info.java | 22 +
.../org/apache/hadoop/test/OzoneTestDriver.java | 59 +
hadoop-project/pom.xml | 146 ++
hadoop-project/src/site/site.xml | 9 +
hadoop-tools/hadoop-ozone/pom.xml | 199 +++
.../org/apache/hadoop/fs/ozone/Constants.java | 48 +
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 43 +
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 ++
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 +
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 700 ++++++++++
.../apache/hadoop/fs/ozone/package-info.java | 30 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 155 +++
.../fs/ozone/TestOzoneFileInterfaces.java | 230 ++++
.../contract/ITestOzoneContractCreate.java | 48 +
.../contract/ITestOzoneContractDelete.java | 48 +
.../contract/ITestOzoneContractDistCp.java | 50 +
.../ITestOzoneContractGetFileStatus.java | 61 +
.../ozone/contract/ITestOzoneContractMkdir.java | 48 +
.../ozone/contract/ITestOzoneContractOpen.java | 47 +
.../contract/ITestOzoneContractRename.java | 49 +
.../contract/ITestOzoneContractRootDir.java | 51 +
.../ozone/contract/ITestOzoneContractSeek.java | 47 +
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++
.../src/test/resources/contract/ozone.xml | 113 ++
.../src/test/resources/log4j.properties | 23 +
hadoop-tools/hadoop-tools-dist/pom.xml | 15 +
hadoop-tools/pom.xml | 12 +-
pom.xml | 72 +
673 files changed, 103290 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org