You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/07/09 20:33:44 UTC
[35/37] hadoop git commit: Merge trunk into HDDS-48
Merge trunk into HDDS-48
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c275a9a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c275a9a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c275a9a6
Branch: refs/heads/trunk
Commit: c275a9a6a07b2bd889bdba4d05b420027f430b34
Parents: 44e19fc 83cd84b
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Jul 9 12:13:03 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Jul 9 12:13:03 2018 -0700
----------------------------------------------------------------------
.gitignore | 4 +
dev-support/bin/ozone-dist-layout-stitching | 2 +-
...ExcludePrivateAnnotationsStandardDoclet.java | 6 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 6 +-
.../org/apache/hadoop/conf/Configuration.java | 458 +++---
.../java/org/apache/hadoop/fs/FileContext.java | 9 +-
.../org/apache/hadoop/fs/LocalDirAllocator.java | 7 +-
.../hadoop-common/src/site/markdown/Metrics.md | 39 +-
.../org/apache/hadoop/fs/TestFileContext.java | 44 +-
.../apache/hadoop/fs/TestLocalDirAllocator.java | 59 +
.../src/main/compose/ozone/docker-compose.yaml | 6 +-
.../src/main/compose/ozone/docker-config | 2 +-
.../src/main/compose/ozoneperf/README.md | 4 +-
.../main/compose/ozoneperf/docker-compose.yaml | 6 +-
.../src/main/compose/ozoneperf/docker-config | 2 +-
.../scm/client/ContainerOperationClient.java | 117 +-
hadoop-hdds/common/pom.xml | 18 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-
.../hadoop/hdds/scm/client/ScmClient.java | 43 +-
.../container/common/helpers/ContainerInfo.java | 167 ++-
.../common/helpers/ContainerWithPipeline.java | 131 ++
.../StorageContainerLocationProtocol.java | 18 +-
...rLocationProtocolClientSideTranslatorPB.java | 34 +-
.../org/apache/hadoop/ozone/OzoneConsts.java | 22 +-
.../apache/hadoop/ozone/audit/AuditAction.java | 30 +
.../hadoop/ozone/audit/AuditEventStatus.java | 36 +
.../apache/hadoop/ozone/audit/AuditLogger.java | 128 ++
.../hadoop/ozone/audit/AuditLoggerType.java | 37 +
.../apache/hadoop/ozone/audit/AuditMarker.java | 38 +
.../apache/hadoop/ozone/audit/Auditable.java | 32 +
.../apache/hadoop/ozone/audit/package-info.java | 123 ++
.../org/apache/hadoop/ozone/common/Storage.java | 6 +-
...rLocationProtocolServerSideTranslatorPB.java | 33 +-
.../main/proto/ScmBlockLocationProtocol.proto | 10 +-
.../StorageContainerLocationProtocol.proto | 34 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 28 +-
.../common/src/main/resources/ozone-default.xml | 131 +-
.../apache/hadoop/ozone/audit/DummyAction.java | 51 +
.../apache/hadoop/ozone/audit/DummyEntity.java | 57 +
.../ozone/audit/TestOzoneAuditLogger.java | 147 ++
.../apache/hadoop/ozone/audit/package-info.java | 23 +
.../common/src/test/resources/log4j2.properties | 76 +
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 11 -
.../DeleteBlocksCommandHandler.java | 30 +-
.../protocol/StorageContainerNodeProtocol.java | 4 +-
.../src/main/resources/webapps/static/ozone.js | 4 +-
.../webapps/static/templates/config.html | 4 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 80 +-
.../block/DatanodeDeletedBlockTransactions.java | 11 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 2 +-
.../container/CloseContainerEventHandler.java | 35 +-
.../hdds/scm/container/ContainerMapping.java | 128 +-
.../scm/container/ContainerStateManager.java | 30 +-
.../hadoop/hdds/scm/container/Mapping.java | 26 +-
.../scm/container/closer/ContainerCloser.java | 15 +-
.../scm/container/states/ContainerStateMap.java | 13 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 80 ++
.../hadoop/hdds/scm/events/package-info.java | 23 +
.../hadoop/hdds/scm/node/CommandQueue.java | 2 +-
.../hadoop/hdds/scm/node/DatanodeInfo.java | 109 ++
.../hdds/scm/node/HeartbeatQueueItem.java | 98 --
.../hadoop/hdds/scm/node/NodeManager.java | 16 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 575 ++++++++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 511 +------
.../node/states/NodeAlreadyExistsException.java | 45 +
.../hdds/scm/node/states/NodeException.java | 44 +
.../scm/node/states/NodeNotFoundException.java | 49 +
.../hdds/scm/node/states/NodeStateMap.java | 281 ++++
.../hdds/scm/pipelines/PipelineManager.java | 27 +-
.../hdds/scm/pipelines/PipelineSelector.java | 16 +
.../scm/pipelines/ratis/RatisManagerImpl.java | 1 +
.../standalone/StandaloneManagerImpl.java | 1 +
.../hdds/scm/server/SCMBlockProtocolServer.java | 2 +-
.../scm/server/SCMClientProtocolServer.java | 74 +-
.../server/SCMDatanodeHeartbeatDispatcher.java | 13 +-
.../scm/server/SCMDatanodeProtocolServer.java | 2 +-
.../scm/server/StorageContainerManager.java | 7 +-
.../hdds/scm/block/TestDeletedBlockLog.java | 15 +-
.../hdds/scm/container/MockNodeManager.java | 58 +-
.../TestCloseContainerEventHandler.java | 54 +-
.../scm/container/TestContainerMapping.java | 27 +-
.../container/closer/TestContainerCloser.java | 18 +-
.../hdds/scm/node/TestContainerPlacement.java | 16 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 186 +--
.../TestSCMDatanodeHeartbeatDispatcher.java | 20 +-
.../testutils/ReplicationNodeManagerMock.java | 37 +-
.../hadoop/hdds/scm/cli/OzoneBaseCLI.java | 2 +-
.../cli/container/CloseContainerHandler.java | 10 +-
.../cli/container/DeleteContainerHandler.java | 9 +-
.../scm/cli/container/InfoContainerHandler.java | 11 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 19 -
.../org/apache/hadoop/hdfs/DFSInputStream.java | 46 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 15 +-
.../hdfs/client/HdfsClientConfigKeys.java | 3 +
.../hdfs/client/impl/BlockReaderFactory.java | 21 +-
.../hdfs/client/impl/BlockReaderLocal.java | 93 +-
.../client/impl/BlockReaderLocalLegacy.java | 44 +-
.../hdfs/client/impl/BlockReaderRemote.java | 33 +-
.../datanode/ReplicaNotFoundException.java | 2 +-
.../ha/ConfiguredFailoverProxyProvider.java | 9 +-
.../InMemoryAliasMapFailoverProxyProvider.java | 38 +
.../hdfs/server/federation/router/Quota.java | 10 +-
.../router/RouterQuotaUpdateService.java | 43 +-
.../federation/router/RouterRpcServer.java | 1 -
.../router/TestDisableRouterQuota.java | 94 ++
.../federation/router/TestRouterQuota.java | 212 ++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 37 +-
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 15 +-
...yAliasMapProtocolClientSideTranslatorPB.java | 95 +-
.../aliasmap/InMemoryAliasMapProtocol.java | 5 +
.../aliasmap/InMemoryLevelDBAliasMapServer.java | 19 +-
.../impl/InMemoryLevelDBAliasMapClient.java | 80 +-
.../impl/TextFileRegionAliasMap.java | 5 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 21 +-
.../hdfs/server/datanode/DiskBalancer.java | 29 +-
.../erasurecode/StripedBlockReader.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 8 +
.../hdfs/server/namenode/NamenodeFsck.java | 1 -
.../src/main/resources/hdfs-default.xml | 35 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 13 +-
.../apache/hadoop/hdfs/MiniDFSNNTopology.java | 2 +-
.../hdfs/client/impl/BlockReaderTestUtil.java | 2 -
.../hdfs/client/impl/TestBlockReaderLocal.java | 2 -
.../blockmanagement/TestBlockTokenWithDFS.java | 2 -
.../TestNameNodePrunesMissingStorages.java | 5 +-
.../impl/TestInMemoryLevelDBAliasMapClient.java | 7 +
.../datanode/TestDataNodeVolumeFailure.java | 2 -
.../server/diskbalancer/TestDiskBalancer.java | 80 +-
.../shortcircuit/TestShortCircuitCache.java | 89 ++
.../src/test/acceptance/basic/basic.robot | 6 +-
.../test/acceptance/basic/docker-compose.yaml | 8 +-
.../src/test/acceptance/basic/docker-config | 4 +-
.../src/test/acceptance/basic/ozone-shell.robot | 18 +-
.../src/test/acceptance/commonlib.robot | 4 +-
.../test/acceptance/ozonefs/docker-compose.yaml | 8 +-
.../src/test/acceptance/ozonefs/docker-config | 4 +-
.../src/test/acceptance/ozonefs/ozonefs.robot | 6 +-
.../apache/hadoop/ozone/client/BucketArgs.java | 4 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 89 +-
.../apache/hadoop/ozone/client/OzoneKey.java | 2 +-
.../apache/hadoop/ozone/client/VolumeArgs.java | 4 +-
.../ozone/client/io/ChunkGroupInputStream.java | 33 +-
.../ozone/client/io/ChunkGroupOutputStream.java | 63 +-
.../client/rest/DefaultRestServerSelector.java | 2 +-
.../hadoop/ozone/client/rest/RestClient.java | 15 +-
.../ozone/client/rest/RestServerSelector.java | 2 +-
.../hadoop/ozone/client/rpc/RpcClient.java | 142 +-
.../ozone/client/TestHddsClientUtils.java | 24 +-
hadoop-ozone/common/pom.xml | 2 +-
hadoop-ozone/common/src/main/bin/ozone | 9 +-
hadoop-ozone/common/src/main/bin/start-ozone.sh | 16 +-
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 16 +-
.../java/org/apache/hadoop/ozone/KsmUtils.java | 87 --
.../java/org/apache/hadoop/ozone/OmUtils.java | 94 ++
.../org/apache/hadoop/ozone/audit/OMAction.java | 51 +
.../apache/hadoop/ozone/audit/package-info.java | 22 +
.../apache/hadoop/ozone/freon/OzoneGetConf.java | 16 +-
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 --
.../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 ---
.../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 ---
.../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 --
.../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 277 ----
.../ozone/ksm/helpers/KsmKeyLocationInfo.java | 129 --
.../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 --
.../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 --
.../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 ---
.../ozone/ksm/helpers/OpenKeySession.java | 50 -
.../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 ---
.../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 --
.../hadoop/ozone/ksm/helpers/package-info.java | 18 -
.../apache/hadoop/ozone/ksm/package-info.java | 21 -
.../ksm/protocol/KeySpaceManagerProtocol.java | 252 ----
.../hadoop/ozone/ksm/protocol/package-info.java | 19 -
...ceManagerProtocolClientSideTranslatorPB.java | 769 ----------
.../protocolPB/KeySpaceManagerProtocolPB.java | 34 -
.../ozone/ksm/protocolPB/package-info.java | 19 -
.../apache/hadoop/ozone/om/OMConfigKeys.java | 81 ++
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 233 +++
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 235 +++
.../hadoop/ozone/om/helpers/OmKeyArgs.java | 119 ++
.../hadoop/ozone/om/helpers/OmKeyInfo.java | 277 ++++
.../ozone/om/helpers/OmKeyLocationInfo.java | 129 ++
.../om/helpers/OmKeyLocationInfoGroup.java | 118 ++
.../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 110 ++
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 223 +++
.../hadoop/ozone/om/helpers/OpenKeySession.java | 50 +
.../hadoop/ozone/om/helpers/ServiceInfo.java | 237 +++
.../hadoop/ozone/om/helpers/VolumeArgs.java | 140 ++
.../hadoop/ozone/om/helpers/package-info.java | 18 +
.../apache/hadoop/ozone/om/package-info.java | 21 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 252 ++++
.../hadoop/ozone/om/protocol/package-info.java | 19 +
...neManagerProtocolClientSideTranslatorPB.java | 769 ++++++++++
.../om/protocolPB/OzoneManagerProtocolPB.java | 34 +
.../ozone/om/protocolPB/package-info.java | 19 +
.../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 --
.../hadoop/ozone/protocolPB/OMPBHelper.java | 113 ++
.../hadoop/ozone/protocolPB/OzonePBHelper.java | 30 +
.../main/proto/KeySpaceManagerProtocol.proto | 474 ------
.../src/main/proto/OzoneManagerProtocol.proto | 480 +++++++
hadoop-ozone/docs/content/GettingStarted.md | 18 +-
hadoop-ozone/docs/content/Metrics.md | 10 +-
hadoop-ozone/docs/content/_index.md | 12 +-
hadoop-ozone/docs/static/OzoneOverview.svg | 2 +-
.../container/TestContainerStateManager.java | 161 ++-
.../apache/hadoop/ozone/MiniOzoneCluster.java | 24 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 66 +-
.../hadoop/ozone/TestContainerOperations.java | 11 +-
.../ozone/TestOzoneConfigurationFields.java | 4 +-
.../ozone/TestStorageContainerManager.java | 28 +-
.../TestStorageContainerManagerHelper.java | 22 +-
.../ozone/client/rest/TestOzoneRestClient.java | 6 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 22 +-
.../commandhandler/TestBlockDeletion.java | 212 +++
.../TestCloseContainerByPipeline.java | 97 +-
.../TestCloseContainerHandler.java | 14 +-
.../ozone/ksm/TestContainerReportWithKeys.java | 143 --
.../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 306 ----
.../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 284 ----
.../hadoop/ozone/ksm/TestKeySpaceManager.java | 1350 ------------------
.../ksm/TestKeySpaceManagerRestInterface.java | 135 --
.../ozone/ksm/TestKsmBlockVersioning.java | 253 ----
.../ksm/TestMultipleContainerReadWrite.java | 215 ---
.../ozone/om/TestContainerReportWithKeys.java | 143 ++
.../om/TestMultipleContainerReadWrite.java | 215 +++
.../hadoop/ozone/om/TestOmBlockVersioning.java | 253 ++++
.../apache/hadoop/ozone/om/TestOmMetrics.java | 313 ++++
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 284 ++++
.../hadoop/ozone/om/TestOzoneManager.java | 1349 +++++++++++++++++
.../ozone/om/TestOzoneManagerRestInterface.java | 135 ++
.../hadoop/ozone/ozShell/TestOzoneShell.java | 14 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 6 +-
.../hadoop/ozone/scm/TestContainerSQLCli.java | 3 +-
.../ozone/scm/TestContainerSmallFile.java | 36 +-
.../org/apache/hadoop/ozone/scm/TestSCMCli.java | 127 +-
.../ozone/scm/TestXceiverClientManager.java | 62 +-
.../ozone/scm/TestXceiverClientMetrics.java | 14 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 19 +-
.../ozone/web/TestDistributedOzoneVolumes.java | 12 +-
.../hadoop/ozone/web/client/TestKeys.java | 58 +-
.../src/test/resources/webapps/ksm/.gitkeep | 15 -
.../resources/webapps/ozoneManager/.gitkeep | 15 +
.../server/datanode/ObjectStoreHandler.java | 33 +-
.../ozone/web/handlers/KeyProcessTemplate.java | 4 +-
.../web/handlers/VolumeProcessTemplate.java | 4 +-
.../web/storage/DistributedStorageHandler.java | 153 +-
.../apache/hadoop/ozone/ksm/BucketManager.java | 79 -
.../hadoop/ozone/ksm/BucketManagerImpl.java | 315 ----
.../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 -
.../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ----
.../ozone/ksm/KSMMetadataManagerImpl.java | 526 -------
.../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 459 ------
.../org/apache/hadoop/ozone/ksm/KSMStorage.java | 90 --
.../hadoop/ozone/ksm/KeyDeletingService.java | 142 --
.../org/apache/hadoop/ozone/ksm/KeyManager.java | 175 ---
.../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 566 --------
.../hadoop/ozone/ksm/KeySpaceManager.java | 914 ------------
.../ozone/ksm/KeySpaceManagerHttpServer.java | 78 -
.../hadoop/ozone/ksm/OpenKeyCleanupService.java | 117 --
.../ozone/ksm/ServiceListJSONServlet.java | 103 --
.../apache/hadoop/ozone/ksm/VolumeManager.java | 100 --
.../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 -----
.../ozone/ksm/exceptions/KSMException.java | 118 --
.../ozone/ksm/exceptions/package-info.java | 19 -
.../apache/hadoop/ozone/ksm/package-info.java | 21 -
.../apache/hadoop/ozone/om/BucketManager.java | 79 +
.../hadoop/ozone/om/BucketManagerImpl.java | 315 ++++
.../hadoop/ozone/om/KeyDeletingService.java | 142 ++
.../org/apache/hadoop/ozone/om/KeyManager.java | 175 +++
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 566 ++++++++
.../org/apache/hadoop/ozone/om/OMMXBean.java | 31 +
.../hadoop/ozone/om/OMMetadataManager.java | 253 ++++
.../org/apache/hadoop/ozone/om/OMMetrics.java | 459 ++++++
.../org/apache/hadoop/ozone/om/OMStorage.java | 90 ++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 526 +++++++
.../hadoop/ozone/om/OpenKeyCleanupService.java | 117 ++
.../apache/hadoop/ozone/om/OzoneManager.java | 911 ++++++++++++
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 78 +
.../hadoop/ozone/om/ServiceListJSONServlet.java | 103 ++
.../apache/hadoop/ozone/om/VolumeManager.java | 100 ++
.../hadoop/ozone/om/VolumeManagerImpl.java | 390 +++++
.../hadoop/ozone/om/exceptions/OMException.java | 118 ++
.../ozone/om/exceptions/package-info.java | 19 +
.../apache/hadoop/ozone/om/package-info.java | 21 +
...ceManagerProtocolServerSideTranslatorPB.java | 559 --------
...neManagerProtocolServerSideTranslatorPB.java | 571 ++++++++
.../hadoop/ozone/protocolPB/package-info.java | 2 +-
.../src/main/webapps/ksm/index.html | 70 -
.../src/main/webapps/ksm/ksm-metrics.html | 44 -
.../ozone-manager/src/main/webapps/ksm/ksm.js | 110 --
.../ozone-manager/src/main/webapps/ksm/main.css | 23 -
.../src/main/webapps/ksm/main.html | 18 -
.../src/main/webapps/ozoneManager/index.html | 70 +
.../src/main/webapps/ozoneManager/main.css | 23 +
.../src/main/webapps/ozoneManager/main.html | 18 +
.../main/webapps/ozoneManager/om-metrics.html | 44 +
.../main/webapps/ozoneManager/ozoneManager.js | 110 ++
.../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 -----
.../hadoop/ozone/ksm/TestChunkStreams.java | 234 ---
.../ksm/TestKeySpaceManagerHttpServer.java | 141 --
.../apache/hadoop/ozone/ksm/package-info.java | 21 -
.../hadoop/ozone/om/TestBucketManagerImpl.java | 394 +++++
.../hadoop/ozone/om/TestChunkStreams.java | 234 +++
.../ozone/om/TestOzoneManagerHttpServer.java | 141 ++
.../apache/hadoop/ozone/om/package-info.java | 21 +
.../hadoop/fs/ozone/contract/OzoneContract.java | 4 +-
.../genesis/BenchMarkContainerStateMap.java | 16 +-
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 111 +-
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 10 +
.../s3guard/AbstractS3GuardToolTestBase.java | 18 +
.../namenode/ITestProvidedImplementation.java | 373 ++++-
.../dev-support/findbugs-exclude.xml | 17 +-
.../hadoop/yarn/api/records/Resource.java | 13 +
.../api/records/impl/LightWeightResource.java | 23 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 7 +
.../impl/pb/GetApplicationsRequestPBImpl.java | 44 +-
.../logaggregation/AggregatedLogFormat.java | 6 +-
.../timeline/RollingLevelDBTimelineStore.java | 6 +
.../server/timeline/TimelineDataManager.java | 7 +-
.../timeline/webapp/TimelineWebServices.java | 4 +
.../webapp/TestTimelineWebServices.java | 2 +-
.../amrmproxy/BroadcastAMRMProxyPolicy.java | 11 -
.../amrmproxy/RejectAMRMProxyPolicy.java | 4 -
.../TestBroadcastAMRMProxyFederationPolicy.java | 11 +-
.../yarn/server/nodemanager/NodeManager.java | 66 +-
.../runtime/DockerLinuxContainerRuntime.java | 4 +-
.../runtime/ContainerExecutionException.java | 6 +
.../impl/container-executor.c | 30 +-
.../container-executor/impl/utils/docker-util.c | 2 +-
.../test/test-container-executor.c | 20 +
.../nodemanager/TestNodeManagerResync.java | 56 +
.../runtime/TestDockerContainerRuntime.java | 10 +-
.../conf/capacity-scheduler.xml | 10 +
.../scheduler/capacity/CapacityScheduler.java | 45 +-
.../CapacitySchedulerConfiguration.java | 10 +
.../scheduler/capacity/ParentQueue.java | 36 +-
.../allocator/AbstractContainerAllocator.java | 13 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 5 +
.../scheduler/fair/ConfigurableResource.java | 69 +-
.../fair/FairSchedulerConfiguration.java | 174 ++-
.../allocation/AllocationFileQueueParser.java | 2 +-
.../resourcemanager/webapp/dao/AppInfo.java | 2 +-
.../webapp/dao/SchedulerInfo.java | 8 +-
.../TestWorkPreservingRMRestart.java | 2 +
.../fair/TestFairSchedulerConfiguration.java | 160 ++-
.../webapp/TestRMWebServices.java | 31 +-
.../webapp/TestRMWebServicesApps.java | 14 +-
...estRMWebServicesAppsCustomResourceTypes.java | 242 ++++
.../webapp/TestRMWebServicesCapacitySched.java | 30 +-
.../TestRMWebServicesConfigurationMutation.java | 5 +
.../webapp/TestRMWebServicesFairScheduler.java | 95 +-
.../TestRMWebServicesSchedulerActivities.java | 2 +-
...ustomResourceTypesConfigurationProvider.java | 138 ++
.../FairSchedulerJsonVerifications.java | 139 ++
.../FairSchedulerXmlVerifications.java | 153 ++
...ervicesFairSchedulerCustomResourceTypes.java | 271 ++++
.../webapp/helper/AppInfoJsonVerifications.java | 123 ++
.../webapp/helper/AppInfoXmlVerifications.java | 132 ++
.../webapp/helper/BufferedClientResponse.java | 57 +
.../helper/JsonCustomResourceTypeTestcase.java | 77 +
.../ResourceRequestsJsonVerifications.java | 252 ++++
.../ResourceRequestsXmlVerifications.java | 215 +++
.../helper/XmlCustomResourceTypeTestCase.java | 112 ++
.../router/clientrm/RouterClientRMService.java | 53 +-
.../router/rmadmin/RouterRMAdminService.java | 51 +-
.../server/router/webapp/RouterWebServices.java | 48 +-
.../clientrm/TestRouterClientRMService.java | 60 +
.../rmadmin/TestRouterRMAdminService.java | 60 +
.../router/webapp/TestRouterWebServices.java | 65 +
.../pom.xml | 10 +
.../storage/TestTimelineReaderHBaseDown.java | 220 +++
.../storage/HBaseTimelineReaderImpl.java | 93 ++
.../reader/TimelineFromIdConverter.java | 93 ++
.../reader/TimelineReaderWebServices.java | 198 ++-
.../TestTimelineReaderWebServicesBasicAcl.java | 154 ++
.../src/site/markdown/FairScheduler.md | 6 +-
.../src/main/webapp/app/initializers/loader.js | 10 +-
379 files changed, 22363 insertions(+), 15606 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 82d67b7,4fad5d8..0db5993
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -98,11 -93,8 +98,11 @@@ public final class OzoneConsts
public static final String BLOCK_DB = "block.db";
public static final String OPEN_CONTAINERS_DB = "openContainers.db";
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
- public static final String KSM_DB_NAME = "ksm.db";
+ public static final String OM_DB_NAME = "om.db";
+ public static final String STORAGE_DIR_CHUNKS = "chunks";
+ public static final String CONTAINER_FILE_CHECKSUM_EXTENSION = ".chksm";
+
/**
* Supports Bucket Versioning.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index 4fc1cd9,d215da9..c3d1596
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@@ -31,13 -29,11 +31,12 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers
.DeletedContainerBlocksSummary;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
- import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.statemachine
.EndpointStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine
@@@ -167,21 -145,28 +166,28 @@@ public class DeleteBlocksCommandHandle
* Move a bunch of blocks from a container to deleting state.
* This is a meta update, the actual deletes happen in async mode.
*
+ * @param containerData - KeyValueContainerData
* @param delTX a block deletion transaction.
- * @param config configuration.
* @throws IOException if I/O error occurs.
*/
- private void deleteContainerBlocks(DeletedBlocksTransaction delTX,
- Configuration config) throws IOException {
+ private void deleteKeyValueContainerBlocks(
+ KeyValueContainerData containerData, DeletedBlocksTransaction delTX)
+ throws IOException {
long containerId = delTX.getContainerID();
- ContainerData containerInfo = containerManager.readContainer(containerId);
if (LOG.isDebugEnabled()) {
LOG.debug("Processing Container : {}, DB path : {}", containerId,
- containerInfo.getDBPath());
+ containerData.getMetadataPath());
}
- if (delTX.getTxID() < containerInfo.getDeleteTransactionId()) {
++ if (delTX.getTxID() < containerData.getDeleteTransactionId()) {
+ LOG.debug(String.format("Ignoring delete blocks for containerId: %d."
+ + " Outdated delete transactionId %d < %d", containerId,
- delTX.getTxID(), containerInfo.getDeleteTransactionId()));
++ delTX.getTxID(), containerData.getDeleteTransactionId()));
+ return;
+ }
+
int newDeletionBlocks = 0;
- MetadataStore containerDB = KeyUtils.getDB(containerInfo, config);
+ MetadataStore containerDB = KeyUtils.getDB(containerData, conf);
for (Long blk : delTX.getLocalIDList()) {
BatchOperation batch = new BatchOperation();
byte[] blkBytes = Longs.toByteArray(blk);
@@@ -208,13 -203,15 +224,15 @@@
LOG.debug("Block {} not found or already under deletion in"
+ " container {}, skip deleting it.", blk, containerId);
}
- containerDB.put(DFSUtil.string2Bytes(
- OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + containerId),
- Longs.toByteArray(delTX.getTxID()));
}
+ containerDB.put(DFSUtil.string2Bytes(
+ OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()),
+ Longs.toByteArray(delTX.getTxID()));
- containerManager
- .updateDeleteTransactionId(delTX.getContainerID(), delTX.getTxID());
++ containerData
++ .updateDeleteTransactionId(delTX.getTxID());
// update pending deletion blocks count in in-memory container status
- containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId);
+ containerData.incrPendingDeletionBlocks(newDeletionBlocks);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
index ad1e706,a30c6f4..fff8611
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
@@@ -27,13 -27,11 +27,13 @@@ import org.apache.hadoop.hdds.scm.conta
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
@@@ -160,14 -158,15 +160,16 @@@ public class TestStorageContainerManage
private MetadataStore getContainerMetadata(Long containerID)
throws IOException {
- ContainerInfo container = cluster.getStorageContainerManager()
- .getClientProtocolServer().getContainer(containerID);
- DatanodeDetails leadDN = container.getPipeline().getLeader();
+ ContainerWithPipeline containerWithPipeline = cluster
+ .getStorageContainerManager().getClientProtocolServer()
+ .getContainerWithPipeline(containerID);
+
+ DatanodeDetails leadDN = containerWithPipeline.getPipeline().getLeader();
OzoneContainer containerServer =
getContainerServerByDatanodeUuid(leadDN.getUuidString());
- ContainerData containerData = containerServer.getContainerManager()
- .readContainer(containerID);
+ KeyValueContainerData containerData = (KeyValueContainerData) containerServer
+ .getContainerSet()
+ .getContainer(containerID).getContainerData();
return KeyUtils.getDB(containerData, conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index b832dd2,58b831b..30b18c2
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@@ -32,10 -32,10 +32,10 @@@ import org.apache.hadoop.ozone.client.O
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
@@@ -204,14 -257,8 +257,8 @@@ public class TestCloseContainerByPipeli
if (datanode.equals(datanodeService.getDatanodeDetails())) {
containerData =
datanodeService.getDatanodeStateMachine().getContainer()
- .getContainerManager().readContainer(containerID);
+ .getContainerSet().getContainer(containerID).getContainerData();
- if (!containerData.isOpen()) {
- // make sure the closeContainerHandler on the Datanode is invoked
- Assert.assertTrue(
- datanodeService.getDatanodeStateMachine().getCommandDispatcher()
- .getCloseContainerHandler().getInvocationCount() > 0);
- return true;
- }
+ return !containerData.isOpen();
}
} catch (StorageContainerException e) {
throw new AssertionError(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
index 114bd04,58a5154..682bd63
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
@@@ -27,9 -27,9 +27,9 @@@ import org.apache.hadoop.hdds.client.Re
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
index 0000000,5481506..c25b00e
mode 000000,100644..100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
@@@ -1,0 -1,143 +1,143 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+ package org.apache.hadoop.ozone.om;
+
+ import org.apache.commons.lang3.RandomStringUtils;
+
+ import org.apache.hadoop.hdds.client.ReplicationFactor;
+ import org.apache.hadoop.hdds.client.ReplicationType;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+ import org.apache.hadoop.ozone.MiniOzoneCluster;
+ import org.apache.hadoop.ozone.OzoneConfigKeys;
+ import org.apache.hadoop.ozone.OzoneConsts;
+ import org.apache.hadoop.ozone.client.*;
+ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
++import org.apache.hadoop.ozone.container.common.impl.ContainerData;
++import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+ import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+ import org.junit.AfterClass;
+ import org.junit.BeforeClass;
+ import org.junit.Rule;
+ import org.junit.Test;
+ import org.junit.rules.ExpectedException;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.IOException;
+
+ /**
+ * This class tests container report with DN container state info.
+ */
+ public class TestContainerReportWithKeys {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestContainerReportWithKeys.class);
+ private static MiniOzoneCluster cluster = null;
+ private static OzoneConfiguration conf;
+ private static StorageContainerManager scm;
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true and
+ * OZONE_HANDLER_TYPE_KEY = "distributed"
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
+ OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
+ cluster = MiniOzoneCluster.newBuilder(conf).build();
+ cluster.waitForClusterToBeReady();
+ scm = cluster.getStorageContainerManager();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testContainerReportKeyWrite() throws Exception {
+ final String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ final String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ final String keyName = "key" + RandomStringUtils.randomNumeric(5);
+ final int keySize = 100;
+
+ OzoneClient client = OzoneClientFactory.getClient(conf);
+ ObjectStore objectStore = client.getObjectStore();
+ objectStore.createVolume(volumeName);
+ objectStore.getVolume(volumeName).createBucket(bucketName);
+ OzoneOutputStream key =
+ objectStore.getVolume(volumeName).getBucket(bucketName)
+ .createKey(keyName, keySize, ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE);
+ String dataString = RandomStringUtils.randomAlphabetic(keySize);
+ key.write(dataString.getBytes());
+ key.close();
+
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setType(HddsProtos.ReplicationType.STAND_ALONE)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(keySize)
+ .build();
+
+
+ OmKeyLocationInfo keyInfo =
+ cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
+ .get(0).getBlocksLatestVersionOnly().get(0);
+
+ ContainerData cd = getContainerData(keyInfo.getContainerID());
+
- LOG.info("DN Container Data: keyCount: {} used: {} ",
- cd.getKeyCount(), cd.getBytesUsed());
++/* LOG.info("DN Container Data: keyCount: {} used: {} ",
++ cd.getKeyCount(), cd.getBytesUsed());*/
+
+ ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerID());
+
+ LOG.info("SCM Container Info keyCount: {} usedBytes: {}",
+ cinfo.getNumberOfKeys(), cinfo.getUsedBytes());
+ }
+
+
+ private static ContainerData getContainerData(long containerID) {
+ ContainerData containerData;
+ try {
- ContainerManager containerManager = cluster.getHddsDatanodes().get(0)
- .getDatanodeStateMachine().getContainer().getContainerManager();
- containerData = containerManager.readContainer(containerID);
++ ContainerSet containerManager = cluster.getHddsDatanodes().get(0)
++ .getDatanodeStateMachine().getContainer().getContainerSet();
++ containerData = containerManager.getContainer(containerID).getContainerData();
+ } catch (StorageContainerException e) {
+ throw new AssertionError(e);
+ }
+ return containerData;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
index 5c62803,42bb936..a2d95e8
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
@@@ -141,9 -144,8 +144,8 @@@ public class TestContainerSmallFile
ContainerProtocolCalls.writeSmallFile(client, blockID,
"data123".getBytes(), traceID);
-
thrown.expect(StorageContainerException.class);
- thrown.expectMessage("Unable to find the container");
+ thrown.expectMessage("ContainerID 8888 does not exist");
// Try to read a invalid key
ContainerProtos.GetSmallFileResponseProto response =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
index 12d444a,a6bb586..cc11feb
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
@@@ -162,21 -158,22 +163,22 @@@ public class TestSCMCli
// 1. Test to delete a non-empty container.
// ****************************************
// Create an non-empty container
- ContainerInfo container = containerOperationClient
+ ContainerWithPipeline container = containerOperationClient
.createContainer(xceiverClientManager.getType(),
HddsProtos.ReplicationFactor.ONE, containerOwner);
--
- ContainerData cdata = ContainerData
- .getFromProtBuf(containerOperationClient.readContainer(
- container.getContainerInfo().getContainerID()), conf);
- KeyUtils.getDB(cdata, conf)
+ KeyValueContainerData kvData = KeyValueContainerData
+ .getFromProtoBuf(containerOperationClient.readContainer(
- container.getContainerID(), container.getPipeline()));
++ container.getContainerInfo().getContainerID(), container
++ .getPipeline()));
+ KeyUtils.getDB(kvData, conf)
- .put(Longs.toByteArray(container.getContainerID()),
+ .put(Longs.toByteArray(container.getContainerInfo().getContainerID()),
"someKey".getBytes());
- Assert.assertTrue(containerExist(container.getContainerID()));
- Assert.assertTrue(
- containerExist(container.getContainerInfo().getContainerID()));
++ Assert.assertTrue(containerExist(container.getContainerInfo()
++ .getContainerID()));
// Gracefully delete a container should fail because it is open.
- delCmd = new String[] {"-container", "-delete", "-c",
- Long.toString(container.getContainerID())};
+ delCmd = new String[]{"-container", "-delete", "-c",
+ Long.toString(container.getContainerInfo().getContainerID())};
testErr = new ByteArrayOutputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
exitCode = runCommandAndGetOutput(delCmd, out, testErr);
@@@ -275,26 -267,24 +272,27 @@@
EXECUTION_ERROR, exitCode);
// Create an empty container.
- ContainerInfo container = containerOperationClient
+ ContainerWithPipeline container = containerOperationClient
.createContainer(xceiverClientManager.getType(),
HddsProtos.ReplicationFactor.ONE, containerOwner);
- ContainerData data = ContainerData.getFromProtBuf(containerOperationClient
- .readContainer(container.getContainerInfo().getContainerID()), conf);
-
+ KeyValueContainerData data = KeyValueContainerData
+ .getFromProtoBuf(containerOperationClient.
- readContainer(container.getContainerID(),
++ readContainer(container.getContainerInfo().getContainerID(),
+ container.getPipeline()));
-
- info = new String[] { "-container", "-info", "-c",
- Long.toString(container.getContainerID()) };
+ info = new String[]{"-container", "-info", "-c",
+ Long.toString(container.getContainerInfo().getContainerID())};
ByteArrayOutputStream out = new ByteArrayOutputStream();
exitCode = runCommandAndGetOutput(info, out, null);
assertEquals("Expected Success, did not find it.", ResultCode.SUCCESS,
- exitCode);
+ exitCode);
String openStatus = data.isOpen() ? "OPEN" : "CLOSED";
- String expected = String.format(formatStr, container.getContainerInfo()
- .getContainerID(), openStatus, data.getDBPath(),
- data.getContainerPath(), "", datanodeDetails.getHostName(),
- datanodeDetails.getHostName());
+ String expected =
- String.format(formatStr, container.getContainerID(), openStatus,
- data.getDbFile().getPath(), data.getContainerPath(), "",
- datanodeDetails.getHostName(), datanodeDetails.getHostName());
++ String.format(formatStr, container.getContainerInfo().getContainerID
++ (), openStatus, data.getDbFile().getPath(), data
++ .getContainerPath(), "", datanodeDetails.getHostName(),
++ datanodeDetails.getHostName());
++
assertEquals(expected, out.toString());
out.reset();
@@@ -303,9 -293,9 +301,10 @@@
container = containerOperationClient
.createContainer(xceiverClientManager.getType(),
HddsProtos.ReplicationFactor.ONE, containerOwner);
- data = ContainerData
- .getFromProtBuf(containerOperationClient.readContainer(
- container.getContainerInfo().getContainerID()), conf);
+ data = KeyValueContainerData
+ .getFromProtoBuf(containerOperationClient.readContainer(
- container.getContainerID(), container.getPipeline()));
++ container.getContainerInfo().getContainerID(), container
++ .getPipeline()));
KeyUtils.getDB(data, conf)
.put(containerID.getBytes(), "someKey".getBytes());
@@@ -315,25 -305,24 +314,27 @@@
assertEquals(ResultCode.SUCCESS, exitCode);
openStatus = data.isOpen() ? "OPEN" : "CLOSED";
- expected = String.format(formatStr, container.getContainerID(), openStatus,
- data.getDbFile().getPath(), data.getContainerPath(), "",
- datanodeDetails.getHostName(), datanodeDetails.getHostName());
- expected = String.format(formatStr, container.getContainerInfo().
- getContainerID(), openStatus, data.getDBPath(),
- data.getContainerPath(), "", datanodeDetails.getHostName(),
++
++ expected = String.format(formatStr, container.getContainerInfo()
++ .getContainerID(), openStatus, data.getDbFile().getPath(), data
++ .getContainerPath(), "", datanodeDetails.getHostName(),
+ datanodeDetails.getHostName());
assertEquals(expected, out.toString());
out.reset();
-
// Close last container and test info again.
- containerOperationClient.closeContainer(
- container.getContainerID(), container.getPipeline());
+ containerOperationClient
+ .closeContainer(container.getContainerInfo().getContainerID());
- info = new String[] { "-container", "-info", "-c",
- Long.toString(container.getContainerID()) };
+ info = new String[]{"-container", "-info", "-c",
+ Long.toString(container.getContainerInfo().getContainerID())};
exitCode = runCommandAndGetOutput(info, out, null);
assertEquals(ResultCode.SUCCESS, exitCode);
- data = ContainerData.getFromProtBuf(containerOperationClient
- .readContainer(container.getContainerInfo().getContainerID()), conf);
+ data = KeyValueContainerData
+ .getFromProtoBuf(containerOperationClient.readContainer(
- container.getContainerID(), container.getPipeline()));
++ container.getContainerInfo().getContainerID(), container
++ .getPipeline()));
openStatus = data.isOpen() ? "OPEN" : "CLOSED";
expected = String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c275a9a6/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 2f592c1,a95bd0e..c144db2
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@@ -44,18 -44,17 +44,18 @@@ import org.apache.hadoop.ozone.client.i
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
- import org.apache.hadoop.ozone.ksm.KeySpaceManager;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
- import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
- import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
- import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
- import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
+ import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.Status;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@@@ -663,12 -661,11 +663,12 @@@ public class TestKeys
}
@Test
+ @Ignore("Needs to be fixed for new SCM and Storage design")
public void testDeleteKey() throws Exception {
- KeySpaceManager ksm = ozoneCluster.getKeySpaceManager();
+ OzoneManager ozoneManager = ozoneCluster.getOzoneManager();
// To avoid interference from other test cases,
// we collect number of existing keys at the beginning
- int numOfExistedKeys = countKsmKeys(ksm);
+ int numOfExistedKeys = countOmKeys(ozoneManager);
// Keep tracking bucket keys info while creating them
PutHelper helper = new PutHelper(client, path);
@@@ -697,20 -694,17 +697,20 @@@
// Memorize chunks that has been created,
// so we can verify actual deletions at DN side later.
- for (KsmKeyInfo keyInfo : createdKeys) {
- List<KsmKeyLocationInfo> locations =
+ for (OmKeyInfo keyInfo : createdKeys) {
+ List<OmKeyLocationInfo> locations =
keyInfo.getLatestVersionLocations().getLocationList();
- for (KsmKeyLocationInfo location : locations) {
+ for (OmKeyLocationInfo location : locations) {
- KeyData keyData = new KeyData(location.getBlockID());
- KeyData blockInfo = cm.getContainerManager()
- .getKeyManager().getKey(keyData);
- ContainerData containerData = cm.getContainerManager()
- .readContainer(keyData.getContainerID());
- File dataDir = ContainerUtils
- .getDataDirectory(containerData).toFile();
+ KeyValueHandler keyValueHandler = (KeyValueHandler) cm
+ .getDispatcher().getHandler(ContainerProtos.ContainerType
+ .KeyValueContainer);
+ KeyValueContainer container = (KeyValueContainer) cm.getContainerSet()
+ .getContainer(location.getBlockID().getContainerID());
+ KeyData blockInfo = keyValueHandler
+ .getKeyManager().getKey(container, location.getBlockID());
+ KeyValueContainerData containerData = (KeyValueContainerData) container
+ .getContainerData();
+ File dataDir = new File(containerData.getChunksPath());
for (ContainerProtos.ChunkInfo chunkInfo : blockInfo.getChunks()) {
File chunkFile = dataDir.toPath()
.resolve(chunkInfo.getChunkName()).toFile();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org