You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ae...@apache.org on 2019/11/08 17:22:55 UTC
[hadoop-ozone] branch HDDS-1880-Decom updated (278699e -> 88147d0)
This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a change to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from 278699e HDDS-2329. Destroy pipelines on any decommission or maintenance nodes Signed-off-by: Anu Engineer <ae...@apache.org>
add d92223a HDDS-2346. Replication manager config has wrong description (#73)
add fdd1b15 HDDS-2214. TestSCMContainerPlacementRackAware has an intermittent failure. (#6)
add 4bfad90 HDDS-2131. Optimize replication type and creation time calculation in S3 MPU list call. (#50)
add 1baa5a1 HDDS-2278. Run S3 test suite on OM HA cluste. (#27)
add 59d0786 HDDS-2334. Dummy chunk manager fails with length mismatch error. Contributed by Attila Doroszlai. (#65)
add 9475d26 HDDS-2352. Client gets internal error instead of volume not found in secure cluster (#76)
add 1c6b02a HDDS-2296. ozoneperf compose cluster shouln't start freon by default (#15)
add e16275d HDDS-2297. Enable Opentracing for new Freon tests (#16)
add c0cbdd2 HDDS-2354. SCM log is full of AllocateBlock logs. (#77)
add 4d1e811 HDDS-1228. Chunk Scanner Checkpoints (#7)
add 28170e5 HDDS-2358. Change to replication factor THREE in acceptance tests (#78)
add da01364 HDDS-2357. Add replication factor option to new Freon tests (#79)
add faf10fe HDDS-2071. Support filters in ozone insight point (#4)
add 682599e HDDS-2360. Update Ratis snapshot to d6d58d0 (#83)
add d7e2fb1 HDDS-2344. Add immutable entries in to the DoubleBuffer for Volume requests. (#71)
add e1641fb Revert "HDDS-2206. Separate handling for OMException and IOException in the Ozone Manager. Contributed by Supratim Deka (#12)"
add d1f57f8 HDDS-2341. Validate tar entry path during extraction (#72)
add 82833a0 HDDS-2361. Ozone Manager init & start command prints out unnecessary line in the beginning. (#91)
add 6a65580 HDDS-2365. Fix TestRatisPipelineProvider#testCreatePipelinesDnExclude (#84)
add 8541c56 HDDS-2368. TestOzoneManagerDoubleBufferWithDummyResponse failing intermittently (#93)
add 91e0c02 HDDS-2285. GetBlock and ReadChunk command from the client should be sent to the same datanode to re-use the same connection.
add 758bd09 concurrent hash map for dn cache
add 41b51b9 Merge pull request #40 from hanishakoneru/HDDS-2285
add 9dd2f05 HDDS-2345. Add a UT for newly added clone() in OmBucketInfo (#92)
add 19f0044 HDDS-2374. Make Ozone Readme.txt point to the Ozone websites instead of Hadoop. (#96)
add 7b043e9 HDDS-2240. Ozone Admin OM command line tool. (#9)
add aea6c37 HDDS-2292. Create Ozone specific README.md to the new hadoop-ozone re… (#55)
add e3e6dc5 HDDS-2322. DoubleBuffer flush termination and OM shutdown's after that. Make entry returned from cache a new copy. (#89)
add 86925de HDDS-2373 Move isUseRatis getFactor and getType from XCeiverClientManager (#95)
add 112d1fb Revert "HDDS-2292. Create Ozone specific README.md to the new hadoop-ozone re… (#55)"
add 27b6042 HDDS-2379. OM terminates with RocksDB error while continuously writing keys. (#100)
add d3021fb HDDS-2347 XCeiverClientGrpc's parallel use leads to NPE (#81)
add 9f75e5a HDDS-2378 - Change OZONE as string used in the code where OzoneConsts.OZONE is suitable (#103)
add 69aa475 HDDS-2366. Remove ozone.enabled as a flag and config item. (#90)
add e857643 HDDS-2292. Create Ozone specific README.md to the new hadoop-ozone repository (#106)
add 11ff08f HDDS-2381. In ExcludeList, add if not exist only. (#104)
add e16c7ed HDDS-2348. remove log4j properties for org.apache.hadoop.ozone
add b6704fd HDDS-2208. Propagate System Exceptions from OM transaction apply phase
add cbb8b66 HDDS-2219. Move all the ozone dist scripts/configs to one location
add 5b082e7 HDDS-2293. Create a new CONTRIBUTION.md for the new repository (#58)
add 629ab5d HDDS-2380. OMFileRequest should use the isExist API while checking for pre-existing files in the directory path.
add 8527a9d HDDS-1847: Datanode Kerberos principal and keytab config key looks inconsistent Contributed by christeoh.
add 5922420 HDDS-2363. Failed to create Ratis container. (#98)
add a04a5d8 HDDS-2311. Fix logic of RetryPolicy in OzoneClientSideTranslatorPB. (#51)
add 6ba7115 HDDS-2397. Fix calling cleanup for few missing tables in OM. (#112)
add feedc6a Revert "HDDS-1847: Datanode Kerberos principal and keytab config key looks inconsistent"
add c47aaa1 HDDS-2398. Remove usage of LogUtils class from ratis-common. (#113)
add e91cc32 HDDS-2272. Avoid buffer copying in GrpcReplicationClient
add 9857b0c HDDS-2291. Acceptance tests for OM HA
add 6b2cda1 HDDS-1515. Create ozone dev-support script to check hadolint violiations
add d718146 HDDS-2255. Improve Acl Handler Messages (#94)
add f162c05 HDDS-2321. Ozone Block Token verify should not apply to all datanode … (#110)
add 9565cc5 HDDS-2359. Seeking randomly in a key with more than 2 blocks of data leads to inconsistent reads (#82)
add 21e5761 HDDS-2064. Add tests for incorrect OM HA config when node ID or RPC address is not configured (#119)
add 33b374a HDDS-1643. Send hostName also part of OMRequest. (#70)
add a0dd918 HDDS-2407. Reduce log level of per-node failure in XceiverClientGrpc (#120)
add 615571e HDDS-2270. Avoid buffer copying in ContainerStateMachine.loadSnapshot/persistContainerSet (#118)
add 06c70f7 HDDS-2377. Speed up TestOzoneManagerHA#testOMRetryProxy and #testTwoOMNodesDown (#99)
add 384fee0 HDDS-2369. Fix typo in param description.
add dc3a3c3 HDDS-2327. Provide new Freon test to test Ratis pipeline with pure XceiverClientRatis
add d893cd3 HDDS-2370. Support Ozone HddsDatanodeService run as plugin with HDFS Datanode
add ab7987c HDDS-2404. Added support for Registered id as service identifier for CSR. Based on the discussion with reviewer, otherName field make more sence then registeredId.
new 88147d0 Merge branch 'master' into HDDS-1880-Decom
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
BUILDING.txt | 511 ---------------------
CONTRIBUTION.md | 206 +++++++++
README.md | 98 ++++
README.txt | 7 -
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 88 ++--
.../hadoop/hdds/scm/XceiverClientManager.java | 38 +-
.../hdds/scm/client/ContainerOperationClient.java | 111 +++--
.../hadoop/hdds/scm/storage/BlockInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 6 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 111 ++++-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 2 +-
.../scm/container/common/helpers/ExcludeList.java | 14 +-
.../hdds/security/token/BlockTokenVerifier.java | 114 ++---
.../security/token/OzoneBlockTokenIdentifier.java | 3 +-
.../hadoop/hdds/security/token/TokenVerifier.java | 13 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +
.../certificates/utils/CertificateSignRequest.java | 41 +-
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 4 +-
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/Codec.java | 6 +
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 16 +
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 9 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 5 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 3 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 -
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../common/src/main/resources/ozone-default.xml | 27 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 21 +
.../certificate/authority/TestDefaultCAServer.java | 1 +
.../certificate/authority/TestDefaultProfile.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 109 ++---
.../container/common/helpers/ContainerUtils.java | 2 +-
.../ozone/container/common/impl/ContainerData.java | 41 +-
.../common/impl/ContainerDataScanOrder.java | 57 +++
.../container/common/impl/ContainerDataYaml.java | 15 +-
.../ozone/container/common/impl/ContainerSet.java | 2 +
.../container/common/impl/HddsDispatcher.java | 41 +-
.../container/common/interfaces/Container.java | 4 +
.../transport/server/GrpcXceiverService.java | 16 -
.../server/ServerCredentialInterceptor.java | 74 ---
.../common/transport/server/XceiverServer.java | 89 ----
.../common/transport/server/XceiverServerGrpc.java | 18 +-
.../server/ratis/ContainerStateMachine.java | 34 +-
.../transport/server/ratis/XceiverServerRatis.java | 13 +-
.../container/keyvalue/KeyValueContainer.java | 16 +-
.../container/keyvalue/TarContainerPacker.java | 31 +-
.../container/keyvalue/helpers/ChunkUtils.java | 23 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 17 +-
.../container/ozoneimpl/ContainerController.java | 7 +
.../container/ozoneimpl/ContainerDataScanner.java | 31 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 7 +-
.../replication/GrpcReplicationClient.java | 39 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 2 -
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 1 -
.../ozone/container/common/SCMTestUtils.java | 22 +
.../common/impl/TestContainerDataYaml.java | 33 +-
.../container/common/impl/TestContainerSet.java | 59 ++-
.../container/common/impl/TestHddsDispatcher.java | 6 +-
.../container/common/interfaces/TestHandler.java | 2 +-
.../container/keyvalue/TestBlockManagerImpl.java | 11 +-
.../container/keyvalue/TestChunkManagerImpl.java | 113 +++--
.../container/keyvalue/TestKeyValueContainer.java | 12 +-
.../container/keyvalue/TestTarContainerPacker.java | 204 ++++++--
.../ozoneimpl/TestContainerScrubberMetrics.java | 138 +++---
hadoop-hdds/docs/content/beyond/Containers.md | 1 -
hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 4 +-
hadoop-hdds/docs/content/start/OnPrem.md | 16 -
.../server/OzoneProtocolMessageDispatcher.java | 4 +-
.../hadoop/hdds/server/events/EventQueue.java | 4 +-
.../src/main/resources/webapps/datanode/dn.js | 2 -
.../hdds/scm/container/ReplicationManager.java | 19 +-
.../metrics/SCMContainerManagerMetrics.java | 3 +-
.../algorithms/SCMContainerPlacementMetrics.java | 3 +-
.../hadoop/hdds/scm/metadata/BigIntegerCodec.java | 5 +
.../metadata/DeletedBlocksTransactionCodec.java | 5 +
.../apache/hadoop/hdds/scm/metadata/LongCodec.java | 5 +
.../hdds/scm/metadata/X509CertificateCodec.java | 5 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 42 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 3 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 7 +-
.../hdds/scm/server/SCMContainerMetrics.java | 3 +-
.../hdds/scm/server/StorageContainerManager.java | 10 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 3 -
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 3 -
.../hadoop/hdds/scm/block/TestBlockManager.java | 27 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 -
.../hadoop/hdds/scm/container/MockNodeManager.java | 5 +-
.../container/TestCloseContainerEventHandler.java | 5 +-
.../scm/container/TestSCMContainerManager.java | 30 +-
.../TestSCMContainerPlacementRackAware.java | 61 ++-
.../hdds/scm/node/TestContainerPlacement.java | 10 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 181 ++++++++
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 89 +---
.../apache/hadoop/ozone/client/ObjectStore.java | 2 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 13 +-
.../ozone/client/protocol/ClientProtocol.java | 2 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 4 +
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +
.../ozone/om/codec/OmMultipartKeyInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 4 +
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/S3SecretValueCodec.java | 5 +
.../ozone/om/codec/TokenIdentifierCodec.java | 4 +
.../hadoop/ozone/om/codec/UserVolumeInfoCodec.java | 4 +
.../ozone/om/exceptions/NotLeaderException.java | 45 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 67 ++-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 28 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 53 ++-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 56 ++-
.../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 31 ++
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 18 +
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 29 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 78 ++--
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 3 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 8 +-
.../src/main/proto/OzoneManagerProtocol.proto | 6 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 ++-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 83 ++++
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 64 +++
...TestOmBucketInfo.java => TestOmPrefixInfo.java} | 42 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 87 ++++
.../checks/{isolation.sh => hadolint.sh} | 28 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 -
.../dist/dev-support/bin/dist-layout-stitching | 35 +-
.../main/compose/ozone-hdfs/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-hdfs/docker-config | 2 -
.../dist/src/main/compose/ozone-mr/common-config | 3 +-
.../compose/{ozonescripts => ozone-om-ha-s3}/.env | 0
.../docker-compose.yaml | 10 +
.../{ozone-om-ha => ozone-om-ha-s3}/docker-config | 9 +-
.../compose/{ozones3 => ozone-om-ha-s3}/test.sh | 4 +-
.../.ssh/authorized_keys | 0
.../{ozonescripts => ozone-om-ha}/.ssh/config | 0
.../{ozonescripts => ozone-om-ha}/.ssh/environment | 0
.../{ozonescripts => ozone-om-ha}/.ssh/id_rsa | 0
.../{ozonescripts => ozone-om-ha}/.ssh/id_rsa.pub | 0
.../{ozonescripts => ozone-om-ha}/Dockerfile | 15 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 20 +-
.../src/main/compose/ozone-om-ha/docker-config | 59 ++-
.../{ozonescripts/ps.sh => ozone-om-ha/startOM.sh} | 13 +-
.../stop.sh => ozone-om-ha/stopOM.sh} | 5 +-
.../main/compose/{ozone => ozone-om-ha}/test.sh | 18 +-
.../src/main/compose/ozone-recon/docker-config | 3 +-
.../src/main/compose/ozone-topology/docker-config | 3 +-
.../dist/src/main/compose/ozone/docker-config | 3 +-
.../src/main/compose/ozoneblockade/docker-config | 1 -
.../dist/src/main/compose/ozoneperf/README.md | 20 +-
.../src/main/compose/ozoneperf/docker-compose.yaml | 9 -
.../dist/src/main/compose/ozoneperf/docker-config | 3 +-
.../datasources.yml => ozoneperf/freon-ockg.yaml} | 18 +-
.../datasources.yml => ozoneperf/freon-rk.yaml} | 18 +-
.../src/main/compose/ozones3-haproxy/docker-config | 3 +-
.../dist/src/main/compose/ozones3/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/ozones3/test.sh | 2 +
.../src/main/compose/ozonescripts/docker-config | 1 -
.../src/main/compose/ozonesecure-mr/docker-config | 3 +-
.../src/main/compose/ozonesecure/docker-config | 3 +-
.../src/main/k8s/definitions/ozone/config.yaml | 1 -
.../examples/getting-started/config-configmap.yaml | 1 -
.../k8s/examples/minikube/config-configmap.yaml | 1 -
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 -
.../main/k8s/examples/ozone/config-configmap.yaml | 1 -
.../dist/src/main/smoketest/basic/basic.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 2 +
.../dist/src/main/smoketest/omha/testOMHA.robot | 176 +++++++
.../dist/src/main/smoketest/s3/commonawslib.robot | 13 +-
.../main/smoketest/security/ozone-secure-s3.robot | 2 +-
.../conf/dn-audit-log4j2.properties | 0
.../dist/src/{main => shell}/conf/log4j.properties | 21 -
.../conf/om-audit-log4j2.properties | 0
.../conf/ozone-shell-log4j.properties | 0
.../dist/src/{main => shell}/conf/ozone-site.xml | 0
.../conf/scm-audit-log4j2.properties | 0
.../dist/src/shell/hdds}/hadoop-config.cmd | 0
.../dist/src/shell/hdds}/hadoop-config.sh | 0
.../dist/src/shell/hdds}/hadoop-daemons.sh | 0
.../dist/src/shell/hdds}/hadoop-functions.sh | 0
.../dist/src/shell/hdds}/workers.sh | 0
.../src/main/bin => dist/src/shell/ozone}/ozone | 0
.../bin => dist/src/shell/ozone}/ozone-config.sh | 0
.../bin => dist/src/shell/ozone}/start-ozone.sh | 9 -
.../bin => dist/src/shell/ozone}/stop-ozone.sh | 9 -
.../shell}/shellprofile.d/hadoop-ozone-manager.sh | 8 +-
.../src/shell}/shellprofile.d/hadoop-ozone.sh | 0
.../network-tests/src/test/compose/docker-config | 1 -
.../hadoop/ozone/insight/BaseInsightPoint.java | 56 +--
.../apache/hadoop/ozone/insight/InsightPoint.java | 6 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 26 +-
.../ozone/insight/datanode/RatisInsight.java | 5 +
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 73 +++
.../TestContainerStateManagerIntegration.java | 134 +++---
.../metrics/TestSCMContainerManagerMetrics.java | 6 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 207 ---------
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 6 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 -
.../hadoop/ozone/TestContainerOperations.java | 20 +-
.../TestContainerStateMachineIdempotency.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 3 -
.../hadoop/ozone/TestStorageContainerManager.java | 11 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 4 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 9 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../ozone/client/rpc/TestKeyInputStream.java | 104 ++++-
.../client/rpc/TestOzoneRpcClientAbstract.java | 2 +-
.../ozone/client/rpc/TestWatchForCommit.java | 10 +-
.../ozone/container/ContainerTestHelper.java | 34 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 6 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 189 ++++----
.../ozone/om/TestOzoneManagerConfiguration.java | 53 ++-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 46 +-
.../ozone/ozShell/TestOzoneDatanodeShell.java | 104 +----
.../apache/hadoop/ozone/ozShell/TestS3Shell.java | 4 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 13 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 17 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 17 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 46 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 9 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 5 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 91 +++-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../java/org/apache/hadoop/ozone/om/IOzoneAcl.java | 2 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 31 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 9 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 2 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 5 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 36 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 32 +-
.../om/request/bucket/OMBucketCreateRequest.java | 3 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 3 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 3 +-
.../ozone/om/request/file/OMFileRequest.java | 4 +-
.../S3InitiateMultipartUploadRequest.java | 3 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 23 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 6 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 6 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 6 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 6 +-
.../ozone/web/ozShell/keys/GetKeyHandler.java | 2 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 6 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 6 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 6 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 6 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 8 +-
.../hadoop/ozone/om/TestS3BucketManager.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 69 ++-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 3 +-
.../request/TestOMClientRequestWithUserInfo.java | 13 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 4 +-
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../om/request/key/TestOMKeyCreateRequest.java | 4 +-
.../s3/bucket/TestS3BucketDeleteRequest.java | 3 +-
.../s3/bucket/TestS3BucketDeleteResponse.java | 3 +-
.../s3/multipart/TestS3MultipartResponse.java | 4 +-
.../security/TestOzoneBlockTokenSecretManager.java | 48 ++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 -
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 11 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 2 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 5 +
.../s3/endpoint/TestAbortMultipartUpload.java | 7 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 11 +-
.../hadoop/ozone/s3/endpoint/TestBucketHead.java | 5 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 9 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 29 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 33 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 32 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 3 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 24 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 3 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 2 +-
.../ozone/admin/om/GetServiceRolesSubcommand.java | 2 +-
.../apache/hadoop/ozone/admin/om/package-info.java | 2 +-
.../apache/hadoop/ozone/admin/package-info.java | 2 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 40 ++
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 190 ++++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 8 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 8 +-
.../GenerateOzoneRequiredConfigurations.java | 4 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 11 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../hadoop/ozone/genesis/BenchMarkOMClient.java | 30 +-
.../ozone/genesis/BenchMarkOzoneManager.java | 33 +-
.../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 18 +-
pom.xml | 2 +-
297 files changed, 4354 insertions(+), 2645 deletions(-)
delete mode 100644 BUILDING.txt
create mode 100644 CONTRIBUTION.md
create mode 100644 README.md
delete mode 100644 README.txt
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataScanOrder.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
copy hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/{TestOmBucketInfo.java => TestOmPrefixInfo.java} (50%)
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmVolumeArgs.java
copy hadoop-ozone/dev-support/checks/{isolation.sh => hadolint.sh} (70%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-om-ha-s3}/.env (100%)
copy hadoop-ozone/dist/src/main/compose/{ozone-om-ha => ozone-om-ha-s3}/docker-compose.yaml (92%)
copy hadoop-ozone/dist/src/main/compose/{ozone-om-ha => ozone-om-ha-s3}/docker-config (77%)
copy hadoop-ozone/dist/src/main/compose/{ozones3 => ozone-om-ha-s3}/test.sh (96%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-om-ha}/.ssh/authorized_keys (100%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-om-ha}/.ssh/config (100%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-om-ha}/.ssh/environment (100%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-om-ha}/.ssh/id_rsa (100%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-om-ha}/.ssh/id_rsa.pub (100%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-om-ha}/Dockerfile (81%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts/ps.sh => ozone-om-ha/startOM.sh} (79%)
copy hadoop-ozone/dist/src/main/compose/{ozonescripts/stop.sh => ozone-om-ha/stopOM.sh} (93%)
copy hadoop-ozone/dist/src/main/compose/{ozone => ozone-om-ha}/test.sh (73%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/compose/{common/grafana/provisioning/datasources/datasources.yml => ozoneperf/freon-ockg.yaml} (77%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/compose/{common/grafana/provisioning/datasources/datasources.yml => ozoneperf/freon-rk.yaml} (78%)
mode change 100755 => 100644
create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
rename hadoop-ozone/dist/src/{main => shell}/conf/dn-audit-log4j2.properties (100%)
rename hadoop-ozone/dist/src/{main => shell}/conf/log4j.properties (84%)
rename hadoop-ozone/dist/src/{main => shell}/conf/om-audit-log4j2.properties (100%)
rename hadoop-ozone/dist/src/{main => shell}/conf/ozone-shell-log4j.properties (100%)
rename hadoop-ozone/dist/src/{main => shell}/conf/ozone-site.xml (100%)
rename hadoop-ozone/dist/src/{main => shell}/conf/scm-audit-log4j2.properties (100%)
rename {hadoop-hdds/common/src/main/bin => hadoop-ozone/dist/src/shell/hdds}/hadoop-config.cmd (100%)
rename {hadoop-hdds/common/src/main/bin => hadoop-ozone/dist/src/shell/hdds}/hadoop-config.sh (100%)
rename {hadoop-hdds/common/src/main/bin => hadoop-ozone/dist/src/shell/hdds}/hadoop-daemons.sh (100%)
rename {hadoop-hdds/common/src/main/bin => hadoop-ozone/dist/src/shell/hdds}/hadoop-functions.sh (100%)
rename {hadoop-hdds/common/src/main/bin => hadoop-ozone/dist/src/shell/hdds}/workers.sh (100%)
rename hadoop-ozone/{common/src/main/bin => dist/src/shell/ozone}/ozone (100%)
rename hadoop-ozone/{common/src/main/bin => dist/src/shell/ozone}/ozone-config.sh (100%)
rename hadoop-ozone/{common/src/main/bin => dist/src/shell/ozone}/start-ozone.sh (91%)
rename hadoop-ozone/{common/src/main/bin => dist/src/shell/ozone}/stop-ozone.sh (89%)
rename hadoop-ozone/{common/src/main => dist/src/shell}/shellprofile.d/hadoop-ozone-manager.sh (83%)
rename hadoop-ozone/{common/src/main => dist/src/shell}/shellprofile.d/hadoop-ozone.sh (100%)
create mode 100644 hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestBaseInsightPoint.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkGenerator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: Merge branch 'master' into HDDS-1880-Decom
Posted by ae...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a commit to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 88147d09b6a34f203596a26b0aef89c334d33894
Merge: 278699e ab7987c
Author: Anu Engineer <ae...@apache.org>
AuthorDate: Fri Nov 8 09:22:35 2019 -0800
Merge branch 'master' into HDDS-1880-Decom
BUILDING.txt | 511 ---------------------
CONTRIBUTION.md | 206 +++++++++
README.md | 98 ++++
README.txt | 7 -
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 88 ++--
.../hadoop/hdds/scm/XceiverClientManager.java | 38 +-
.../hdds/scm/client/ContainerOperationClient.java | 111 +++--
.../hadoop/hdds/scm/storage/BlockInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 6 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 111 ++++-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 2 +-
.../scm/container/common/helpers/ExcludeList.java | 14 +-
.../hdds/security/token/BlockTokenVerifier.java | 114 ++---
.../security/token/OzoneBlockTokenIdentifier.java | 3 +-
.../hadoop/hdds/security/token/TokenVerifier.java | 13 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +
.../certificates/utils/CertificateSignRequest.java | 41 +-
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 4 +-
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/Codec.java | 6 +
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 16 +
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 9 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 5 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 3 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 -
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../common/src/main/resources/ozone-default.xml | 27 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 21 +
.../certificate/authority/TestDefaultCAServer.java | 1 +
.../certificate/authority/TestDefaultProfile.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 109 ++---
.../container/common/helpers/ContainerUtils.java | 2 +-
.../ozone/container/common/impl/ContainerData.java | 41 +-
.../common/impl/ContainerDataScanOrder.java | 57 +++
.../container/common/impl/ContainerDataYaml.java | 15 +-
.../ozone/container/common/impl/ContainerSet.java | 2 +
.../container/common/impl/HddsDispatcher.java | 41 +-
.../container/common/interfaces/Container.java | 4 +
.../transport/server/GrpcXceiverService.java | 16 -
.../server/ServerCredentialInterceptor.java | 74 ---
.../common/transport/server/XceiverServer.java | 89 ----
.../common/transport/server/XceiverServerGrpc.java | 18 +-
.../server/ratis/ContainerStateMachine.java | 34 +-
.../transport/server/ratis/XceiverServerRatis.java | 13 +-
.../container/keyvalue/KeyValueContainer.java | 16 +-
.../container/keyvalue/TarContainerPacker.java | 31 +-
.../container/keyvalue/helpers/ChunkUtils.java | 23 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 17 +-
.../container/ozoneimpl/ContainerController.java | 7 +
.../container/ozoneimpl/ContainerDataScanner.java | 31 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 7 +-
.../replication/GrpcReplicationClient.java | 39 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 2 -
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 1 -
.../ozone/container/common/SCMTestUtils.java | 22 +
.../common/impl/TestContainerDataYaml.java | 33 +-
.../container/common/impl/TestContainerSet.java | 59 ++-
.../container/common/impl/TestHddsDispatcher.java | 6 +-
.../container/common/interfaces/TestHandler.java | 2 +-
.../container/keyvalue/TestBlockManagerImpl.java | 11 +-
.../container/keyvalue/TestChunkManagerImpl.java | 113 +++--
.../container/keyvalue/TestKeyValueContainer.java | 12 +-
.../container/keyvalue/TestTarContainerPacker.java | 204 ++++++--
.../ozoneimpl/TestContainerScrubberMetrics.java | 138 +++---
hadoop-hdds/docs/content/beyond/Containers.md | 1 -
hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 4 +-
hadoop-hdds/docs/content/start/OnPrem.md | 16 -
.../server/OzoneProtocolMessageDispatcher.java | 4 +-
.../hadoop/hdds/server/events/EventQueue.java | 4 +-
.../src/main/resources/webapps/datanode/dn.js | 2 -
.../hdds/scm/container/ReplicationManager.java | 19 +-
.../metrics/SCMContainerManagerMetrics.java | 3 +-
.../algorithms/SCMContainerPlacementMetrics.java | 3 +-
.../hadoop/hdds/scm/metadata/BigIntegerCodec.java | 5 +
.../metadata/DeletedBlocksTransactionCodec.java | 5 +
.../apache/hadoop/hdds/scm/metadata/LongCodec.java | 5 +
.../hdds/scm/metadata/X509CertificateCodec.java | 5 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 42 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 3 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 7 +-
.../hdds/scm/server/SCMContainerMetrics.java | 3 +-
.../hdds/scm/server/StorageContainerManager.java | 10 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 3 -
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 3 -
.../hadoop/hdds/scm/block/TestBlockManager.java | 27 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 -
.../hadoop/hdds/scm/container/MockNodeManager.java | 5 +-
.../container/TestCloseContainerEventHandler.java | 5 +-
.../scm/container/TestSCMContainerManager.java | 30 +-
.../TestSCMContainerPlacementRackAware.java | 61 ++-
.../hdds/scm/node/TestContainerPlacement.java | 10 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 181 ++++++++
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 89 +---
.../apache/hadoop/ozone/client/ObjectStore.java | 2 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 13 +-
.../ozone/client/protocol/ClientProtocol.java | 2 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 4 +
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +
.../ozone/om/codec/OmMultipartKeyInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 4 +
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/S3SecretValueCodec.java | 5 +
.../ozone/om/codec/TokenIdentifierCodec.java | 4 +
.../hadoop/ozone/om/codec/UserVolumeInfoCodec.java | 4 +
.../ozone/om/exceptions/NotLeaderException.java | 45 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 67 ++-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 28 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 53 ++-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 56 ++-
.../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 31 ++
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 18 +
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 29 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 78 ++--
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 3 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 8 +-
.../src/main/proto/OzoneManagerProtocol.proto | 6 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 ++-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 83 ++++
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 64 +++
...TestOmBucketInfo.java => TestOmPrefixInfo.java} | 42 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 87 ++++
.../checks/hadolint.sh} | 31 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 -
.../dist/dev-support/bin/dist-layout-stitching | 35 +-
.../main/compose/ozone-hdfs/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-hdfs/docker-config | 2 -
.../dist/src/main/compose/ozone-mr/common-config | 3 +-
.../{ozones3/test.sh => ozone-om-ha-s3/.env} | 18 +-
.../docker-compose.yaml | 10 +
.../{ozone-om-ha => ozone-om-ha-s3}/docker-config | 9 +-
.../compose/{ozones3 => ozone-om-ha-s3}/test.sh | 4 +-
.../test.sh => ozone-om-ha/.ssh/authorized_keys} | 18 +-
.../{ozones3/test.sh => ozone-om-ha/.ssh/config} | 20 +-
.../test.sh => ozone-om-ha/.ssh/environment} | 18 +-
.../dist/src/main/compose/ozone-om-ha/.ssh/id_rsa | 42 ++
.../test.sh => ozone-om-ha/.ssh/id_rsa.pub} | 18 +-
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 47 ++
.../main/compose/ozone-om-ha/docker-compose.yaml | 20 +-
.../src/main/compose/ozone-om-ha/docker-config | 59 ++-
.../{ozones3/test.sh => ozone-om-ha/startOM.sh} | 22 +-
.../{ozones3/test.sh => ozone-om-ha/stopOM.sh} | 18 +-
.../main/compose/{ozones3 => ozone-om-ha}/test.sh | 12 +-
.../src/main/compose/ozone-recon/docker-config | 3 +-
.../src/main/compose/ozone-topology/docker-config | 3 +-
.../dist/src/main/compose/ozone/docker-config | 3 +-
.../src/main/compose/ozoneblockade/docker-config | 1 -
.../dist/src/main/compose/ozoneperf/README.md | 20 +-
.../src/main/compose/ozoneperf/docker-compose.yaml | 9 -
.../dist/src/main/compose/ozoneperf/docker-config | 3 +-
.../{ozones3/test.sh => ozoneperf/freon-ockg.yaml} | 25 +-
.../{ozones3/test.sh => ozoneperf/freon-rk.yaml} | 25 +-
.../src/main/compose/ozones3-haproxy/docker-config | 3 +-
.../dist/src/main/compose/ozones3/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/ozones3/test.sh | 2 +
.../src/main/compose/ozonescripts/docker-config | 1 -
.../src/main/compose/ozonesecure-mr/docker-config | 3 +-
.../src/main/compose/ozonesecure/docker-config | 3 +-
.../src/main/k8s/definitions/ozone/config.yaml | 1 -
.../examples/getting-started/config-configmap.yaml | 1 -
.../k8s/examples/minikube/config-configmap.yaml | 1 -
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 -
.../main/k8s/examples/ozone/config-configmap.yaml | 1 -
.../dist/src/main/smoketest/basic/basic.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 2 +
.../dist/src/main/smoketest/omha/testOMHA.robot | 176 +++++++
.../dist/src/main/smoketest/s3/commonawslib.robot | 13 +-
.../main/smoketest/security/ozone-secure-s3.robot | 2 +-
.../conf/dn-audit-log4j2.properties | 0
.../dist/src/{main => shell}/conf/log4j.properties | 21 -
.../conf/om-audit-log4j2.properties | 0
.../conf/ozone-shell-log4j.properties | 0
.../dist/src/{main => shell}/conf/ozone-site.xml | 0
.../conf/scm-audit-log4j2.properties | 0
.../dist/src/shell/hdds}/hadoop-config.cmd | 0
.../dist/src/shell/hdds}/hadoop-config.sh | 0
.../dist/src/shell/hdds}/hadoop-daemons.sh | 0
.../dist/src/shell/hdds}/hadoop-functions.sh | 0
.../dist/src/shell/hdds}/workers.sh | 0
.../src/main/bin => dist/src/shell/ozone}/ozone | 0
.../bin => dist/src/shell/ozone}/ozone-config.sh | 0
.../bin => dist/src/shell/ozone}/start-ozone.sh | 9 -
.../bin => dist/src/shell/ozone}/stop-ozone.sh | 9 -
.../shell}/shellprofile.d/hadoop-ozone-manager.sh | 8 +-
.../src/shell}/shellprofile.d/hadoop-ozone.sh | 0
.../network-tests/src/test/compose/docker-config | 1 -
.../hadoop/ozone/insight/BaseInsightPoint.java | 56 +--
.../apache/hadoop/ozone/insight/InsightPoint.java | 6 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 26 +-
.../ozone/insight/datanode/RatisInsight.java | 5 +
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 73 +++
.../TestContainerStateManagerIntegration.java | 134 +++---
.../metrics/TestSCMContainerManagerMetrics.java | 6 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 207 ---------
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 6 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 -
.../hadoop/ozone/TestContainerOperations.java | 20 +-
.../TestContainerStateMachineIdempotency.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 3 -
.../hadoop/ozone/TestStorageContainerManager.java | 11 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 4 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 9 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../ozone/client/rpc/TestKeyInputStream.java | 104 ++++-
.../client/rpc/TestOzoneRpcClientAbstract.java | 2 +-
.../ozone/client/rpc/TestWatchForCommit.java | 10 +-
.../ozone/container/ContainerTestHelper.java | 34 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 6 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 189 ++++----
.../ozone/om/TestOzoneManagerConfiguration.java | 53 ++-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 46 +-
.../ozone/ozShell/TestOzoneDatanodeShell.java | 104 +----
.../apache/hadoop/ozone/ozShell/TestS3Shell.java | 4 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 13 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 17 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 17 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 46 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 9 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 5 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 91 +++-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../java/org/apache/hadoop/ozone/om/IOzoneAcl.java | 2 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 31 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 9 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 2 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 5 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 36 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 32 +-
.../om/request/bucket/OMBucketCreateRequest.java | 3 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 3 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 3 +-
.../ozone/om/request/file/OMFileRequest.java | 4 +-
.../S3InitiateMultipartUploadRequest.java | 3 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 23 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 6 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 6 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 6 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 6 +-
.../ozone/web/ozShell/keys/GetKeyHandler.java | 2 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 6 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 6 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 6 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 6 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 8 +-
.../hadoop/ozone/om/TestS3BucketManager.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 69 ++-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 3 +-
.../request/TestOMClientRequestWithUserInfo.java | 13 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 4 +-
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../om/request/key/TestOMKeyCreateRequest.java | 4 +-
.../s3/bucket/TestS3BucketDeleteRequest.java | 3 +-
.../s3/bucket/TestS3BucketDeleteResponse.java | 3 +-
.../s3/multipart/TestS3MultipartResponse.java | 4 +-
.../security/TestOzoneBlockTokenSecretManager.java | 48 ++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 -
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 11 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 2 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 5 +
.../s3/endpoint/TestAbortMultipartUpload.java | 7 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 11 +-
.../hadoop/ozone/s3/endpoint/TestBucketHead.java | 5 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 9 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 29 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 33 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 32 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 3 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 24 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 3 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 2 +-
.../ozone/admin/om/GetServiceRolesSubcommand.java | 2 +-
.../apache/hadoop/ozone/admin/om/package-info.java | 2 +-
.../apache/hadoop/ozone/admin/package-info.java | 2 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 40 ++
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 190 ++++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 8 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 8 +-
.../GenerateOzoneRequiredConfigurations.java | 4 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 11 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../hadoop/ozone/genesis/BenchMarkOMClient.java | 30 +-
.../ozone/genesis/BenchMarkOzoneManager.java | 33 +-
.../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 18 +-
pom.xml | 2 +-
297 files changed, 4435 insertions(+), 2763 deletions(-)
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
index 03dd829,63fe152..003035c
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
@@@ -29,19 -34,8 +33,15 @@@ import org.apache.hadoop.hdds.scm.net.N
import org.apache.hadoop.hdds.scm.net.NodeSchema;
import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
- import java.util.ArrayList;
- import java.util.Arrays;
- import java.util.Collection;
- import java.util.List;
-
+ import org.apache.commons.lang.StringUtils;
import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index 0000000,6f0425d..fc16e98
mode 000000,100644..100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@@ -1,0 -1,180 +1,181 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.hdds.scm.pipeline;
+
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+ import org.apache.hadoop.hdds.scm.TestUtils;
+ import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+ import org.apache.hadoop.hdds.scm.node.NodeManager;
++import org.apache.hadoop.hdds.scm.node.NodeStatus;
+ import org.junit.Assume;
+ import org.junit.Before;
+ import org.junit.Test;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Set;
+
+ import static org.apache.commons.collections.CollectionUtils.intersection;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertTrue;
+
+ /**
+ * Test for RatisPipelineProvider.
+ */
+ public class TestRatisPipelineProvider {
+
+ private static final HddsProtos.ReplicationType REPLICATION_TYPE =
+ HddsProtos.ReplicationType.RATIS;
+
+ private NodeManager nodeManager;
+ private PipelineProvider provider;
+ private PipelineStateManager stateManager;
+
+ @Before
+ public void init() throws Exception {
+ nodeManager = new MockNodeManager(true, 10);
+ stateManager = new PipelineStateManager(new OzoneConfiguration());
+ provider = new MockRatisPipelineProvider(nodeManager,
+ stateManager, new OzoneConfiguration());
+ }
+
+ private void createPipelineAndAssertions(
+ HddsProtos.ReplicationFactor factor) throws IOException {
+ Pipeline pipeline = provider.create(factor);
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+ stateManager.addPipeline(pipeline);
+
+ Pipeline pipeline1 = provider.create(factor);
+ assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE);
+ // New pipeline should not overlap with the previous created pipeline
+ assertTrue(
+ intersection(pipeline.getNodes(), pipeline1.getNodes())
+ .isEmpty());
+ stateManager.addPipeline(pipeline1);
+ }
+
+ @Test
+ public void testCreatePipelineWithFactor() throws IOException {
+ HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
+ Pipeline pipeline = provider.create(factor);
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+ stateManager.addPipeline(pipeline);
+
+ factor = HddsProtos.ReplicationFactor.ONE;
+ Pipeline pipeline1 = provider.create(factor);
+ assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE);
+ stateManager.addPipeline(pipeline1);
+ // New pipeline should overlap with the previous created pipeline,
+ // and one datanode should overlap between the two types.
+ assertEquals(1,
+ intersection(pipeline.getNodes(), pipeline1.getNodes()).size());
+ }
+
+ @Test
+ public void testCreatePipelineWithFactorThree() throws IOException {
+ createPipelineAndAssertions(HddsProtos.ReplicationFactor.THREE);
+ }
+
+ @Test
+ public void testCreatePipelineWithFactorOne() throws IOException {
+ createPipelineAndAssertions(HddsProtos.ReplicationFactor.ONE);
+ }
+
+ private List<DatanodeDetails> createListOfNodes(int nodeCount) {
+ List<DatanodeDetails> nodes = new ArrayList<>();
+ for (int i = 0; i < nodeCount; i++) {
+ nodes.add(TestUtils.randomDatanodeDetails());
+ }
+ return nodes;
+ }
+
+ @Test
+ public void testCreatePipelineWithNodes() {
+ HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
+ Pipeline pipeline =
+ provider.create(factor, createListOfNodes(factor.getNumber()));
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+
+ factor = HddsProtos.ReplicationFactor.ONE;
+ pipeline = provider.create(factor, createListOfNodes(factor.getNumber()));
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+ }
+
+ @Test
+ public void testCreatePipelinesDnExclude() throws IOException {
+ List<DatanodeDetails> healthyNodes =
- nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
++ nodeManager.getNodes(NodeStatus.inServiceHealthy());
+
+ Assume.assumeTrue(healthyNodes.size() == 8);
+
+ HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
+
+ // Use up first 3 DNs for an open pipeline.
+ List<DatanodeDetails> dns = healthyNodes.subList(0, 3);
+ addPipeline(dns, factor, Pipeline.PipelineState.OPEN, REPLICATION_TYPE);
+ Set<DatanodeDetails> membersOfOpenPipelines = new HashSet<>(dns);
+
+ // Use up next 3 DNs for a closed pipeline.
+ dns = healthyNodes.subList(3, 6);
+ addPipeline(dns, factor, Pipeline.PipelineState.CLOSED, REPLICATION_TYPE);
+ Set<DatanodeDetails> membersOfClosedPipelines = new HashSet<>(dns);
+
+ // only 2 healthy DNs left that are not part of any pipeline
+ Pipeline pipeline = provider.create(factor);
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+
+ List<DatanodeDetails> nodes = pipeline.getNodes();
+
+ assertTrue(
+ "nodes of new pipeline cannot be from open pipelines",
+ nodes.stream().noneMatch(membersOfOpenPipelines::contains));
+
+ assertTrue(
+ "at least 1 node should have been from members of closed pipelines",
+ nodes.stream().anyMatch(membersOfClosedPipelines::contains));
+ }
+
+ private static void assertPipelineProperties(
+ Pipeline pipeline, HddsProtos.ReplicationFactor expectedFactor,
+ HddsProtos.ReplicationType expectedReplicationType) {
+ assertEquals(Pipeline.PipelineState.OPEN, pipeline.getPipelineState());
+ assertEquals(expectedReplicationType, pipeline.getType());
+ assertEquals(expectedFactor, pipeline.getFactor());
+ assertEquals(expectedFactor.getNumber(), pipeline.getNodes().size());
+ }
+
+ private void addPipeline(
+ List<DatanodeDetails> dns, HddsProtos.ReplicationFactor factor,
+ Pipeline.PipelineState open, HddsProtos.ReplicationType replicationType)
+ throws IOException {
+ Pipeline openPipeline = Pipeline.newBuilder()
+ .setType(replicationType)
+ .setFactor(factor)
+ .setNodes(dns)
+ .setState(open)
+ .setId(PipelineID.randomId())
+ .build();
+
+ stateManager.addPipeline(openPipeline);
+ }
+ }
diff --cc hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
index 8649992,8c0fb03..75e2aa6
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
@@@ -25,12 -27,9 +27,10 @@@ import org.apache.hadoop.hdds.HddsUtils
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
- import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
- import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands;
import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands;
+import org.apache.hadoop.hdds.scm.cli.node.DatanodeAdminCommands;
import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java
index bada595,0000000..5cf0864
mode 100644,000000..100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java
@@@ -1,137 -1,0 +1,135 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm.node;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static junit.framework.TestCase.assertEquals;
+import static org.apache.hadoop.hdds.HddsConfigKeys.*;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.*;
+
+/**
+ * Test from the scmclient for decommission and maintenance.
+ */
+
+public class TestDecommissionAndMaintenance {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestDecommissionAndMaintenance.class);
+
+ private static int numOfDatanodes = 5;
+ private MiniOzoneCluster cluster;
+
+ private ContainerOperationClient scmClient;
+
+ @Before
+ public void setUp() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ final int interval = 100;
+
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+ interval, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(numOfDatanodes)
+ .build();
+ cluster.waitForClusterToBeReady();
- scmClient = new ContainerOperationClient(cluster
- .getStorageContainerLocationClient(),
- new XceiverClientManager(conf));
++ scmClient = new ContainerOperationClient(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testNodeCanBeDecommMaintAndRecommissioned()
+ throws IOException {
+ NodeManager nm = cluster.getStorageContainerManager().getScmNodeManager();
+
+ List<DatanodeDetails> dns = nm.getAllNodes();
+ scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0))));
+
+ // Ensure one node is decommissioning
+ List<DatanodeDetails> decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(1, decomNodes.size());
+
+ scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0))));
+
+ // Ensure zero nodes are now decommissioning
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(0, decomNodes.size());
+
+ scmClient.startMaintenanceNodes(Arrays.asList(
+ getDNHostAndPort(dns.get(0))), 10);
+
+ // None are decommissioning
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(0, decomNodes.size());
+
+ // One is in Maintenance
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(1, decomNodes.size());
+
+ scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0))));
+
+ // None are in maintenance
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(0, decomNodes.size());
+ }
+
+ private String getDNHostAndPort(DatanodeDetails dn) {
+ return dn.getHostName()+":"+dn.getPorts().get(0).getValue();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org