You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2020/01/10 16:53:49 UTC
[hadoop-ozone] branch HDDS-1880-Decom updated (692420f -> 64e25a9)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from 692420f HDDS-2671. Have NodeManager.getNodeStatus throw NodeNotFoundException (#328)
add 4732a19 HDDS-2684. Refactor common test utilities to hadoop-hdds/common (#322)
add eeb4ac7 HDDS-2242. Avoid unnecessary rpc needed to discover the pipeline leader. (#313)
add d3782c5 HDDS-2553. Sonar: Iterator.next() methods should throw NoSuchElementException
add d03fd1a HDDS-2423. Add the recover-trash command client side code
add 114ff41 HDDS-2414. Simplify robot tests with removing output greps
add 9f2818c HDDS-2680. Fix updating lastAppliedIndex in OzoneManagerStateMachine. (#316)
add a14a6bc HDDS-2675. Sonar : fix issues in PipelineStateManager (#347)
add 7095ded HDDS-2578. Handle InterruptedException in RandomKeyGenerator (#305)
add 6ee5fd1 HDDS-2539. Sonar: Fix sonar warnings in SCMContainerManager class. (#332)
add 45b80c5 HDDS-2729. Enable multilingual Hugo features in ozone docs (#357)
add 7f2acaa HDDS-2751. Wrong number of placeholders in log message (#362)
add 9d24f23 HDDS-2723. Wrap InputStream with try-with-resource in TestOzoneFSInputStream. (#361)
add 6c8b837 HDDS-2724. Sonar : Fix sonar issues in package 'org.apache.hadoop.ozo… (#354)
add 5f4202b HDDS-2669. Sonar : fix issues in scm Pipeline (#360)
add 2a91bb6 HDDS-2752. Unnecessary calls to isNoneEmpty and isAllEmpty (#364)
add 9b72208 HDDS-2630. NullPointerException in S3g. (#336)
add 4d0e264 Revert "HDDS-2704. Source tar file is not created during the relase build (#334)" (#368)
add ce6858c HDDS-2623. Expose SCMDatanodeProtocolServer RPC endpoint through Recon. (#353)
add bf23a2c HDDS-2070. Create insight point to debug one specific pipeline (#299)
add a98eec0 HDDS-2722. Let ChunkManager read/write ChunkBuffer instead of ByteBuffer (#352)
add dfd5d8e HDDS-2703. OzoneFSInputStream to support ByteBufferReadable (#345)
add fb45b20 HDDS-2739. No need to try install awscli before each test (#359)
add 39f3e68 HDDS-2652. Remove unnecessary log statement. (#371)
add e48eee6 HDDS-2709. Maven property skipShade should not skip ozonefs compilation (#342)
add 86fbec6 HDDS-2735. Let GitHub Actions run acceptance check in parallel (#358)
add d778472 HDDS-2774. Hadoop 3.1 acceptance test fails with apk command not found (#374)
add d57b868 HDDS-2776. NPE when stop recon while start recon failed (#376)
add 0cee30e HDDS-2762. Sonar warnings fixed (#373)
add b834fa4 HDDS-2755. Compare transactionID and updateID of Volume operations to avoid replaying transactions (#369)
add 20b908d HDDS-1989. Fix ApplyTransaction error handling in OzoneManagerStateMachine. (#351)
add a71c9bf HDDS-2685. Fix Rename API in BasicOzoneFileSystem (#321)
add c1dfdea HDDS-2704. Source tar file is not created during the relase build. (#380)
add b5008d0 HDDS-2777. Add bytes read statistics to Ozone FileSystem implementation (#382)
add a457616 HDDS-2786. ITestOzoneContractSeek zero byte file failures (#384)
add 2f9db71 HDDS-2788. Fix the wrong source link in docs (#388)
add c658374 HDDS-2705. Use Ozone specific LICENSE and NOTICE in the root of the git repo (#335)
add 092b1eb HDDS-2787. Use try-with-resources or close resource in finally block (#387)
add 529438a HDDS-2805. Fix sudo reset the environment variables about proxy (#392)
add 42bd150 HDDS-2812. Fix low version wget cannot resolve the proxy of https (#395)
add 87f69cb HDDS-2504. Handle InterruptedException properly (#386)
add e5559f7 HDDS-2552. Sonar: Save and reuse Random object in GenesisUtil (#404)
add 28e370e HDDS-2830. Remove unnecessary sleep in acceptance test. (#409)
add 8942888 HDDS-2806. Remove the LongCodec in the metadata package, and redundancy addCodec for Long class. (#391)
add 41910d7 HDDS-2679. Ratis ring creation might be failed with async pipeline creation. (#363)
add 7fb46e1 HDDS-2784. Read to ByteBuffer uses wrong offset (#406)
add c2c2c26 HDDS-2820. OM Ratis dir creation may fail (#403)
add f2b0651 HDDS-2813. Provide a ByteArrayCodec to CodecRegistry. (#394)
add f1a8f8f HDDS-2773. Configure Goldmark renderer
add 5f6ce75 HDDS-2814. Fail to connect s3g in docker container with network proxy
add edc9349 HDDS-2737. Insight point should provide service type to get correct config
add 3f92632 HDDS-2661. Use field based Config annotation instead of method based
add 28cefc6 HDDS-2785. Move ozonefs and tools minicluster tests to integration-test
add 9372779 HDDS-2690. Improve the command usage of audit parser tool
add 46bfd43 HDDS-2829. Deduplicate KDC docker image definition (#407)
add 7a99050 HDDS-2719. Rename S3Token to S3AuthInfo (#349)
add f57ef24 HDDS-2712. _index.md translated to Chinese (#365)
add 573ce21 HDDS-2815. Fix shell description for --start parameter of listing buckets (#402)
add 6baa1a3 HDDS-2484 New Metrics for Trash Key Lists and Fails (#408)
add e297cf9 HDDS-2826. Add initial UI of Dashboard for Recon
add 37a01da HDDS-2835. update ozone to latest ratis. (#412)
add cbf2a53 HDDS-2721. Allow wrapping list of ByteBuffers with ChunkBuffer (#378)
add cacf3d1 HDDS-2651 Make startId parameter non-mandatory while listing containers through shell command (#283)
add 0a3f0f8 HDDS-2725. start/_index.md translated to Chinese (#397)
add 8748498 HDDS-2590 Integration tests for Recon with Ozone Manager (#356)
add 34683e4 HDDS-2834. Directly read into ByteBuffer if it has array (#413)
add 245d335 HDDS-2853. NPE in OzoneContainer Start. (#419)
add 20834c5 HDDS-2827. Add initial UI of Datanodes in Recon
add 53ef7dc HDDS-2710. Add Filesystem functionality in MiniOzoneChaosCluster. (#344)
add acc3dde HDDS-2854. Add a config for the write chunk executor queue limit. (#420)
add 406428c HDDS-2188. Implement LocatedFileStatus & getFileBlockLocations to pro… (#331)
add 4b256aa HDDS-2542. Race condition between read and write stateMachineData. (#310)
add 2fa37ef HDDS-2779. Fix list volume for --start parameter. (#385)
add 046a06f HDDS-2579. Ozone client should refresh pipeline info if reads from all Datanodes fail (#372)
add 44b48b7 HDDS-2706. Add class CRLCodec - used for certificate revocation list. (#337)
add 7e6b821 HDDS-2857. Ozone Recon fails to start while ozone install (#424)
add 088d76e HDDS-2763. Handle Datanode registration in Recon. (#405)
add eceef90 HDDS-2851. Some OM Ratis config properties missing from ozone-default.xml. (#421)
add 25b4baf HDDS-2863. BindException in TestSCMRestart (#425)
new 64e25a9 Merge branch 'master' into HDDS-1880-Decom
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 6 -
.github/workflows/pr.yml | 6 -
LICENSE.txt | 53 +-
NOTICE.txt | 5 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 60 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 34 +-
.../hadoop/hdds/scm/storage/CommitWatcher.java | 23 +-
.../hdds/scm/storage/TestBlockInputStream.java | 73 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 26 +
.../hadoop/hdds/conf/OzoneConfiguration.java | 103 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 26 +-
.../ReconConfigKeys.java} | 36 +-
.../apache/hadoop/hdds/recon}/package-info.java | 4 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 16 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 30 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 7 -
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 23 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 25 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 5 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 49 +-
.../certificate/authority/DefaultCAServer.java | 4 +-
.../client/DefaultCertificateClient.java | 2 +-
.../security/x509/certificate/utils/CRLCodec.java | 197 +
.../hadoop/hdds/utils/BackgroundService.java | 10 +-
.../hdds/utils/{package-info.java => Cache.java} | 20 +-
.../hadoop/hdds/utils/ResourceLimitCache.java | 91 +
.../hadoop/hdds/utils/ResourceSemaphore.java | 170 +
.../hadoop/hdds/utils/RocksDBStoreMBean.java | 2 +-
.../org/apache/hadoop/hdds/utils/Scheduler.java | 31 +-
...rNotFoundException.java => ByteArrayCodec.java} | 21 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 1 +
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 35 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 13 +-
.../apache/hadoop/ozone/common/ChunkBuffer.java | 47 +-
.../common/ChunkBufferImplWithByteBuffer.java | 35 +-
.../common/ChunkBufferImplWithByteBufferList.java | 215 ++
.../ozone/common/IncrementalChunkBuffer.java | 29 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 1 +
.../common/src/main/resources/ozone-default.xml | 19 +-
.../hadoop/hdds/conf/SimpleConfiguration.java | 47 +-
.../hdds/conf/SimpleConfigurationParent.java} | 22 +-
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 19 +-
.../hadoop/hdds/protocol/MockDatanodeDetails.java | 125 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 74 +
.../x509/certificate/utils/TestCRLCodec.java | 163 +
.../hadoop/hdds/utils/MockGatheringChannel.java | 74 +
.../hadoop/hdds/utils/TestResourceLimitCache.java | 87 +
.../hadoop/hdds/utils/TestResourceSemaphore.java | 76 +
.../hdds/utils/db/TestTypedRDBTableStore.java | 20 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 10 +-
.../apache/hadoop/ozone/audit/package-info.java | 2 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 90 +-
.../TestChunkBufferImplWithByteBufferList.java | 80 +
.../{log4j2.properties => auditlog.properties} | 0
.../java/org/apache/hadoop/hdds/conf/Config.java | 2 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 77 +-
.../hadoop/hdds/conf/ConfigurationExample.java | 26 +-
.../conf/ConfigurationExampleGrandParent.java} | 18 +-
.../hdds/conf/ConfigurationExampleParent.java} | 19 +-
.../hadoop/hdds/conf/TestConfigFileGenerator.java | 60 +
hadoop-hdds/container-service/pom.xml | 22 +-
.../org/apache/hadoop/hdds/scm/HddsServerUtil.java | 22 +
.../common/statemachine/DatanodeConfiguration.java | 65 +-
.../common/statemachine/DatanodeStateMachine.java | 6 +-
.../common/statemachine/EndpointStateMachine.java | 42 +-
.../common/statemachine/SCMConnectionManager.java | 26 +-
.../commandhandler/CommandDispatcher.java | 2 +-
.../CreatePipelineCommandHandler.java | 46 +-
.../common/states/datanode/InitDatanodeState.java | 7 +-
.../states/endpoint/VersionEndpointTask.java | 65 +-
.../server/ratis/ContainerStateMachine.java | 62 +-
.../server/ratis/RatisServerConfiguration.java | 4 +-
.../transport/server/ratis/XceiverServerRatis.java | 32 +-
.../container/common/volume/HddsVolumeChecker.java | 2 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 30 +-
.../container/keyvalue/helpers/ChunkUtils.java | 10 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 7 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 13 +-
.../keyvalue/interfaces/ChunkManager.java | 13 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 58 +-
.../ozone/container/ContainerTestHelper.java | 87 +-
.../common/impl/TestContainerPersistence.java | 33 +-
.../TestCreatePipelineCommandHandler.java | 159 +
.../container/keyvalue/TestChunkManagerImpl.java | 18 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 9 +-
hadoop-hdds/docs/config.yaml | 13 +-
hadoop-hdds/docs/content/_index.md | 10 +-
hadoop-hdds/docs/content/_index.zh.md | 38 +
hadoop-hdds/docs/content/beyond/Containers.md | 4 +-
.../default.md => content/concept/Datanodes.zh.md} | 8 +-
hadoop-hdds/docs/content/concept/Overview.md | 2 +-
.../default.md => content/concept/_index.zh.md} | 8 +-
.../{tools/Genconf.md => interface/JavaApi.zh.md} | 12 +-
.../default.md => content/interface/S3.zh.md} | 9 +-
hadoop-hdds/docs/content/security/SecureOzone.md | 6 +-
.../default.md => content/shell/_index.zh.md} | 8 +-
.../start/{FromSource.md => FromSource.zh.md} | 0
.../start/{Kubernetes.md => Kubernetes.zh.md} | 0
.../content/start/{Minikube.md => Minikube.zh.md} | 0
.../docs/content/start/{OnPrem.md => OnPrem.zh.md} | 0
...{RunningViaDocker.md => RunningViaDocker.zh.md} | 0
...rtFromDockerHub.md => StartFromDockerHub.zh.md} | 0
hadoop-hdds/docs/content/start/_index.zh.md | 85 +
.../themes/ozonedoc/layouts/_default/section.html | 4 +
.../themes/ozonedoc/layouts/_default/single.html | 8 +-
.../docs/themes/ozonedoc/layouts/index.html | 4 +-
.../ozonedoc/layouts/partials/languages.html} | 10 +-
.../themes/ozonedoc/layouts/partials/navbar.html | 2 +-
.../org/apache/hadoop/hdds/server/ServerUtils.java | 2 +-
hadoop-hdds/pom.xml | 7 +
.../hdds/scm/container/ReplicationManager.java | 34 +-
.../hdds/scm/container/SCMContainerManager.java | 48 +-
.../scm/container/states/ContainerStateMap.java | 2 +-
.../apache/hadoop/hdds/scm/metadata/LongCodec.java | 45 -
.../hdds/scm/metadata/SCMMetadataStoreRDBImpl.java | 1 -
.../scm/pipeline/BackgroundPipelineCreator.java | 2 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 2 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 7 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 11 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 2 +-
.../scm/server/OzoneStorageContainerManager.java | 30 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 18 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 21 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 2 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 6 +
.../hdds/scm/server/StorageContainerManager.java | 5 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 5 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 74 +-
.../command/TestCommandStatusReportHandler.java | 3 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 4 +-
.../container/TestCloseContainerEventHandler.java | 4 +-
.../scm/container/TestContainerActionsHandler.java | 4 +-
.../scm/container/TestContainerStateManager.java | 12 +-
.../TestIncrementalContainerReportHandler.java | 20 +-
.../hdds/scm/container/TestReplicationManager.java | 4 +-
.../scm/container/TestSCMContainerManager.java | 5 +-
.../algorithms/TestContainerPlacementFactory.java | 4 +-
.../TestSCMContainerPlacementCapacity.java | 4 +-
.../TestSCMContainerPlacementRackAware.java | 6 +-
.../TestSCMContainerPlacementRandom.java | 4 +-
.../states/TestContainerReplicaCount.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 26 +-
.../scm/node/TestDatanodeAdminNodeDetails.java | 8 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 19 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 4 +-
.../hdds/scm/node/TestNodeReportHandler.java | 3 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 10 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 5 +-
.../scm/pipeline/TestPipelineStateManager.java | 9 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 6 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 7 +-
.../scm/server/TestSCMBlockProtocolServer.java | 4 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 8 +-
.../ozone/container/common/TestEndPoint.java | 21 +-
.../hdds/scm/cli/container/ListSubcommand.java | 4 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 5 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 9 +-
.../client/io/BlockOutputStreamEntryPool.java | 51 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 52 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 152 +-
.../ozone/client/protocol/ClientProtocol.java | 13 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 46 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 3 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 -
.../hadoop/ozone/om/exceptions/OMException.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 14 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 15 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 39 +
.../OzoneDelegationTokenSecretManager.java | 10 +-
.../ozone/security/OzoneTokenIdentifier.java | 8 +-
.../hadoop/ozone/security/acl/OzoneAclConfig.java | 25 +-
.../src/main/proto/OzoneManagerProtocol.proto | 25 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 35 +
.../org/apache/hadoop/ozone/csi/CsiServer.java | 57 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dist/dev-support/bin/dist-tar-stitching | 6 +-
.../dist/src/main/assemblies/ozone-src.xml | 4 +-
.../docker-image/docker-krb5/Dockerfile-krb5 | 2 +-
.../docker-image/docker-krb5/README.md | 0
.../docker-image/docker-krb5/kadm5.acl | 0
.../docker-image/docker-krb5/krb5.conf | 0
.../docker-image/docker-krb5/launcher.sh | 0
.../dist/src/main/compose/ozone-hdfs/docker-config | 2 +
.../main/compose/ozone-mr/hadoop27/docker-config | 2 +
.../src/main/compose/ozone-mr/hadoop27/test.sh | 4 +-
.../main/compose/ozone-mr/hadoop31/docker-config | 2 +
.../src/main/compose/ozone-mr/hadoop31/test.sh | 8 +-
.../main/compose/ozone-mr/hadoop32/docker-config | 2 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 2 +
.../src/main/compose/ozone-om-ha/docker-config | 2 +
.../src/main/compose/ozone-topology/docker-config | 2 +
.../dist/src/main/compose/ozone/docker-config | 2 +
.../src/main/compose/ozoneblockade/docker-config | 2 +
.../src/main/compose/ozones3-haproxy/docker-config | 2 +
.../src/main/compose/ozonescripts/docker-config | 2 +
.../compose/ozonesecure-mr/docker-compose.yaml | 5 +-
.../src/main/compose/ozonesecure-mr/docker-config | 2 +
.../docker-image/docker-krb5/kadm5.acl | 20 -
.../docker-image/docker-krb5/krb5.conf | 41 -
.../docker-image/docker-krb5/launcher.sh | 25 -
.../main/compose/ozonesecure/docker-compose.yaml | 5 +-
.../src/main/compose/ozonesecure/docker-config | 2 +
.../docker-image/docker-krb5/Dockerfile-krb5 | 34 -
.../ozonesecure/docker-image/docker-krb5/README.md | 34 -
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 +-
hadoop-ozone/dist/src/main/license/src/LICENSE.txt | 239 --
hadoop-ozone/dist/src/main/license/src/NOTICE.txt | 33 -
.../src/main/smoketest/basic/ozone-shell.robot | 18 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 16 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 4 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 2 +-
.../dist/src/main/smoketest/s3/__init__.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 2 +-
.../dist/src/main/smoketest/s3/buckethead.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 2 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 4 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 2 +-
.../dist/src/main/smoketest/s3/objectdelete.robot | 2 +-
.../src/main/smoketest/s3/objectmultidelete.robot | 2 +-
.../dist/src/main/smoketest/s3/objectputget.robot | 2 +-
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 16 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 103 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 20 +-
.../org/apache/hadoop/ozone/utils/LoadBucket.java | 224 +-
.../hadoop/ozone/insight/BaseInsightPoint.java | 5 +-
.../ozone/insight/BaseInsightSubCommand.java | 2 +
.../ozone/insight/ConfigurationSubCommand.java | 10 +-
.../apache/hadoop/ozone/insight/InsightPoint.java | 3 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 2 +-
.../ozone/insight/datanode/RatisInsight.java | 44 +-
.../hadoop/ozone/insight/om/KeyManagerInsight.java | 4 +-
.../hadoop/ozone/insight/om/OmProtocolInsight.java | 3 +-
.../ozone/insight/scm/EventQueueInsight.java | 4 +-
.../ozone/insight/scm/NodeManagerInsight.java | 4 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 4 +-
.../scm/ScmProtocolBlockLocationInsight.java | 3 +-
.../scm/ScmProtocolContainerLocationInsight.java | 3 +-
.../insight/scm/ScmProtocolDatanodeInsight.java | 3 +-
.../insight/scm/ScmProtocolSecurityInsight.java | 3 +-
hadoop-ozone/integration-test/pom.xml | 42 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 72 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 21 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 0
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 0
.../hadoop/fs/ozone/TestOzoneFsRenameDir.java | 16 +-
.../ozone/contract/ITestOzoneContractCreate.java | 0
.../ozone/contract/ITestOzoneContractDelete.java | 0
.../ozone/contract/ITestOzoneContractDistCp.java | 0
.../contract/ITestOzoneContractGetFileStatus.java | 0
.../fs/ozone/contract/ITestOzoneContractMkdir.java | 0
.../fs/ozone/contract/ITestOzoneContractOpen.java | 0
.../ozone/contract/ITestOzoneContractRename.java | 0
.../ozone/contract/ITestOzoneContractRootDir.java | 0
.../fs/ozone/contract/ITestOzoneContractSeek.java | 0
.../hadoop/fs/ozone/contract/OzoneContract.java | 0
.../org/apache/hadoop/fs/ozone/package-info.java | 0
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 129 +
.../scm/pipeline/TestRatisPipelineProvider.java | 11 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 94 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 18 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 +-
.../ozone/container/TestContainerReplication.java | 3 +-
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 7 +-
.../container/ozoneimpl/TestOzoneContainer.java | 15 +-
.../ozoneimpl/TestOzoneContainerRatis.java | 4 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 7 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 7 +-
.../container/server/TestContainerServer.java | 11 +-
.../server/TestSecureContainerServer.java | 8 +-
.../hadoop/ozone/freon/TestDataValidate.java | 0
.../freon/TestDataValidateWithDummyContainers.java | 0
.../TestDataValidateWithSafeByteOperations.java | 0
.../TestDataValidateWithUnsafeByteOperations.java | 0
.../freon/TestFreonWithDatanodeFastRestart.java | 0
.../ozone/freon/TestFreonWithDatanodeRestart.java | 0
.../ozone/freon/TestFreonWithPipelineDestroy.java | 0
.../ozone/freon/TestOzoneClientKeyGenerator.java | 95 +
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 0
.../hadoop/ozone/fsck/TestContainerMapper.java | 0
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 16 +-
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 0
.../org/apache/hadoop/ozone/recon/TestRecon.java | 408 +++
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 5 +-
.../src/test/resources/contract/ozone.xml | 0
hadoop-ozone/ozone-manager/pom.xml | 1 -
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 59 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 20 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 41 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 17 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 55 +-
.../ozone/om/ratis/OzoneManagerRatisSnapshot.java | 10 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 154 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 16 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 23 +
.../request/s3/bucket/S3BucketCreateRequest.java | 18 +-
.../om/request/volume/OMVolumeCreateRequest.java | 26 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 51 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 20 +
.../om/request/volume/OMVolumeSetOwnerRequest.java | 24 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 24 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 29 +-
.../response/file/OMDirectoryCreateResponse.java | 28 +-
.../response/s3/bucket/S3BucketCreateResponse.java | 6 +-
.../S3MultipartUploadCommitPartResponse.java | 71 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 2 -
.../om/response/volume/OMVolumeCreateResponse.java | 38 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 42 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 56 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 26 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../web/ozShell/bucket/ListBucketHandler.java | 3 +-
.../web/ozShell/volume/ListVolumeHandler.java | 3 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 14 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 34 +
.../apache/hadoop/ozone/om/TestTrashService.java | 125 +
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 3 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 215 ++
.../ozone/om/request/TestOMRequestUtils.java | 23 +-
.../request/volume/TestOMVolumeCreateRequest.java | 51 +-
.../request/volume/TestOMVolumeDeleteRequest.java | 46 +-
.../om/request/volume/TestOMVolumeRequest.java | 30 +
.../volume/TestOMVolumeSetOwnerRequest.java | 31 +
.../volume/TestOMVolumeSetQuotaRequest.java | 38 +-
.../ozone/om/response/TestOMResponseUtils.java | 2 +-
.../volume/TestOMVolumeCreateResponse.java | 7 +-
.../volume/TestOMVolumeDeleteResponse.java | 13 +-
.../volume/TestOMVolumeSetOwnerResponse.java | 11 +-
.../volume/TestOMVolumeSetQuotaResponse.java | 7 +-
.../TestOzoneDelegationTokenSecretManager.java | 10 +-
hadoop-ozone/ozonefs/pom.xml | 47 -
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 71 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 58 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 12 +-
.../hadoop/fs/ozone/FilteredClassLoader.java | 1 +
.../hadoop/fs/ozone/OzoneClientAdapterFactory.java | 1 -
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 55 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 204 +-
.../hadoop/fs/ozone/TestReadWriteStatistics.java | 452 +++
hadoop-ozone/pom.xml | 2 +-
hadoop-ozone/recon/pom.xml | 5 +-
.../hadoop/ozone/recon/ConfigurationProvider.java | 17 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 11 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 38 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 5 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 5 +
.../recon/recovery/ReconOmMetadataManagerImpl.java | 2 +
.../recon/scm/ReconDatanodeProtocolServer.java | 85 +
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 29 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 28 +-
.../recon/scm/ReconStorageContainerManager.java | 146 +
.../recon/{recovery => scm}/package-info.java | 2 +-
.../recon/spi/ContainerDBServiceProvider.java | 10 +
.../spi/impl/ContainerDBServiceProviderImpl.java | 13 +
.../recon/spi/impl/ReconContainerDBProvider.java | 2 -
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 12 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 79 +-
.../webapps/recon/ozone-recon-web/LICENSE | 3781 ++++++++++----------
.../webapps/recon/ozone-recon-web/README.md | 8 +
.../webapps/recon/ozone-recon-web/api/db.json | 231 ++
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +
.../webapps/recon/ozone-recon-web/package.json | 16 +-
.../recon/ozone-recon-web/public/index.html | 12 +-
.../webapps/recon/ozone-recon-web/src/App.less | 12 +
.../webapps/recon/ozone-recon-web/src/App.tsx | 64 +-
.../src/components/Breadcrumbs/Breadcrumbs.tsx | 12 +-
.../src/components/NavBar/NavBar.less | 21 +-
.../src/components/NavBar/NavBar.tsx | 28 +-
.../components/OverviewCard/OverviewCard.less} | 33 +-
.../src/components/OverviewCard/OverviewCard.tsx | 94 +
.../src/constants/breadcrumbs.constants.tsx | 5 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 18 +-
.../recon/ozone-recon-web/src/utils/common.tsx | 2 +-
.../views/ContainerBrowser/ContainerBrowser.tsx | 33 -
.../src/views/Dashboard/Dashboard.tsx | 32 -
.../src/views/Datanodes/Datanodes.less | 58 +-
.../src/views/Datanodes/Datanodes.tsx | 187 +
.../src/views/NotFound/NotFound.tsx | 2 +-
.../src/views/Overview/Overview.less | 12 +-
.../src/views/Overview/Overview.tsx | 115 +
.../webapps/recon/ozone-recon-web/yarn.lock | 3307 ++++++++++-------
hadoop-ozone/s3gateway/pom.xml | 1 -
.../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 8 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 12 +-
.../hadoop/ozone/s3/exception/OS3Exception.java | 2 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 4 +-
.../s3/header/AuthenticationHeaderParser.java | 4 +
.../ozone/s3/header/AuthorizationHeaderV4.java | 18 +-
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 114 +
hadoop-ozone/tools/pom.xml | 7 -
.../hadoop/ozone/audit/parser/AuditParser.java | 7 +-
.../audit/parser/handler/LoadCommandHandler.java | 4 +-
.../audit/parser/handler/QueryCommandHandler.java | 4 +-
.../parser/handler/TemplateCommandHandler.java | 9 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 68 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 64 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 22 +-
hadoop-ozone/upgrade/pom.xml | 1 -
pom.xml | 9 +-
409 files changed, 12285 insertions(+), 5980 deletions(-)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/{scm/pipeline/PipelineNotFoundException.java => recon/ReconConfigKeys.java} (55%)
copy {hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon}/package-info.java (90%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CRLCodec.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/{package-info.java => Cache.java} (77%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/ResourceLimitCache.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/ResourceSemaphore.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/{SequenceNumberNotFoundException.java => ByteArrayCodec.java} (71%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChunkBufferImplWithByteBufferList.java
copy hadoop-hdds/{config/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java => common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfigurationParent.java} (73%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipeline.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCRLCodec.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/MockGatheringChannel.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestResourceLimitCache.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestResourceSemaphore.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChunkBufferImplWithByteBufferList.java
rename hadoop-hdds/common/src/test/resources/{log4j2.properties => auditlog.properties} (100%)
copy hadoop-hdds/config/src/{main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java => test/java/org/apache/hadoop/hdds/conf/ConfigurationExampleGrandParent.java} (69%)
copy hadoop-hdds/config/src/{main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java => test/java/org/apache/hadoop/hdds/conf/ConfigurationExampleParent.java} (66%)
create mode 100644 hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileGenerator.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCreatePipelineCommandHandler.java
create mode 100644 hadoop-hdds/docs/content/_index.zh.md
copy hadoop-hdds/docs/{archetypes/default.md => content/concept/Datanodes.zh.md} (88%)
copy hadoop-hdds/docs/{archetypes/default.md => content/concept/_index.zh.md} (92%)
copy hadoop-hdds/docs/content/{tools/Genconf.md => interface/JavaApi.zh.md} (75%)
copy hadoop-hdds/docs/{archetypes/default.md => content/interface/S3.zh.md} (77%)
copy hadoop-hdds/docs/{archetypes/default.md => content/shell/_index.zh.md} (90%)
copy hadoop-hdds/docs/content/start/{FromSource.md => FromSource.zh.md} (100%)
copy hadoop-hdds/docs/content/start/{Kubernetes.md => Kubernetes.zh.md} (100%)
copy hadoop-hdds/docs/content/start/{Minikube.md => Minikube.zh.md} (100%)
copy hadoop-hdds/docs/content/start/{OnPrem.md => OnPrem.zh.md} (100%)
copy hadoop-hdds/docs/content/start/{RunningViaDocker.md => RunningViaDocker.zh.md} (100%)
copy hadoop-hdds/docs/content/start/{StartFromDockerHub.md => StartFromDockerHub.zh.md} (100%)
create mode 100644 hadoop-hdds/docs/content/start/_index.zh.md
copy hadoop-hdds/docs/{static/NOTES.md => themes/ozonedoc/layouts/partials/languages.html} (77%)
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/LongCodec.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/OzoneStorageContainerManager.java (54%)
rename hadoop-ozone/dist/src/main/compose/{ozonesecure-mr => common}/docker-image/docker-krb5/Dockerfile-krb5 (96%)
rename hadoop-ozone/dist/src/main/compose/{ozonesecure-mr => common}/docker-image/docker-krb5/README.md (100%)
rename hadoop-ozone/dist/src/main/compose/{ozonesecure => common}/docker-image/docker-krb5/kadm5.acl (100%)
rename hadoop-ozone/dist/src/main/compose/{ozonesecure => common}/docker-image/docker-krb5/krb5.conf (100%)
rename hadoop-ozone/dist/src/main/compose/{ozonesecure => common}/docker-image/docker-krb5/launcher.sh (100%)
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-image/docker-krb5/kadm5.acl
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-image/docker-krb5/krb5.conf
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-image/docker-krb5/launcher.sh
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
delete mode 100644 hadoop-ozone/dist/src/main/license/src/LICENSE.txt
delete mode 100644 hadoop-ozone/dist/src/main/license/src/NOTICE.txt
copy hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java (72%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java (93%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsRenameDir.java (86%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java (100%)
rename hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java (100%)
copy hadoop-ozone/{ozonefs => integration-test}/src/test/java/org/apache/hadoop/fs/ozone/package-info.java (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithSafeByteOperations.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithUnsafeByteOperations.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOzoneClientKeyGenerator.java
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java (100%)
rename hadoop-ozone/{tools => integration-test}/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestRecon.java
rename hadoop-ozone/{ozonefs => integration-test}/src/test/resources/contract/ozone.xml (100%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
create mode 100644 hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestReadWriteStatistics.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDatanodeProtocolServer.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/PendingDeleteHandler.java => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java (54%)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/IntegerCodec.java => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageConfig.java (58%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManager.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/{recovery => scm}/package-info.java (95%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/{config-overrides.js => src/components/OverviewCard/OverviewCard.less} (70%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java => hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/common.tsx (88%)
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/ContainerBrowser/ContainerBrowser.tsx
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Dashboard/Dashboard.tsx
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java => hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.less (54%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.tsx
copy hadoop-hdds/docs/themes/ozonedoc/static/js/ozonedoc.js => hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less (87%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: Merge branch 'master' into HDDS-1880-Decom
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 64e25a9fdc26b2cc6b1a933e358d8078cf3d5701
Merge: 692420f 25b4baf
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Fri Jan 10 16:52:19 2020 +0000
Merge branch 'master' into HDDS-1880-Decom
.github/workflows/post-commit.yml | 6 -
.github/workflows/pr.yml | 6 -
LICENSE.txt | 53 +-
NOTICE.txt | 5 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 60 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 34 +-
.../hadoop/hdds/scm/storage/CommitWatcher.java | 23 +-
.../hdds/scm/storage/TestBlockInputStream.java | 73 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 26 +
.../hadoop/hdds/conf/OzoneConfiguration.java | 103 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 26 +-
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 45 +-
.../org/apache/hadoop/hdds/recon/package-info.java | 18 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 16 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 30 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 7 -
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 23 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 25 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 5 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 49 +-
.../certificate/authority/DefaultCAServer.java | 4 +-
.../client/DefaultCertificateClient.java | 2 +-
.../security/x509/certificate/utils/CRLCodec.java | 197 +
.../hadoop/hdds/utils/BackgroundService.java | 10 +-
.../java/org/apache/hadoop/hdds/utils/Cache.java | 31 +-
.../hadoop/hdds/utils/ResourceLimitCache.java | 91 +
.../hadoop/hdds/utils/ResourceSemaphore.java | 170 +
.../hadoop/hdds/utils/RocksDBStoreMBean.java | 2 +-
.../org/apache/hadoop/hdds/utils/Scheduler.java | 31 +-
.../hadoop/hdds/utils/db/ByteArrayCodec.java | 39 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 1 +
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 35 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 13 +-
.../apache/hadoop/ozone/common/ChunkBuffer.java | 47 +-
.../common/ChunkBufferImplWithByteBuffer.java | 35 +-
.../common/ChunkBufferImplWithByteBufferList.java | 215 ++
.../ozone/common/IncrementalChunkBuffer.java | 29 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 1 +
.../common/src/main/resources/ozone-default.xml | 19 +-
.../hadoop/hdds/conf/SimpleConfiguration.java | 47 +-
.../hdds/conf/SimpleConfigurationParent.java | 26 +-
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 19 +-
.../hadoop/hdds/protocol/MockDatanodeDetails.java | 125 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 74 +
.../x509/certificate/utils/TestCRLCodec.java | 163 +
.../hadoop/hdds/utils/MockGatheringChannel.java | 74 +
.../hadoop/hdds/utils/TestResourceLimitCache.java | 87 +
.../hadoop/hdds/utils/TestResourceSemaphore.java | 76 +
.../hdds/utils/db/TestTypedRDBTableStore.java | 20 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 10 +-
.../apache/hadoop/ozone/audit/package-info.java | 2 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 90 +-
.../TestChunkBufferImplWithByteBufferList.java | 80 +
.../{log4j2.properties => auditlog.properties} | 0
.../java/org/apache/hadoop/hdds/conf/Config.java | 2 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 77 +-
.../hadoop/hdds/conf/ConfigurationExample.java | 26 +-
.../hdds/conf/ConfigurationExampleGrandParent.java | 37 +-
.../hdds/conf/ConfigurationExampleParent.java | 38 +-
.../hadoop/hdds/conf/TestConfigFileGenerator.java | 60 +
hadoop-hdds/container-service/pom.xml | 22 +-
.../org/apache/hadoop/hdds/scm/HddsServerUtil.java | 22 +
.../common/statemachine/DatanodeConfiguration.java | 65 +-
.../common/statemachine/DatanodeStateMachine.java | 6 +-
.../common/statemachine/EndpointStateMachine.java | 42 +-
.../common/statemachine/SCMConnectionManager.java | 26 +-
.../commandhandler/CommandDispatcher.java | 2 +-
.../CreatePipelineCommandHandler.java | 46 +-
.../common/states/datanode/InitDatanodeState.java | 7 +-
.../states/endpoint/VersionEndpointTask.java | 65 +-
.../server/ratis/ContainerStateMachine.java | 62 +-
.../server/ratis/RatisServerConfiguration.java | 4 +-
.../transport/server/ratis/XceiverServerRatis.java | 32 +-
.../container/common/volume/HddsVolumeChecker.java | 2 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 30 +-
.../container/keyvalue/helpers/ChunkUtils.java | 10 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 7 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 13 +-
.../keyvalue/interfaces/ChunkManager.java | 13 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 58 +-
.../ozone/container/ContainerTestHelper.java | 87 +-
.../common/impl/TestContainerPersistence.java | 33 +-
.../TestCreatePipelineCommandHandler.java | 159 +
.../container/keyvalue/TestChunkManagerImpl.java | 18 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 9 +-
hadoop-hdds/docs/config.yaml | 13 +-
hadoop-hdds/docs/content/_index.md | 10 +-
hadoop-hdds/docs/content/_index.zh.md | 38 +
hadoop-hdds/docs/content/beyond/Containers.md | 4 +-
.../index.html => content/concept/Datanodes.zh.md} | 27 +-
hadoop-hdds/docs/content/concept/Overview.md | 2 +-
.../index.html => content/concept/_index.zh.md} | 29 +-
.../index.html => content/interface/JavaApi.zh.md} | 27 +-
.../index.html => content/interface/S3.zh.md} | 26 +-
hadoop-hdds/docs/content/security/SecureOzone.md | 6 +-
.../index.html => content/shell/_index.zh.md} | 27 +-
hadoop-hdds/docs/content/start/FromSource.zh.md | 68 +
hadoop-hdds/docs/content/start/Kubernetes.zh.md | 53 +
hadoop-hdds/docs/content/start/Minikube.zh.md | 70 +
hadoop-hdds/docs/content/start/OnPrem.zh.md | 171 +
.../docs/content/start/RunningViaDocker.zh.md | 61 +
.../docs/content/start/StartFromDockerHub.zh.md | 111 +
hadoop-hdds/docs/content/start/_index.zh.md | 85 +
.../themes/ozonedoc/layouts/_default/section.html | 4 +
.../themes/ozonedoc/layouts/_default/single.html | 8 +-
.../docs/themes/ozonedoc/layouts/index.html | 4 +-
.../{index.html => partials/languages.html} | 27 +-
.../themes/ozonedoc/layouts/partials/navbar.html | 2 +-
.../org/apache/hadoop/hdds/server/ServerUtils.java | 2 +-
hadoop-hdds/pom.xml | 7 +
.../hdds/scm/container/ReplicationManager.java | 34 +-
.../hdds/scm/container/SCMContainerManager.java | 48 +-
.../scm/container/states/ContainerStateMap.java | 2 +-
.../apache/hadoop/hdds/scm/metadata/LongCodec.java | 45 -
.../hdds/scm/metadata/SCMMetadataStoreRDBImpl.java | 1 -
.../scm/pipeline/BackgroundPipelineCreator.java | 2 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 2 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 7 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 11 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 2 +-
.../scm/server/OzoneStorageContainerManager.java | 41 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 18 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 21 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 2 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 6 +
.../hdds/scm/server/StorageContainerManager.java | 5 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 5 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 74 +-
.../command/TestCommandStatusReportHandler.java | 3 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 4 +-
.../container/TestCloseContainerEventHandler.java | 4 +-
.../scm/container/TestContainerActionsHandler.java | 4 +-
.../scm/container/TestContainerStateManager.java | 12 +-
.../TestIncrementalContainerReportHandler.java | 20 +-
.../hdds/scm/container/TestReplicationManager.java | 4 +-
.../scm/container/TestSCMContainerManager.java | 5 +-
.../algorithms/TestContainerPlacementFactory.java | 4 +-
.../TestSCMContainerPlacementCapacity.java | 4 +-
.../TestSCMContainerPlacementRackAware.java | 6 +-
.../TestSCMContainerPlacementRandom.java | 4 +-
.../states/TestContainerReplicaCount.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 26 +-
.../scm/node/TestDatanodeAdminNodeDetails.java | 8 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 19 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 4 +-
.../hdds/scm/node/TestNodeReportHandler.java | 3 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 10 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 5 +-
.../scm/pipeline/TestPipelineStateManager.java | 9 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 6 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 7 +-
.../scm/server/TestSCMBlockProtocolServer.java | 4 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 8 +-
.../ozone/container/common/TestEndPoint.java | 21 +-
.../hdds/scm/cli/container/ListSubcommand.java | 4 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 5 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 9 +-
.../client/io/BlockOutputStreamEntryPool.java | 51 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 52 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 152 +-
.../ozone/client/protocol/ClientProtocol.java | 13 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 46 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 3 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 -
.../hadoop/ozone/om/exceptions/OMException.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 14 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 15 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 39 +
.../OzoneDelegationTokenSecretManager.java | 10 +-
.../ozone/security/OzoneTokenIdentifier.java | 8 +-
.../hadoop/ozone/security/acl/OzoneAclConfig.java | 25 +-
.../src/main/proto/OzoneManagerProtocol.proto | 25 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 35 +
.../org/apache/hadoop/ozone/csi/CsiServer.java | 57 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dist/dev-support/bin/dist-tar-stitching | 6 +-
.../dist/src/main/assemblies/ozone-src.xml | 4 +-
.../docker-image/docker-krb5/Dockerfile-krb5 | 2 +-
.../docker-image/docker-krb5/README.md | 0
.../docker-image/docker-krb5/kadm5.acl | 0
.../docker-image/docker-krb5/krb5.conf | 0
.../docker-image/docker-krb5/launcher.sh | 0
.../dist/src/main/compose/ozone-hdfs/docker-config | 2 +
.../main/compose/ozone-mr/hadoop27/docker-config | 2 +
.../src/main/compose/ozone-mr/hadoop27/test.sh | 4 +-
.../main/compose/ozone-mr/hadoop31/docker-config | 2 +
.../src/main/compose/ozone-mr/hadoop31/test.sh | 8 +-
.../main/compose/ozone-mr/hadoop32/docker-config | 2 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 2 +
.../src/main/compose/ozone-om-ha/docker-config | 2 +
.../src/main/compose/ozone-topology/docker-config | 2 +
.../dist/src/main/compose/ozone/docker-config | 2 +
.../src/main/compose/ozoneblockade/docker-config | 2 +
.../src/main/compose/ozones3-haproxy/docker-config | 2 +
.../src/main/compose/ozonescripts/docker-config | 2 +
.../compose/ozonesecure-mr/docker-compose.yaml | 5 +-
.../src/main/compose/ozonesecure-mr/docker-config | 2 +
.../docker-image/docker-krb5/kadm5.acl | 20 -
.../docker-image/docker-krb5/krb5.conf | 41 -
.../docker-image/docker-krb5/launcher.sh | 25 -
.../main/compose/ozonesecure/docker-compose.yaml | 5 +-
.../src/main/compose/ozonesecure/docker-config | 2 +
.../docker-image/docker-krb5/Dockerfile-krb5 | 34 -
.../ozonesecure/docker-image/docker-krb5/README.md | 34 -
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 +-
hadoop-ozone/dist/src/main/license/src/LICENSE.txt | 239 --
hadoop-ozone/dist/src/main/license/src/NOTICE.txt | 33 -
.../src/main/smoketest/basic/ozone-shell.robot | 18 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 16 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 4 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 2 +-
.../dist/src/main/smoketest/s3/__init__.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 2 +-
.../dist/src/main/smoketest/s3/buckethead.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 2 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 4 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 2 +-
.../dist/src/main/smoketest/s3/objectdelete.robot | 2 +-
.../src/main/smoketest/s3/objectmultidelete.robot | 2 +-
.../dist/src/main/smoketest/s3/objectputget.robot | 2 +-
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 16 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 103 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 20 +-
.../org/apache/hadoop/ozone/utils/LoadBucket.java | 224 +-
.../hadoop/ozone/insight/BaseInsightPoint.java | 5 +-
.../ozone/insight/BaseInsightSubCommand.java | 2 +
.../ozone/insight/ConfigurationSubCommand.java | 10 +-
.../apache/hadoop/ozone/insight/InsightPoint.java | 3 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 2 +-
.../ozone/insight/datanode/RatisInsight.java | 44 +-
.../hadoop/ozone/insight/om/KeyManagerInsight.java | 4 +-
.../hadoop/ozone/insight/om/OmProtocolInsight.java | 3 +-
.../ozone/insight/scm/EventQueueInsight.java | 4 +-
.../ozone/insight/scm/NodeManagerInsight.java | 4 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 4 +-
.../scm/ScmProtocolBlockLocationInsight.java | 3 +-
.../scm/ScmProtocolContainerLocationInsight.java | 3 +-
.../insight/scm/ScmProtocolDatanodeInsight.java | 3 +-
.../insight/scm/ScmProtocolSecurityInsight.java | 3 +-
hadoop-ozone/integration-test/pom.xml | 42 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 72 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 21 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 0
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 0
.../hadoop/fs/ozone/TestOzoneFsRenameDir.java | 16 +-
.../ozone/contract/ITestOzoneContractCreate.java | 0
.../ozone/contract/ITestOzoneContractDelete.java | 0
.../ozone/contract/ITestOzoneContractDistCp.java | 0
.../contract/ITestOzoneContractGetFileStatus.java | 0
.../fs/ozone/contract/ITestOzoneContractMkdir.java | 0
.../fs/ozone/contract/ITestOzoneContractOpen.java | 0
.../ozone/contract/ITestOzoneContractRename.java | 0
.../ozone/contract/ITestOzoneContractRootDir.java | 0
.../fs/ozone/contract/ITestOzoneContractSeek.java | 0
.../hadoop/fs/ozone/contract/OzoneContract.java | 0
.../org/apache/hadoop/fs/ozone}/package-info.java | 17 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 129 +
.../scm/pipeline/TestRatisPipelineProvider.java | 11 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 94 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 18 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 +-
.../ozone/container/TestContainerReplication.java | 3 +-
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 7 +-
.../container/ozoneimpl/TestOzoneContainer.java | 15 +-
.../ozoneimpl/TestOzoneContainerRatis.java | 4 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 7 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 7 +-
.../container/server/TestContainerServer.java | 11 +-
.../server/TestSecureContainerServer.java | 8 +-
.../hadoop/ozone/freon/TestDataValidate.java | 0
.../freon/TestDataValidateWithDummyContainers.java | 0
.../TestDataValidateWithSafeByteOperations.java | 0
.../TestDataValidateWithUnsafeByteOperations.java | 0
.../freon/TestFreonWithDatanodeFastRestart.java | 0
.../ozone/freon/TestFreonWithDatanodeRestart.java | 0
.../ozone/freon/TestFreonWithPipelineDestroy.java | 0
.../ozone/freon/TestOzoneClientKeyGenerator.java | 95 +
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 0
.../hadoop/ozone/fsck/TestContainerMapper.java | 0
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 16 +-
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 0
.../org/apache/hadoop/ozone/recon/TestRecon.java | 408 +++
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 5 +-
.../src/test/resources/contract/ozone.xml | 0
hadoop-ozone/ozone-manager/pom.xml | 1 -
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 59 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 20 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 41 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 17 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 55 +-
.../ozone/om/ratis/OzoneManagerRatisSnapshot.java | 10 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 154 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 16 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 23 +
.../request/s3/bucket/S3BucketCreateRequest.java | 18 +-
.../om/request/volume/OMVolumeCreateRequest.java | 26 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 51 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 20 +
.../om/request/volume/OMVolumeSetOwnerRequest.java | 24 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 24 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 29 +-
.../response/file/OMDirectoryCreateResponse.java | 28 +-
.../response/s3/bucket/S3BucketCreateResponse.java | 6 +-
.../S3MultipartUploadCommitPartResponse.java | 71 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 2 -
.../om/response/volume/OMVolumeCreateResponse.java | 38 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 42 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 56 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 26 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../web/ozShell/bucket/ListBucketHandler.java | 3 +-
.../web/ozShell/volume/ListVolumeHandler.java | 3 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 14 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 34 +
.../apache/hadoop/ozone/om/TestTrashService.java | 125 +
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 3 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 215 ++
.../ozone/om/request/TestOMRequestUtils.java | 23 +-
.../request/volume/TestOMVolumeCreateRequest.java | 51 +-
.../request/volume/TestOMVolumeDeleteRequest.java | 46 +-
.../om/request/volume/TestOMVolumeRequest.java | 30 +
.../volume/TestOMVolumeSetOwnerRequest.java | 31 +
.../volume/TestOMVolumeSetQuotaRequest.java | 38 +-
.../ozone/om/response/TestOMResponseUtils.java | 2 +-
.../volume/TestOMVolumeCreateResponse.java | 7 +-
.../volume/TestOMVolumeDeleteResponse.java | 13 +-
.../volume/TestOMVolumeSetOwnerResponse.java | 11 +-
.../volume/TestOMVolumeSetQuotaResponse.java | 7 +-
.../TestOzoneDelegationTokenSecretManager.java | 10 +-
hadoop-ozone/ozonefs/pom.xml | 47 -
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 71 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 58 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 12 +-
.../hadoop/fs/ozone/FilteredClassLoader.java | 1 +
.../hadoop/fs/ozone/OzoneClientAdapterFactory.java | 1 -
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 55 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 204 +-
.../hadoop/fs/ozone/TestReadWriteStatistics.java | 452 +++
hadoop-ozone/pom.xml | 2 +-
hadoop-ozone/recon/pom.xml | 5 +-
.../hadoop/ozone/recon/ConfigurationProvider.java | 17 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 11 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 38 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 5 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 5 +
.../recon/recovery/ReconOmMetadataManagerImpl.java | 2 +
.../recon/scm/ReconDatanodeProtocolServer.java | 85 +
.../hadoop/ozone/recon/scm/ReconNodeManager.java} | 40 +-
.../ozone/recon/scm/ReconStorageConfig.java} | 39 +-
.../recon/scm/ReconStorageContainerManager.java | 146 +
.../hadoop/ozone/recon/scm/package-info.java} | 18 +-
.../recon/spi/ContainerDBServiceProvider.java | 10 +
.../spi/impl/ContainerDBServiceProviderImpl.java | 13 +
.../recon/spi/impl/ReconContainerDBProvider.java | 2 -
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 12 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 79 +-
.../webapps/recon/ozone-recon-web/LICENSE | 3781 ++++++++++----------
.../webapps/recon/ozone-recon-web/README.md | 8 +
.../webapps/recon/ozone-recon-web/api/db.json | 231 ++
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +
.../webapps/recon/ozone-recon-web/package.json | 16 +-
.../recon/ozone-recon-web/public/index.html | 12 +-
.../webapps/recon/ozone-recon-web/src/App.less | 12 +
.../webapps/recon/ozone-recon-web/src/App.tsx | 64 +-
.../src/components/Breadcrumbs/Breadcrumbs.tsx | 12 +-
.../src/components/NavBar/NavBar.less | 21 +-
.../src/components/NavBar/NavBar.tsx | 28 +-
.../OverviewCard/OverviewCard.less} | 26 +-
.../src/components/OverviewCard/OverviewCard.tsx | 94 +
.../src/constants/breadcrumbs.constants.tsx | 5 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 18 +-
.../breadcrumbs.constants.tsx => utils/common.tsx} | 9 +-
.../{App.less => views/Datanodes/Datanodes.less} | 63 +-
.../src/views/Datanodes/Datanodes.tsx | 187 +
.../src/views/NotFound/NotFound.tsx | 2 +-
.../Overview/Overview.less} | 15 +-
.../src/views/Overview/Overview.tsx | 115 +
.../webapps/recon/ozone-recon-web/yarn.lock | 3307 ++++++++++-------
hadoop-ozone/s3gateway/pom.xml | 1 -
.../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 8 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 12 +-
.../hadoop/ozone/s3/exception/OS3Exception.java | 2 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 4 +-
.../s3/header/AuthenticationHeaderParser.java | 4 +
.../ozone/s3/header/AuthorizationHeaderV4.java | 18 +-
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 114 +
hadoop-ozone/tools/pom.xml | 7 -
.../hadoop/ozone/audit/parser/AuditParser.java | 7 +-
.../audit/parser/handler/LoadCommandHandler.java | 4 +-
.../audit/parser/handler/QueryCommandHandler.java | 4 +-
.../parser/handler/TemplateCommandHandler.java | 9 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 68 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 64 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 22 +-
hadoop-ozone/upgrade/pom.xml | 1 -
pom.xml | 9 +-
407 files changed, 12874 insertions(+), 6136 deletions(-)
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index dfb5961,bfa411d..6c7f89c
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@@ -895,23 -815,18 +883,35 @@@ public class ReplicationManager impleme
description = "Timeout for the container replication/deletion commands "
+ "sent to datanodes. After this timeout the command will be "
+ "retried.")
+ private long eventTimeout = 10 * 60 * 1000;
+
-
+ public void setInterval(long interval) {
+ this.interval = interval;
+ }
+
-
public void setEventTimeout(long eventTimeout) {
this.eventTimeout = eventTimeout;
}
++ /**
++ * The number of container replica which must be available for a node to
++ * enter maintenance.
++ */
+ @Config(key = "maintenance.replica.minimum",
+ type = ConfigType.INT,
+ defaultValue = "2",
+ tags = {SCM, OZONE},
+ description = "The minimum number of container replicas which must " +
+ " be available for a node to enter maintenance. If putting a " +
+ " node into maintenance reduces the available replicas for any " +
+ " container below this level, the node will remain in the " +
+ " entering maintenance state until a new replica is created.")
++ private int maintenanceReplicaMinimum = 2;
++
+ public void setMaintenanceReplicaMinimum(int replicaCount) {
+ this.maintenanceReplicaMinimum = replicaCount;
+ }
+
public long getInterval() {
return interval;
}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
index 54c4080,b685ba9..05ea5be
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
@@@ -19,8 -19,9 +19,8 @@@ package org.apache.hadoop.hdds.scm.cont
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
- import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
index fcb4f44,ddca0fa..ac6f6c2
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
@@@ -24,7 -24,8 +24,7 @@@ import java.util.Map
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
- import org.apache.hadoop.hdds.scm.TestUtils;
+ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
index abf7f9f,992f1c5..699c947
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
@@@ -24,7 -24,8 +24,7 @@@ import java.util.List
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
- import org.apache.hadoop.hdds.scm.TestUtils;
+ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetConstants;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
index 5edb25f,91509a0..9e8c336
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
@@@ -22,7 -22,8 +22,7 @@@ import java.util.List
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
- import org.apache.hadoop.hdds.scm.TestUtils;
+ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java
index 6c239c3,0000000..f6e2187
mode 100644,000000..100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java
@@@ -1,442 -1,0 +1,442 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.states;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
++import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
- import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ContainerReplicaCount;
+import org.junit.Before;
+import org.junit.Test;
+import java.util.*;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.OPEN;
+import static org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State
+ .DECOMMISSIONED;
+import static org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State
+ .MAINTENANCE;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Class used to test the ContainerReplicaCount class.
+ */
+public class TestContainerReplicaCount {
+
+ @Before
+ public void setup() {
+ }
+
+ @Test
+ public void testThreeHealthyReplica() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ public void testTwoHealthyReplica() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testOneHealthyReplica() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, false, 2, false);
+ }
+
+ @Test
+ public void testTwoHealthyAndInflightAdd() {
+
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 0, 3, 2);
+ validate(rcnt, false, 0, false);
+ }
+
+ @Test
+ /**
+ * This does not schedule a container to be removed, as the inFlight add may
+ * fail and then the delete would make things under-replicated. Once the add
+ * completes there will be 4 healthy and it will get taken care of then.
+ */
+ public void testThreeHealthyAndInflightAdd() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 0, 3, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ /**
+ * As the inflight delete may fail, but as it will make the the container
+ * under replicated, we go ahead and schedule another replica to be added.
+ */
+ public void testThreeHealthyAndInflightDelete() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 1, 3, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ /**
+ * This is NOT sufficiently replicated as the inflight add may fail and the
+ * inflight del could succeed, leaving only 2 healthy replicas.
+ */
+ public void testThreeHealthyAndInflightAddAndInFlightDelete() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 1, 3, 2);
+ validate(rcnt, false, 0, false);
+ }
+
+ @Test
+ public void testFourHealthyReplicas() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, true, -1, true);
+ }
+
+ @Test
+ public void testFourHealthyReplicasAndInFlightDelete() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 1, 3, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ public void testFourHealthyReplicasAndTwoInFlightDelete() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 2, 3, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testOneHealthyReplicaRepFactorOne() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 1, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ public void testOneHealthyReplicaRepFactorOneInFlightDelete() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 1, 1, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testTwoHealthyReplicaTwoInflightAdd() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 2, 0, 3, 2);
+ validate(rcnt, false, 0, false);
+ }
+
+ /**
+ * From here consider decommission replicas.
+ */
+
+ @Test
+ public void testThreeHealthyAndTwoDecommission() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED, CLOSED,
+ DECOMMISSIONED, DECOMMISSIONED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ public void testOneDecommissionedReplica() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, DECOMMISSIONED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testTwoHealthyOneDecommissionedneInFlightAdd() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, DECOMMISSIONED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 0, 3, 2);
+ validate(rcnt, false, 0, false);
+ }
+
+ @Test
+ public void testAllDecommissioned() {
+ Set<ContainerReplica> replica =
+ registerNodes(DECOMMISSIONED, DECOMMISSIONED, DECOMMISSIONED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, false, 3, false);
+ }
+
+ @Test
+ public void testAllDecommissionedRepFactorOne() {
+ Set<ContainerReplica> replica = registerNodes(DECOMMISSIONED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 1, 2);
+ validate(rcnt, false, 1, false);
+
+ }
+
+ @Test
+ public void testAllDecommissionedRepFactorOneInFlightAdd() {
+ Set<ContainerReplica> replica = registerNodes(DECOMMISSIONED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 0, 1, 2);
+ validate(rcnt, false, 0, false);
+ }
+
+ @Test
+ public void testOneHealthyOneDecommissioningRepFactorOne() {
+ Set<ContainerReplica> replica = registerNodes(DECOMMISSIONED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 1, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ /**
+ * Maintenance tests from here.
+ */
+
+ @Test
+ public void testOneHealthyTwoMaintenanceMinRepOfTwo() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testOneHealthyThreeMaintenanceMinRepOfTwo() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED,
+ MAINTENANCE, MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testOneHealthyTwoMaintenanceMinRepOfOne() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 1);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ public void testOneHealthyThreeMaintenanceMinRepOfTwoInFlightAdd() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED,
+ MAINTENANCE, MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 0, 3, 2);
+ validate(rcnt, false, 0, false);
+ }
+
+ @Test
+ public void testAllMaintenance() {
+ Set<ContainerReplica> replica =
+ registerNodes(MAINTENANCE, MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, false, 2, false);
+ }
+
+ @Test
+ /**
+ * As we have exactly 3 healthy, but then an excess of maintenance copies
+ * we ignore the over-replication caused by the maintenance copies until they
+ * come back online, and then deal with them.
+ */
+ public void testThreeHealthyTwoInMaintenance() {
+ Set<ContainerReplica> replica = registerNodes(CLOSED, CLOSED, CLOSED,
+ MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ /**
+ * This is somewhat similar to testThreeHealthyTwoInMaintenance() except now
+ * one of the maintenance copies has become healthy and we will need to remove
+ * the over-replicated healthy container.
+ */
+ public void testFourHealthyOneInMaintenance() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, CLOSED, CLOSED, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ validate(rcnt, true, -1, true);
+ }
+
+ @Test
+ public void testOneMaintenanceMinRepOfTwoRepFactorOne() {
+ Set<ContainerReplica> replica = registerNodes(MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 1, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testOneMaintenanceMinRepOfTwoRepFactorOneInFlightAdd() {
+ Set<ContainerReplica> replica = registerNodes(MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 0, 1, 2);
+ validate(rcnt, false, 0, false);
+ }
+
+ @Test
+ public void testOneHealthyOneMaintenanceRepFactorOne() {
+ Set<ContainerReplica> replica = registerNodes(MAINTENANCE, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 1, 2);
+ validate(rcnt, true, 0, false);
+ }
+
+ @Test
+ public void testTwoDecomTwoMaintenanceOneInflightAdd() {
+ Set<ContainerReplica> replica =
+ registerNodes(DECOMMISSIONED, DECOMMISSIONED, MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 1, 0, 3, 2);
+ validate(rcnt, false, 1, false);
+ }
+
+ @Test
+ public void testHealthyContainerIsHealthy() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, CLOSED);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ assertTrue(rcnt.isHealthy());
+ }
+
+ @Test
+ public void testIsHealthyWithDifferentReplicaStateNotHealthy() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, OPEN);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ assertFalse(rcnt.isHealthy());
+ }
+
+ @Test
+ public void testIsHealthyWithMaintReplicaIsHealthy() {
+ Set<ContainerReplica> replica =
+ registerNodes(CLOSED, CLOSED, MAINTENANCE, MAINTENANCE);
+ ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED);
+ ContainerReplicaCount rcnt =
+ new ContainerReplicaCount(container, replica, 0, 0, 3, 2);
+ assertTrue(rcnt.isHealthy());
+ }
+
+ private void validate(ContainerReplicaCount rcnt,
+ boolean sufficientlyReplicated, int replicaDelta,
+ boolean overReplicated) {
+ assertEquals(sufficientlyReplicated, rcnt.isSufficientlyReplicated());
+ assertEquals(overReplicated, rcnt.isOverReplicated());
+ assertEquals(replicaDelta, rcnt.additionalReplicaNeeded());
+ }
+
+ private Set<ContainerReplica> registerNodes(
+ ContainerReplicaProto.State... states) {
+ Set<ContainerReplica> replica = new HashSet<>();
+ for (ContainerReplicaProto.State s : states) {
- DatanodeDetails dn = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ replica.add(new ContainerReplica.ContainerReplicaBuilder()
+ .setContainerID(new ContainerID(1))
+ .setContainerState(s)
+ .setDatanodeDetails(dn)
+ .setOriginNodeId(dn.getUuid())
+ .setSequenceId(1)
+ .build());
+ }
+ return replica;
+ }
+
+ private ContainerInfo createContainer(HddsProtos.LifeCycleState state) {
+ return new ContainerInfo.Builder()
+ .setContainerID(new ContainerID(1).getId())
+ .setState(state)
+ .build();
+ }
+}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java
index f3b7d8f,0000000..cb78703
mode 100644,000000..100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java
@@@ -1,532 -1,0 +1,532 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
++import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
- import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.*;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.mockito.Mockito.reset;
+
+/**
+ * Tests to ensure the DatanodeAdminMonitor is working correctly. This class
+ * uses mocks or basic implementations of the key classes outside of the
+ * datanodeAdminMonitor to allow it to be tested in isolation.
+ */
+public class TestDatanodeAdminMonitor {
+
+ private SimpleMockNodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private DatanodeAdminMonitorImpl monitor;
+ private DatanodeAdminHandler startAdminHandler;
+ private ReplicationManager repManager;
+ private EventQueue eventQueue;
+
+ @Before
+ public void setup() throws IOException, AuthenticationException {
+ conf = new OzoneConfiguration();
+
+ eventQueue = new EventQueue();
+ startAdminHandler = new DatanodeAdminHandler();
+ eventQueue.addHandler(SCMEvents.START_ADMIN_ON_NODE, startAdminHandler);
+
+ nodeManager = new SimpleMockNodeManager();
+
+ repManager = Mockito.mock(ReplicationManager.class);
+
+ monitor =
+ new DatanodeAdminMonitorImpl(conf, eventQueue, nodeManager, repManager);
+ }
+
+ @After
+ public void teardown() {
+ }
+
+ @Test
+ public void testNodeCanBeQueuedAndCancelled() {
- DatanodeDetails dn = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ monitor.startMonitoring(dn, 0);
+ assertEquals(1, monitor.getPendingCount());
+
+ monitor.stopMonitoring(dn);
+ assertEquals(0, monitor.getPendingCount());
+ assertEquals(1, monitor.getCancelledCount());
+
+ monitor.startMonitoring(dn, 0);
+ assertEquals(1, monitor.getPendingCount());
+ assertEquals(0, monitor.getCancelledCount());
+ }
+
+ /**
+ * In this test we ensure there are some pipelines for the node being
+ * decommissioned, but there are no containers. Therefore the workflow
+ * must wait until the pipelines have closed before completing the flow.
+ */
+ @Test
+ public void testClosePipelinesEventFiredWhenAdminStarted()
+ throws NodeNotFoundException{
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY));
+ // Ensure the node has some pipelines
+ nodeManager.setPipelines(dn1, 2);
+ // Add the node to the monitor
+ monitor.startMonitoring(dn1, 0);
+ monitor.run();
+ // Ensure a StartAdmin event was fired
+ eventQueue.processAll(20000);
+ assertEquals(1, startAdminHandler.getInvocation());
+ // Ensure a node is now tracked for decommission
+ assertEquals(1, monitor.getTrackedNodeCount());
+ // Ensure the node remains decommissioning
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ // Run the monitor again, and it should remain decommissioning
+ monitor.run();
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+
+ // Clear the pipelines and the node should transition to DECOMMISSIONED
+ nodeManager.setPipelines(dn1, 0);
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ /**
+ * In this test, there are no open pipelines and no containers on the node.
+ * Therefore, we expect the decommission flow to finish on the first run
+ * on the monitor, leaving zero nodes tracked and the node in DECOMMISSIONED
+ * state.
+ */
+ @Test
+ public void testDecommissionNodeTransitionsToCompleteWhenNoContainers()
+ throws NodeNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY));
+
+ // Add the node to the monitor. By default we have zero pipelines and
+ // zero containers in the test setup, so the node should immediately
+ // transition to COMPLETED state
+ monitor.startMonitoring(dn1, 0);
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ NodeStatus newStatus = nodeManager.getNodeStatus(dn1);
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED,
+ newStatus.getOperationalState());
+ }
+
+ @Test
+ public void testDecommissionNodeWaitsForContainersToReplicate()
+ throws NodeNotFoundException, ContainerNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY));
+
+ nodeManager.setContainers(dn1, generateContainers(3));
+ // Mock Replication Manager to return ContainerReplicaCount's which
+ // always have a DECOMMISSIONED replica.
+ mockGetContainerReplicaCount(
+ HddsProtos.LifeCycleState.CLOSED,
+ ContainerReplicaProto.State.DECOMMISSIONED,
+ ContainerReplicaProto.State.CLOSED,
+ ContainerReplicaProto.State.CLOSED);
+
+ // Run the monitor for the first time and the node will transition to
+ // REPLICATE_CONTAINERS as there are no pipelines to close.
+ monitor.startMonitoring(dn1, 0);
+ monitor.run();
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+
+ // Running the monitor again causes it to remain DECOMMISSIONING
+ // as nothing has changed.
+ monitor.run();
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ assertEquals(0, node.getSufficientlyReplicatedContainers());
+ assertEquals(0, node.getUnHealthyContainers());
+ assertEquals(3, node.getUnderReplicatedContainers());
+
+ // Now change the replicationManager mock to return 3 CLOSED replicas
+ // and the node should complete the REPLICATE_CONTAINERS step, moving to
+ // complete which will end the decommission workflow
+ mockGetContainerReplicaCount(
+ HddsProtos.LifeCycleState.CLOSED,
+ ContainerReplicaProto.State.CLOSED,
+ ContainerReplicaProto.State.CLOSED,
+ ContainerReplicaProto.State.CLOSED);
+
+ monitor.run();
+
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(3, node.getSufficientlyReplicatedContainers());
+ assertEquals(0, node.getUnHealthyContainers());
+ assertEquals(0, node.getUnderReplicatedContainers());
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ @Test
+ public void testDecommissionAbortedWhenNodeInUnexpectedState()
+ throws NodeNotFoundException, ContainerNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY));
+
+ nodeManager.setContainers(dn1, generateContainers(3));
+ mockGetContainerReplicaCount(
+ HddsProtos.LifeCycleState.CLOSED,
+ ContainerReplicaProto.State.DECOMMISSIONED,
+ ContainerReplicaProto.State.CLOSED,
+ ContainerReplicaProto.State.CLOSED);
+
+ // Add the node to the monitor, it should have 3 under-replicated containers
+ // after the first run
+ monitor.startMonitoring(dn1, 0);
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ assertEquals(3, node.getUnderReplicatedContainers());
+
+ // Set the node to dead, and then the workflow should get aborted, setting
+ // the node state back to IN_SERVICE on the next run.
+ nodeManager.setNodeStatus(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.IN_SERVICE,
+ HddsProtos.NodeState.HEALTHY));
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ @Test
+ public void testDecommissionAbortedWhenNodeGoesDead()
+ throws NodeNotFoundException, ContainerNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY));
+
+ nodeManager.setContainers(dn1, generateContainers(3));
+ mockGetContainerReplicaCount(
+ HddsProtos.LifeCycleState.CLOSED,
+ ContainerReplicaProto.State.DECOMMISSIONED,
+ ContainerReplicaProto.State.CLOSED,
+ ContainerReplicaProto.State.CLOSED);
+
+ // Add the node to the monitor, it should have 3 under-replicated containers
+ // after the first run
+ monitor.startMonitoring(dn1, 0);
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ assertEquals(3, node.getUnderReplicatedContainers());
+
+ // Set the node to dead, and then the workflow should get aborted, setting
+ // the node state back to IN_SERVICE.
+ nodeManager.setNodeStatus(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.DEAD));
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ @Test
+ public void testMaintenanceWaitsForMaintenanceToComplete()
+ throws NodeNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY));
+
+ // Add the node to the monitor, it should transiting to
+ // IN_MAINTENANCE as there are no containers to replicate.
+ monitor.startMonitoring(dn1, 1);
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertEquals(0, node.getUnderReplicatedContainers());
+ assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance());
+
+ // Running the monitor again causes the node to remain in maintenance
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance());
+
+ // Set the maintenance end time to a time in the past and then the node
+ // should complete the workflow and transition to IN_SERVICE
+ node.setMaintenanceEnd(-1);
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ @Test
+ public void testMaintenanceEndsClosingPipelines()
+ throws NodeNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY));
+ // Ensure the node has some pipelines
+ nodeManager.setPipelines(dn1, 2);
+ // Add the node to the monitor
+ monitor.startMonitoring(dn1, 1);
+ monitor.run();
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ assertTrue(nodeManager.getNodeStatus(dn1).isEnteringMaintenance());
+
+ // Set the maintenance end time to the past and the node should complete
+ // the workflow and return to IN_SERVICE
+ node.setMaintenanceEnd(-1);
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ @Test
+ public void testMaintenanceEndsWhileReplicatingContainers()
+ throws ContainerNotFoundException, NodeNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY));
+
+ nodeManager.setContainers(dn1, generateContainers(3));
+ mockGetContainerReplicaCount(
+ HddsProtos.LifeCycleState.CLOSED,
+ ContainerReplicaProto.State.MAINTENANCE,
+ ContainerReplicaProto.State.MAINTENANCE,
+ ContainerReplicaProto.State.MAINTENANCE);
+
+ // Add the node to the monitor, it should transiting to
+ // REPLICATE_CONTAINERS as the containers are under-replicated for
+ // maintenance.
+ monitor.startMonitoring(dn1, 1);
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertTrue(nodeManager.getNodeStatus(dn1).isEnteringMaintenance());
+ assertEquals(3, node.getUnderReplicatedContainers());
+
+ node.setMaintenanceEnd(-1);
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ @Test
+ public void testDeadMaintenanceNodeDoesNotAbortWorkflow()
+ throws NodeNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY));
+
+ // Add the node to the monitor, it should transiting to
+ // AWAIT_MAINTENANCE_END as there are no under-replicated containers.
+ monitor.startMonitoring(dn1, 1);
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance());
+ assertEquals(0, node.getUnderReplicatedContainers());
+
+ // Set the node dead and ensure the workflow does not end
+ NodeStatus status = nodeManager.getNodeStatus(dn1);
+ nodeManager.setNodeStatus(dn1, new NodeStatus(
+ status.getOperationalState(), HddsProtos.NodeState.DEAD));
+
+ // Running the monitor again causes the node to remain in maintenance
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance());
+ }
+
+ @Test
+ public void testCancelledNodesMovedToInService()
+ throws NodeNotFoundException {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
+ nodeManager.register(dn1,
+ new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY));
+
+ // Add the node to the monitor, it should transiting to
+ // AWAIT_MAINTENANCE_END as there are no under-replicated containers.
+ monitor.startMonitoring(dn1, 1);
+ monitor.run();
+ assertEquals(1, monitor.getTrackedNodeCount());
+ DatanodeAdminNodeDetails node = getFirstTrackedNode();
+ assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance());
+ assertEquals(0, node.getUnderReplicatedContainers());
+
+ // Now cancel the node and run the monitor, the node should be IN_SERVICE
+ monitor.stopMonitoring(dn1);
+ monitor.run();
+ assertEquals(0, monitor.getTrackedNodeCount());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dn1).getOperationalState());
+ }
+
+ /**
+ * Generate a set of ContainerID, starting from an ID of zero up to the given
+ * count minus 1.
+ * @param count The number of ContainerID objects to generate.
+ * @return A Set of ContainerID objects.
+ */
+ private Set<ContainerID> generateContainers(int count) {
+ Set<ContainerID> containers = new HashSet<>();
+ for (int i=0; i<count; i++) {
+ containers.add(new ContainerID(i));
+ }
+ return containers;
+ }
+
+ /**
+ * Create a ContainerReplicaCount object, including a container with the
+ * requested ContainerID and state, along with a set of replicas of the given
+ * states.
+ * @param containerID The ID of the container to create an included
+ * @param containerState The state of the container
+ * @param states Create a replica for each of the given states.
+ * @return A ContainerReplicaCount containing the generated container and
+ * replica set
+ */
+ private ContainerReplicaCount generateReplicaCount(ContainerID containerID,
+ HddsProtos.LifeCycleState containerState,
+ ContainerReplicaProto.State... states) {
+ Set<ContainerReplica> replicas = new HashSet<>();
+ for (ContainerReplicaProto.State s : states) {
+ replicas.add(generateReplica(containerID, s));
+ }
+ ContainerInfo container = new ContainerInfo.Builder()
+ .setContainerID(containerID.getId())
+ .setState(containerState)
+ .build();
+
+ return new ContainerReplicaCount(container, replicas, 0, 0, 3, 2);
+ }
+
+ /**
+ * Generate a new ContainerReplica with the given containerID and State.
+ * @param containerID The ID the replica is associated with
+ * @param state The state of the generated replica.
+ * @return A containerReplica with the given ID and state
+ */
+ private ContainerReplica generateReplica(ContainerID containerID,
+ ContainerReplicaProto.State state) {
+ return ContainerReplica.newBuilder()
+ .setContainerState(state)
+ .setContainerID(containerID)
+ .setSequenceId(1)
- .setDatanodeDetails(TestUtils.randomDatanodeDetails())
++ .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails())
+ .build();
+ }
+
+ /**
+ * Helper method to get the first node from the set of trackedNodes within
+ * the monitor.
+ * @return DatanodeAdminNodeDetails for the first tracked node found.
+ */
+ private DatanodeAdminNodeDetails getFirstTrackedNode() {
+ return
+ monitor.getTrackedNodes().toArray(new DatanodeAdminNodeDetails[0])[0];
+ }
+
+ /**
+ * The only interaction the DatanodeAdminMonitor has with the
+ * ReplicationManager, is to request a ContainerReplicaCount object for each
+ * container on nodes being deocmmissioned or moved to maintenance. This
+ * method mocks that interface to return a ContainerReplicaCount with a
+ * container in the given containerState and a set of replias in the given
+ * replicaStates.
+ * @param containerState
+ * @param replicaStates
+ * @throws ContainerNotFoundException
+ */
+ private void mockGetContainerReplicaCount(
+ HddsProtos.LifeCycleState containerState,
+ ContainerReplicaProto.State... replicaStates)
+ throws ContainerNotFoundException {
+ reset(repManager);
+ Mockito.when(repManager.getContainerReplicaCount(
+ Mockito.any(ContainerID.class)))
+ .thenAnswer(invocation ->
+ generateReplicaCount((ContainerID)invocation.getArguments()[0],
+ containerState, replicaStates));
+ }
+
+ /**
+ * This simple internal class is used to track and handle any DatanodeAdmin
+ * events fired by the DatanodeAdminMonitor during tests.
+ */
+ private class DatanodeAdminHandler implements
+ EventHandler<DatanodeDetails> {
+
+ private AtomicInteger invocation = new AtomicInteger(0);
+
+ @Override
+ public void onMessage(final DatanodeDetails dn,
+ final EventPublisher publisher) {
+ invocation.incrementAndGet();
+ }
+
+ public int getInvocation() {
+ return invocation.get();
+ }
+ }
+}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java
index c5310b9,0000000..7a813a9
mode 100644,000000..100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java
@@@ -1,81 -1,0 +1,81 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
- import org.apache.hadoop.hdds.scm.TestUtils;
++import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.TestCase.*;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+
+/**
+ * Tests to validate the DatanodeAdminNodeDetails class.
+ */
+public class TestDatanodeAdminNodeDetails {
+
+ private OzoneConfiguration conf;
+
+ @Before
+ public void setup() {
+ conf = new OzoneConfiguration();
+ }
+
+ @After
+ public void teardown() {
+ }
+
+ @Test
+ public void testEqualityBasedOnDatanodeDetails() {
- DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
- DatanodeDetails dn2 = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
++ DatanodeDetails dn2 = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeAdminNodeDetails details1 =
+ new DatanodeAdminNodeDetails(dn1, 0);
+ DatanodeAdminNodeDetails details2 =
+ new DatanodeAdminNodeDetails(dn2, 0);
+
+ assertNotEquals(details1, details2);
+ assertEquals(details1,
+ new DatanodeAdminNodeDetails(dn1, 0));
+ assertNotEquals(details1, dn1);
+ }
+
+
+
+ @Test
+ public void testMaintenanceEnd() {
- DatanodeDetails dn = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ // End in zero hours - should never end.
+ DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, 0);
+ assertFalse(details.shouldMaintenanceEnd());
+
+ // End 1 hour - maintenance should not end yet.
+ details.setMaintenanceEnd(1);
+ assertFalse(details.shouldMaintenanceEnd());
+
+ // End 1 hour ago - maintenance should end.
+ details.setMaintenanceEnd(-1);
+ assertTrue(details.shouldMaintenanceEnd());
+ }
+
+}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
index daf6731,0000000..df62438
mode 100644,000000..100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java
@@@ -1,292 -1,0 +1,292 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
++import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.HddsTestUtils;
- import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Before;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.List;
+import java.util.UUID;
+import java.util.Arrays;
+import java.util.ArrayList;
+import static junit.framework.TestCase.assertEquals;
+import static org.assertj.core.api.Fail.fail;
+
+/**
+ * Unit tests for the decommision manager.
+ */
+
+public class TestNodeDecommissionManager {
+
+ private NodeDecommissionManager decom;
+ private StorageContainerManager scm;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private String storageDir;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+ storageDir = GenericTestUtils.getTempPath(
+ TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ nodeManager = createNodeManager(conf);
+ decom = new NodeDecommissionManager(
+ conf, nodeManager, null, null, null);
+ }
+
+ @Test
+ public void testHostStringsParseCorrectly()
+ throws InvalidHostStringException {
+ NodeDecommissionManager.HostDefinition def =
+ new NodeDecommissionManager.HostDefinition("foobar");
+ assertEquals("foobar", def.getHostname());
+ assertEquals(-1, def.getPort());
+
+ def = new NodeDecommissionManager.HostDefinition(" foobar ");
+ assertEquals("foobar", def.getHostname());
+ assertEquals(-1, def.getPort());
+
+ def = new NodeDecommissionManager.HostDefinition("foobar:1234");
+ assertEquals("foobar", def.getHostname());
+ assertEquals(1234, def.getPort());
+
+ def = new NodeDecommissionManager.HostDefinition(
+ "foobar.mycompany.com:1234");
+ assertEquals("foobar.mycompany.com", def.getHostname());
+ assertEquals(1234, def.getPort());
+
+ try {
+ def = new NodeDecommissionManager.HostDefinition("foobar:abcd");
+ fail("InvalidHostStringException should have been thrown");
+ } catch (InvalidHostStringException e) {
+ }
+ }
+
+ @Test
+ public void testAnyInvalidHostThrowsException()
+ throws InvalidHostStringException{
+ List<DatanodeDetails> dns = generateDatanodes();
+
+ // Try to decommission a host that does exist, but give incorrect port
+ try {
+ decom.decommissionNodes(Arrays.asList(dns.get(1).getIpAddress()+":10"));
+ fail("InvalidHostStringException expected");
+ } catch (InvalidHostStringException e) {
+ }
+
+ // Try to decommission a host that does not exist
+ try {
+ decom.decommissionNodes(Arrays.asList("123.123.123.123"));
+ fail("InvalidHostStringException expected");
+ } catch (InvalidHostStringException e) {
+ }
+
+ // Try to decommission a host that does exist and a host that does not
+ try {
+ decom.decommissionNodes(Arrays.asList(
+ dns.get(1).getIpAddress(), "123,123,123,123"));
+ fail("InvalidHostStringException expected");
+ } catch (InvalidHostStringException e) {
+ }
+
+ // Try to decommission a host with many DNs on the address with no port
+ try {
+ decom.decommissionNodes(Arrays.asList(
+ dns.get(0).getIpAddress()));
+ fail("InvalidHostStringException expected");
+ } catch (InvalidHostStringException e) {
+ }
+
+ // Try to decommission a host with many DNs on the address with a port
+ // that does not exist
+ try {
+ decom.decommissionNodes(Arrays.asList(
+ dns.get(0).getIpAddress()+":10"));
+ fail("InvalidHostStringException expected");
+ } catch (InvalidHostStringException e) {
+ }
+ }
+
+ @Test
+ public void testNodesCanBeDecommissionedAndRecommissioned()
+ throws InvalidHostStringException, NodeNotFoundException {
+ List<DatanodeDetails> dns = generateDatanodes();
+
+ // Decommission 2 valid nodes
+ decom.decommissionNodes(Arrays.asList(dns.get(1).getIpAddress(),
+ dns.get(2).getIpAddress()));
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dns.get(1)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dns.get(2)).getOperationalState());
+
+ // Running the command again gives no error - nodes already decommissioning
+ // are silently ignored.
+ decom.decommissionNodes(Arrays.asList(dns.get(1).getIpAddress(),
+ dns.get(2).getIpAddress()));
+
+ // Attempt to decommission dn(10) which has multiple hosts on the same IP
+ // and we hardcoded ports to 3456, 4567, 5678
+ DatanodeDetails multiDn = dns.get(10);
+ String multiAddr =
+ multiDn.getIpAddress()+":"+multiDn.getPorts().get(0).getValue();
+ decom.decommissionNodes(Arrays.asList(multiAddr));
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(multiDn).getOperationalState());
+
+ // Recommission all 3 hosts
+ decom.recommissionNodes(Arrays.asList(
+ multiAddr, dns.get(1).getIpAddress(), dns.get(2).getIpAddress()));
+ decom.getMonitor().run();
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dns.get(1)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dns.get(2)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dns.get(10)).getOperationalState());
+ }
+
+ @Test
+ public void testNodesCanBePutIntoMaintenanceAndRecommissioned()
+ throws InvalidHostStringException, NodeNotFoundException {
+ List<DatanodeDetails> dns = generateDatanodes();
+
+ // Put 2 valid nodes into maintenance
+ decom.startMaintenanceNodes(Arrays.asList(dns.get(1).getIpAddress(),
+ dns.get(2).getIpAddress()), 100);
+ assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ nodeManager.getNodeStatus(dns.get(1)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ nodeManager.getNodeStatus(dns.get(2)).getOperationalState());
+
+ // Running the command again gives no error - nodes already decommissioning
+ // are silently ignored.
+ decom.startMaintenanceNodes(Arrays.asList(dns.get(1).getIpAddress(),
+ dns.get(2).getIpAddress()), 100);
+
+ // Attempt to decommission dn(10) which has multiple hosts on the same IP
+ // and we hardcoded ports to 3456, 4567, 5678
+ DatanodeDetails multiDn = dns.get(10);
+ String multiAddr =
+ multiDn.getIpAddress()+":"+multiDn.getPorts().get(0).getValue();
+ decom.startMaintenanceNodes(Arrays.asList(multiAddr), 100);
+ assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ nodeManager.getNodeStatus(multiDn).getOperationalState());
+
+ // Recommission all 3 hosts
+ decom.recommissionNodes(Arrays.asList(
+ multiAddr, dns.get(1).getIpAddress(), dns.get(2).getIpAddress()));
+ decom.getMonitor().run();
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dns.get(1)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dns.get(2)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ nodeManager.getNodeStatus(dns.get(10)).getOperationalState());
+ }
+
+ @Test
+ public void testNodesCannotTransitionFromDecomToMaint() throws Exception {
+ List<DatanodeDetails> dns = generateDatanodes();
+
+ // Put 1 node into maintenance and another into decom
+ decom.startMaintenance(dns.get(1), 100);
+ decom.startDecommission(dns.get(2));
+ assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ nodeManager.getNodeStatus(dns.get(1)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dns.get(2)).getOperationalState());
+
+ // Try to go from maint to decom:
+ try {
+ decom.startDecommission(dns.get(1));
+ fail("Expected InvalidNodeStateException");
+ } catch (InvalidNodeStateException e) {
+ }
+
+ // Try to go from decom to maint:
+ try {
+ decom.startMaintenance(dns.get(2), 100);
+ fail("Expected InvalidNodeStateException");
+ } catch (InvalidNodeStateException e) {
+ }
+
+ // Ensure the states are still as before
+ assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ nodeManager.getNodeStatus(dns.get(1)).getOperationalState());
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ nodeManager.getNodeStatus(dns.get(2)).getOperationalState());
+ }
+
+
+
+ private SCMNodeManager createNodeManager(OzoneConfiguration config)
+ throws IOException, AuthenticationException {
+ scm = HddsTestUtils.getScm(config);
+ return (SCMNodeManager) scm.getScmNodeManager();
+ }
+
+ /**
+ * Generate a list of random DNs and return the list. A total of 11 DNs will
+ * be generated and registered with the node manager. Index 0 and 10 will
+ * have the same IP and host and the rest will have unique IPs and Hosts.
+ * The DN at index 10, has 3 hard coded ports of 3456, 4567, 5678. All other
+ * DNs will have ports set to 0.
+ * @return The list of DatanodeDetails Generated
+ */
+ private List<DatanodeDetails> generateDatanodes() {
+ List<DatanodeDetails> dns = new ArrayList<>();
+ for (int i=0; i<10; i++) {
- DatanodeDetails dn = TestUtils.randomDatanodeDetails();
++ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ dns.add(dn);
+ nodeManager.register(dn, null, null);
+ }
+ // We have 10 random DNs, we want to create another one that is on the same
+ // host as some of the others.
+ DatanodeDetails multiDn = dns.get(0);
+
+ DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
+ builder.setUuid(UUID.randomUUID().toString())
+ .setHostName(multiDn.getHostName())
+ .setIpAddress(multiDn.getIpAddress())
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.STANDALONE, 3456))
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.RATIS, 4567))
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.REST, 5678))
+ .setNetworkLocation(multiDn.getNetworkLocation());
+
+ DatanodeDetails dn = builder.build();
+ nodeManager.register(dn, null, null);
+ dns.add(dn);
+ return dns;
+ }
+
+}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index 5a9ae85,f5e3f84..292a68a
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@@ -20,11 -20,10 +20,11 @@@ package org.apache.hadoop.hdds.scm.pipe
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
- import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org