You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2020/02/10 02:23:18 UTC
[hadoop-ozone] branch HDDS-1564 updated (43845da -> d5666c5)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a change to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
discard 43845da HDDS-2923 Add fall-back protection for rack awareness in pipeline creation. (#516)
omit 09aac8e HDDS-2924. Fix Pipeline#nodeIdsHash collision issue. (#478)
omit 3123cfa HDDS-2913 Update config names and CLI for multi-raft feature. (#462)
omit 2ebf21b HDDS-2772 Better management for pipeline creation limitation. (#410)
omit 129d464 HDDS-2115 Add acceptance test for createPipeline CLI and datanode list CLI (#375)
omit 5a653fd HDDS-2756. Handle pipeline creation failure in different way when it exceeds pipeline limit
omit cc0060c Resolve rebase conflict.
omit acc9640 HDDS-1574 Average out pipeline allocation on datanodes and add metrcs/test (#291)
omit fced9c1 HDDS-2650 Fix createPipeline CLI and make it message based. (#370)
omit 00f0f81 Revert "HDDS-2650 Fix createPipeline CLI. (#340)"
omit 3796154 HDDS-2035 Implement datanode level CLI to reveal pipeline relation. (#348)
omit a2c65a5 HDDS-2650 Fix createPipeline CLI. (#340)
omit 667a0c0 Rebase Fix
omit a6c1f2a HDDS-1572 Implement a Pipeline scrubber to clean up non-OPEN pipeline. (#237)
omit f8c5eb0 HDDS-1569 Support creating multiple pipelines with same datanode. Contributed by Li Cheng.
omit f9fe003 HDDS-2089: Add createPipeline CLI. (#1418)
omit 7fe5881 HDDS-1571. Create an interface for pipeline placement policy to support network topologies. (#1395)
omit f1d994e HDDS-1577. Add default pipeline placement policy implementation. (#1366)
add 0a3f0f8 HDDS-2725. start/_index.md translated to Chinese (#397)
add 8748498 HDDS-2590 Integration tests for Recon with Ozone Manager (#356)
add 34683e4 HDDS-2834. Directly read into ByteBuffer if it has array (#413)
add 245d335 HDDS-2853. NPE in OzoneContainer Start. (#419)
add 20834c5 HDDS-2827. Add initial UI of Datanodes in Recon
add 53ef7dc HDDS-2710. Add Filesystem functionality in MiniOzoneChaosCluster. (#344)
add acc3dde HDDS-2854. Add a config for the write chunk executor queue limit. (#420)
add 406428c HDDS-2188. Implement LocatedFileStatus & getFileBlockLocations to pro… (#331)
add 4b256aa HDDS-2542. Race condition between read and write stateMachineData. (#310)
add 2fa37ef HDDS-2779. Fix list volume for --start parameter. (#385)
add 046a06f HDDS-2579. Ozone client should refresh pipeline info if reads from all Datanodes fail (#372)
add 44b48b7 HDDS-2706. Add class CRLCodec - used for certificate revocation list. (#337)
add 7e6b821 HDDS-2857. Ozone Recon fails to start while ozone install (#424)
add 088d76e HDDS-2763. Handle Datanode registration in Recon. (#405)
add eceef90 HDDS-2851. Some OM Ratis config properties missing from ozone-default.xml. (#421)
add 25b4baf HDDS-2863. BindException in TestSCMRestart (#425)
add dedbbab HDDS-2872. ozone.recon.scm.db.dirs missing from ozone-default.xml. (#434)
add 92234d4 HDDS-2781. Add ObjectID and updateID to BucketInfo to avoid replaying transactions (#381)
add ba8212c HDDS-2279. S3 commands not working on HA cluster. (#333)
add 1abfe05 HDDS-2727. start/OnPrem.md translated to Chinese (#429)
add becceb6 HDDS-2750. OzoneFSInputStream to support StreamCapabilities (#379)
add 1c75951 HDDS-2828. Add initial UI of Pipelines in Recon
add c17c497 HDDS-2861. Support Freon progressbar in non-interactive environment (#423)
add a479743 HDDS-2868. Add ObjectID and UpdateID to OMKeyInfo. (#428)
add 37730f0 HDDS-1812. Du while calculating used disk space reports that chunk files are file not found (#271)
add b78d9b2 HDDS-2757. Annoying 'Failed to close the container' message because pipeline is removed from PipelineStateMap before close container command returned (#443)
add 883a662 HDDS-2875. Add a config in ozone to tune max outstanding requests in … (#436)
add 2b80e99 HDDS-2740. start/FromSource.md translation (#444)
add 1423491 HDDS-2726. start/StartFromDockerHub.md translated to Chinese (#417)
add 001bd1d HDDS-2859. Hugo error should be propagated to build (#446)
add 1caf1e3 HDDS-2866. Intermittent failure in TestOzoneManagerRocksDBLogging (#427)
add 71c24a8 HDDS-2897. Ozone recon Start failed due to Kerberos principal not being found. (#453)
add 9bc433a HDDS-2904. Remove default dependencies from hadoop-hdds/pom.xml (#459)
add 79aff8b HDDS-2905. Remove unusued BlockLocation protocol related classes (#460)
add 5d27f45 HDDS-2846. Handle Datanode Pipeline & Container Reports reports in Recon. (#431)
add 2ea9afa HDDS-2900. Avoid logging NPE when space usage check is not configured
add 968ec62 HDDS-2910. OzoneManager startup failure with throwing unhelpful exception message
add c29936a HDDS-2898. build failure due to hadoop-hdds-client test (#463)
add e7afa41 HDDS-2896. Use regex to match with ratis properties when creating ratis client. (#457)
add 2e92658 Revert "HDDS-2188. Implement LocatedFileStatus & getFileBlockLocations to pro… (#331)" (#470)
add 73cb410 HDDS-2733. start/Kubernetes.md translated to Chinese (#439)
add c74429d HDDS-2903. Use regex to match with ratis properties when creating ratis server. (#458)
add 1e442f6 HDDS-2738. start/Minikube.md translated to Chinese (#440)
add 01540d7 HDDS-2734. start/RunningViaDocker.md translated to Chinese (#442)
add dc60d18 HDDS-2915. Unit check passes despite Maven error (#465)
add f080c22 HDDS-2741. shell/_index.md translation (#473)
add 935006b HDDS-2742. shell/Format.md translation (#474)
add 803fa29 HDDS-2918. Increase timeout of safe-mode exit in acceptance tests
add 0510b55 HDDS-2902. execute_robot_test on unknown/unavailable container should fail acceptance test
add f98c9ed HDDS-2889. Make DBStore and RDBStore more commons
add 5950224 HDDS-2901. List Trash - Fix Cluster Max Keys Check
add 1f05261 HDDS-2849. OM and SCM Web-server report HTTP 404 error when accessing '/' after cluster runs for several weeks
add ab557db HDDS-2920. Remove ozone ratis client specific config keys. (#472)
add 09df00f Revert "HDDS-2920. Remove ozone ratis client specific config keys. (#472)" (#482)
add 4d603ec HDDS-2864. TestOMDbCheckpointServlet fails due to real Recon
add c8bc3b7 HDDS-2895. Generate only the required keytabs for docker based secure tests (#455)
add acd4619 HDDS-2743. shell/VolumeCommands.md translation (#475)
add 15fb459 HDDS-2906. Added Unit Test Cases ofr CRLCodec (#471)
add 3316aca HDDS-2937. Document bucket encryption option in shell/BucketCommands.md. (#483)
add c013de2 HDDS-2938. Use regex to match with ratis grpc properties when creating ratis server. (#486)
add d497bf6 HDDS-2920. Remove ozone ratis client specific config keys. (#484)
add ae7c268 HDDS_2746. inteface/_index.md translation (#488)
add 1372cc9 HDDS-2744. shell/BucketCommands.md translation (#476)
add 25d1489 HDDS-2745. shell/KeyCommands.md translation (#477)
add 4b8e8f6 HDDS-2888. Refactor Datanode StateContext to send reports and actions to all configured SCMs. (#447)
add 4ce2796 HDDS-2921. Remove ozone ratis server specific config keys. (#485)
add 242f503 HDDS-2925. Remove hdfs-client dependency from hdds-common
add 345087f HDDS-2855. Recon getContainers API should return a maximum of 1000 containers by default.
add ef79f33 HDDS-1335. Add basic UI for showing missing containers and keys
add 66abf65 HDDS-2870. Handle replay of KeyCreate requests. (#448)
add eb914ff HDDS-2916. OM HA cli getserviceroles not working. (#490)
add cec238e HDDS-2926. Intermittent failure in TestRecon due to thread timing. (#481)
add ee7b454 HDDS-2869. Handle pipeline bootstrap from SCM and create pipeline use case in Recon. (#466)
add 6794e02 HDDS-2894. Handle replay of KeyDelete and KeyRename Requests (#452)
add b53e3a5 HDDS-2931. Recon integration test should use ephemeral port for HTTP Server. (#500)
add e234cd3 HDDS-2555. Handle InterruptedException in XceiverClientGrpc (#314)
add a967de2 HDDS-2833. Enable integrations tests for github actions
add 15cf3c8 HDDS-2956. Handle Replay of AllocateBlock request (#505)
add bb57442 HDDS-2893. Handle replay of KeyPurge Request. (#450)
add 7a3dcf9 HDDS-2850. Handle Create container use case in Recon. (#503)
add 76523f2 HDDS-2747. interface/JavaApi.md (#489)
add 0692413 HDDS-2748. interface/OzoneFS.md translation (#491)
add 475b94a HDDS-2749. interface/S3.md translation (#493)
add 61084e6 HDDS-2758. gdpr/_index.md translation (#494)
add 5fb3219 HDDS-2952. Ensure ozone manager service user is part of ozone.adminis… (#506)
add 0aa4b04 HDDS-2789. concept/_index.md translation (#496)
add af43a28 HDDS-2883. Change the default client settings accordingly with change in default chunk size. (#487)
add d1b8c08 HDDS-2973. Fix root deletion logic in delete API. (#517)
add 486e872 HDDS-2892 Fixed typo in createClientRequest of OzoneManagerRatisUtils (#532)
add b895670 HDDS-2977. ozonesecure acceptance test fails due to unexpected error message (#527)
add 2d6d293 HDDS-2983. Acceptance test failures due to lack of disk space (#530)
add cd361bf HDDS-2041. Don't depend on DFSUtil to check HTTP policy. (#533)
add 8917d9a HDDS-2759. gdpr/GDPR in Ozone.md translation (#495)
add 1ac8263 HDDS-2950. Upgrade jetty to the latest 9.4 release (#508)
add 17bc776 HDDS-2845. Fix listing keys for setting --start with last key (#437)
new 6ab3326 HDDS-1577. Add default pipeline placement policy implementation. (#1366)
new 5cbfb37 HDDS-1571. Create an interface for pipeline placement policy to support network topologies. (#1395)
new 3d6e5b1 HDDS-2089: Add createPipeline CLI. (#1418)
new 1ef5e6d HDDS-1569 Support creating multiple pipelines with same datanode. Contributed by Li Cheng.
new 4cbf554 HDDS-1572 Implement a Pipeline scrubber to clean up non-OPEN pipeline. (#237)
new e98c12f Rebase Fix
new e48ec92 HDDS-2650 Fix createPipeline CLI. (#340)
new 81ed727 HDDS-2035 Implement datanode level CLI to reveal pipeline relation. (#348)
new b5ef117 Revert "HDDS-2650 Fix createPipeline CLI. (#340)"
new 6cdfdcb HDDS-2650 Fix createPipeline CLI and make it message based. (#370)
new d6b9ec0 HDDS-1574 Average out pipeline allocation on datanodes and add metrcs/test (#291)
new 4d463d6 Resolve rebase conflict.
new facd403 HDDS-2756. Handle pipeline creation failure in different way when it exceeds pipeline limit
new cb5486b HDDS-2115 Add acceptance test for createPipeline CLI and datanode list CLI (#375)
new b63a745 HDDS-2772 Better management for pipeline creation limitation. (#410)
new 8b42cdc HDDS-2913 Update config names and CLI for multi-raft feature. (#462)
new dd2b7e8 HDDS-2924. Fix Pipeline#nodeIdsHash collision issue. (#478)
new d5666c5 HDDS-2923 Add fall-back protection for rack awareness in pipeline creation. (#516)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (43845da)
\
N -- N -- N refs/heads/HDDS-1564 (d5666c5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/buildenv/Dockerfile | 2 +-
.github/workflows/post-commit.yml | 90 +
.github/workflows/pr.yml | 90 +
hadoop-hdds/client/pom.xml | 8 +
.../apache/hadoop/hdds/conf/RatisClientConfig.java | 84 +
.../apache/hadoop/hdds/conf/RatisGrpcConfig.java | 35 +-
.../org/apache/hadoop/hdds/conf/package-info.java | 11 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 25 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 -
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 35 +-
.../hdds/scm/client/ContainerOperationClient.java | 8 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 34 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 6 +-
.../hadoop/hdds/scm/storage/CommitWatcher.java | 8 +-
.../hdds/scm/storage/DummyBlockInputStream.java | 92 +
.../storage/DummyBlockInputStreamWithRetry.java | 78 +
.../hdds/scm/storage/DummyChunkInputStream.java | 66 +
.../hdds/scm/storage/TestBlockInputStream.java | 59 +-
.../hdds/scm/storage/TestChunkInputStream.java | 52 +-
hadoop-hdds/common/pom.xml | 32 +
.../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 100 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 71 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 76 +
.../hadoop/hdds/fs/AbstractSpaceUsageSource.java | 87 +
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 142 ++
.../main/java/org/apache/hadoop/hdds/fs/DU.java | 136 ++
.../java/org/apache/hadoop/hdds/fs/DUFactory.java | 90 +
.../hadoop/hdds/fs/DedicatedDiskSpaceUsage.java | 52 +
.../hdds/fs/DedicatedDiskSpaceUsageFactory.java | 86 +
.../hadoop/hdds/fs/SaveSpaceUsageToFile.java | 129 ++
.../hadoop/hdds/fs/SpaceUsageCheckFactory.java | 145 ++
.../hadoop/hdds/fs/SpaceUsageCheckParams.java | 92 +
.../hadoop/hdds/fs/SpaceUsagePersistence.java | 63 +
.../apache/hadoop/hdds/fs/SpaceUsageSource.java | 28 +-
.../org/apache/hadoop/hdds/fs/package-info.java | 9 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 129 +-
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 6 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 34 +-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 3 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../hadoop/hdds/scm/protocol/LocatedContainer.java | 127 -
.../hadoop/hdds/scm/protocol/ScmLocatedBlock.java | 100 -
.../protocol/StorageContainerLocationProtocol.java | 10 +
...inerLocationProtocolClientSideTranslatorPB.java | 16 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 5 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 41 +-
.../security/x509/certificate/utils/CRLCodec.java | 206 ++
.../java/org/apache/hadoop/hdds/utils/Cache.java | 27 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 15 +-
.../hadoop/hdds/utils/ResourceLimitCache.java | 91 +
.../hadoop/hdds/utils/ResourceSemaphore.java | 170 ++
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 6 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 12 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 4 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 7 +-
.../apache/hadoop/hdds/utils/db/TableConfig.java | 5 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 68 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 -
.../ozone/conf/DatanodeRatisServerConfig.java | 141 ++
.../org/apache/hadoop/ozone/conf/package-info.java | 11 +-
.../proto/StorageContainerLocationProtocol.proto | 12 +
hadoop-hdds/common/src/main/proto/hdds.proto | 1 +
.../common/src/main/resources/ozone-default.xml | 161 +-
.../hadoop/hdds/fs/MockSpaceUsageCheckFactory.java | 57 +
.../hadoop/hdds/fs/MockSpaceUsageCheckParams.java | 71 +
.../hadoop/hdds/fs/MockSpaceUsagePersistence.java | 44 +-
.../hadoop/hdds/fs/MockSpaceUsageSource.java | 74 +
.../hdds/fs/TestCachingSpaceUsageSource.java | 197 ++
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 112 +
.../org/apache/hadoop/hdds/fs/TestDUFactory.java | 57 +
.../hdds/fs/TestDedicatedDiskSpaceUsage.java | 63 +
.../fs/TestDedicatedDiskSpaceUsageFactory.java | 57 +
.../hadoop/hdds/fs/TestSaveSpaceUsageToFile.java | 150 ++
.../hadoop/hdds/fs/TestSpaceUsageFactory.java | 197 ++
.../apache/hadoop/hdds/ratis/TestRatisHelper.java | 122 +
.../token/TestOzoneBlockTokenIdentifier.java | 5 +-
.../x509/certificate/utils/TestCRLCodec.java | 290 +++
.../hadoop/hdds/utils/TestMetadataStore.java | 53 +-
.../hadoop/hdds/utils/TestResourceLimitCache.java | 87 +
.../hadoop/hdds/utils/TestResourceSemaphore.java | 76 +
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 7 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 25 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 6 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 5 +-
hadoop-hdds/container-service/pom.xml | 14 +
.../hadoop/ozone/HddsDatanodeHttpServer.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 46 +-
.../container/common/helpers/ContainerMetrics.java | 4 +-
.../common/statemachine/DatanodeStateMachine.java | 4 +-
.../common/statemachine/EndpointStateMachine.java | 40 +-
.../common/statemachine/SCMConnectionManager.java | 26 +-
.../common/statemachine/StateContext.java | 114 +-
.../common/states/datanode/InitDatanodeState.java | 9 +-
.../states/endpoint/HeartbeatEndpointTask.java | 9 +-
.../states/endpoint/VersionEndpointTask.java | 65 +-
.../server/ratis/ContainerStateMachine.java | 63 +-
.../transport/server/ratis/XceiverServerRatis.java | 69 +-
.../ozone/container/common/volume/HddsVolume.java | 58 +-
.../container/common/volume/HddsVolumeChecker.java | 101 +-
.../ozone/container/common/volume/VolumeInfo.java | 44 +-
.../ozone/container/common/volume/VolumeSet.java | 11 +-
.../ozone/container/common/volume/VolumeUsage.java | 169 +-
.../StorageContainerDatanodeProtocolPB.java | 8 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 10 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 28 +-
.../ozone/container/common/SCMTestUtils.java | 5 +
.../common/statemachine/TestStateContext.java | 117 +
.../states/endpoint/TestHeartbeatEndpointTask.java | 8 +
.../container/common/volume/TestHddsVolume.java | 59 +-
.../common/volume/TestHddsVolumeChecker.java | 7 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 90 +-
.../container/common/volume/TestVolumeSet.java | 4 +-
.../common/volume/TestVolumeSetDiskChecks.java | 10 +-
.../src/test/resources/ozone-site.xml | 30 +
hadoop-hdds/docs/content/concept/_index.zh.md | 8 +-
.../S3.zh.md => gdpr/GDPR in Ozone.zh.md} | 21 +-
hadoop-hdds/docs/content/gdpr/_index.zh.md | 34 +
hadoop-hdds/docs/content/interface/JavaApi.zh.md | 134 +-
hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 140 ++
hadoop-hdds/docs/content/interface/S3.zh.md | 130 +-
.../docs/content/{shell => interface}/_index.zh.md | 8 +-
hadoop-hdds/docs/content/shell/BucketCommands.md | 1 +
.../docs/content/shell/BucketCommands.zh.md | 98 +
hadoop-hdds/docs/content/shell/Format.md | 4 +-
hadoop-hdds/docs/content/shell/Format.zh.md | 65 +
hadoop-hdds/docs/content/shell/KeyCommands.md | 4 +-
hadoop-hdds/docs/content/shell/KeyCommands.zh.md | 138 ++
.../docs/content/shell/VolumeCommands.zh.md | 107 +
hadoop-hdds/docs/content/shell/_index.zh.md | 7 +-
hadoop-hdds/docs/content/start/FromSource.zh.md | 59 +
hadoop-hdds/docs/content/start/Kubernetes.zh.md | 54 +
hadoop-hdds/docs/content/start/Minikube.zh.md | 70 +
hadoop-hdds/docs/content/start/OnPrem.zh.md | 154 ++
.../docs/content/start/RunningViaDocker.zh.md | 52 +
.../docs/content/start/StartFromDockerHub.zh.md | 96 +
hadoop-hdds/docs/content/start/_index.zh.md | 84 +
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 2 +
hadoop-hdds/framework/pom.xml | 25 +
.../org/apache/hadoop/hdds/server/ServerUtils.java | 11 +-
.../hadoop/hdds/server/events/EventQueue.java | 13 +-
.../hdds/server/http/AdminAuthorizedServlet.java | 37 +-
.../hdds/server/{ => http}/BaseHttpServer.java | 207 +-
.../FilterContainer.java} | 32 +-
.../hadoop/hdds/server/http/FilterInitializer.java | 18 +-
.../hadoop/hdds/server/http/HtmlQuoting.java | 235 ++
.../apache/hadoop/hdds/server/http/HttpConfig.java | 75 +
.../hadoop/hdds/server/http/HttpRequestLog.java | 107 +
.../hdds/server/http/HttpRequestLogAppender.java | 49 +-
.../hadoop/hdds/server/http/HttpServer2.java | 1707 ++++++++++++++
.../hdds/server/{ => http}/LogStreamServlet.java | 2 +-
.../hadoop/hdds/server/http/NoCacheFilter.java | 55 +
.../hdds/server/{ => http}/ProfileServlet.java | 2 +-
.../server/{ => http}/PrometheusMetricsSink.java | 2 +-
.../hdds/server/{ => http}/PrometheusServlet.java | 2 +-
.../server/{ => http}/RatisDropwizardExports.java | 2 +-
.../{ => http}/RatisNameRewriteSampleBuilder.java | 2 +-
.../hdds/server/http/StaticUserWebFilter.java | 157 ++
.../hadoop/hdds/server/http/package-info.java | 8 +-
.../hdds/server/{ => http}/TestBaseHttpServer.java | 3 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 96 +
.../hdds/server/http/TestHttpRequestLog.java | 51 +
.../server/http/TestHttpRequestLogAppender.java | 33 +-
.../hdds/server/{ => http}/TestProfileServlet.java | 6 +-
.../{ => http}/TestPrometheusMetricsSink.java | 2 +-
.../{ => http}/TestRatisDropwizardExports.java | 2 +-
.../server/{ => http}/TestRatisNameRewrite.java | 2 +-
.../hadoop/hdds/server/http/package-info.java | 8 +-
hadoop-hdds/pom.xml | 50 -
hadoop-hdds/server-scm/pom.xml | 43 +-
.../container/AbstractContainerReportHandler.java | 2 +-
.../hdds/scm/container/ContainerManager.java | 2 +
.../hdds/scm/container/ContainerStateManager.java | 15 +-
.../IncrementalContainerReportHandler.java | 3 +
.../hdds/scm/container/SCMContainerManager.java | 30 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 20 +-
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 15 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 2 +
.../hdds/scm/pipeline/PipelineReportHandler.java | 38 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 11 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 9 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 68 +-
...inerLocationProtocolServerSideTranslatorPB.java | 18 +
.../hdds/scm/safemode/SCMSafeModeManager.java | 2 +-
.../hadoop/hdds/scm/safemode/SafeModeManager.java | 14 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 9 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 6 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 6 +
.../hdds/scm/server/StorageContainerManager.java | 3 +-
.../server/StorageContainerManagerHttpServer.java | 2 +-
.../scm/TestStorageContainerManagerHttpServer.java | 25 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 73 +-
.../ozone/container/common/TestEndPoint.java | 83 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 5 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 6 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 2 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 42 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 5 -
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 44 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 -
.../ozone/om/exceptions/OMReplayException.java} | 23 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 84 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 100 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 110 +-
.../src/main/proto/OzoneManagerProtocol.proto | 14 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 12 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 11 +-
hadoop-ozone/dev-support/checks/integration.sh | 10 +-
hadoop-ozone/dev-support/checks/unit.sh | 5 +-
.../dist/src/main/compose/ozone-mr/common-config | 2 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 4 +-
.../src/main/compose/ozone-topology/docker-config | 4 +-
.../src/main/compose/ozone/docker-compose.yaml | 1 -
.../dist/src/main/compose/ozone/docker-config | 4 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 +
.../src/main/compose/ozones3-haproxy/docker-config | 4 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 9 +
.../src/main/compose/ozonesecure-mr/docker-config | 5 +-
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
.../src/main/compose/ozonesecure/docker-config | 7 +-
.../dist/src/main/compose/ozonesecure/test.sh | 2 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 22 +-
.../dist/src/main/smoketest/basic/basic.robot | 1 +
.../main/smoketest/om-ratis/testOMAdminCmd.robot | 21 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 25 +-
.../main/smoketest/security/ozone-secure-fs.robot | 5 +
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 16 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 103 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 20 +-
.../org/apache/hadoop/ozone/utils/LoadBucket.java | 224 +-
.../hadoop/ozone/insight/BaseInsightPoint.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 132 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 18 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 128 +-
.../hadoop/fs/ozone/TestOzoneFsRenameDir.java | 110 -
.../TestContainerStateManagerIntegration.java | 1 +
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 16 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +
.../scm/pipeline/TestRatisPipelineProvider.java | 187 --
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 172 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 9 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 7 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 5 +-
.../hadoop/ozone/TestStorageContainerManager.java | 2 +
.../ozone/client/rpc/Test2WayCommitInRatis.java | 6 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 1 -
.../rpc/TestBlockOutputStreamWithFailures.java | 3 +-
.../rpc/TestCloseContainerHandlingByClient.java | 2 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 7 +-
.../rpc/TestContainerReplicationEndToEnd.java | 12 +-
.../client/rpc/TestContainerStateMachine.java | 3 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +
.../client/rpc/TestDeleteWithSlowFollower.java | 11 +-
.../client/rpc/TestFailureHandlingByClient.java | 4 +-
.../ozone/client/rpc/TestKeyInputStream.java | 1 -
.../rpc/TestMultiBlockWritesWithDnFailures.java | 4 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 10 +-
.../rpc/TestOzoneClientRetriesOnException.java | 3 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 32 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 2 +
.../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 4 +-
.../ozone/client/rpc/TestWatchForCommit.java | 62 +-
.../ozone/container/TestContainerReplication.java | 2 +
.../commandhandler/TestBlockDeletion.java | 2 +
.../TestCloseContainerByPipeline.java | 2 +
.../transport/server/ratis/TestCSMMetrics.java | 2 +
.../container/metrics/TestContainerMetrics.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +
.../server/TestSecureContainerServer.java | 2 +
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +-
.../hadoop/ozone/freon/TestDataValidate.java | 2 -
.../ozone/freon/TestFreonWithPipelineDestroy.java | 2 +
.../ozone/freon/TestOzoneClientKeyGenerator.java | 93 +
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 2 -
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 175 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 3 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 7 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 4 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 2 +
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 46 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +
.../hadoop/ozone/ozShell/TestOzoneShellHA.java | 63 +
.../org/apache/hadoop/ozone/recon/TestRecon.java | 384 ++++
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 5 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 2 +
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 2 +
.../resources/{hdfs-site.xml => mapred-site.xml} | 0
.../src/test/resources/ozone-site.xml | 29 +
.../resources/{hdfs-site.xml => yarn-site.xml} | 0
.../apache/hadoop/ozone/om/KeyDeletingService.java | 46 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 14 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 3 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 15 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 41 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 46 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +-
.../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 3 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 13 -
.../om/ratis/utils/OzoneManagerRatisUtils.java | 2 +-
.../om/request/bucket/OMBucketCreateRequest.java | 45 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 39 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 67 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 18 +
.../request/bucket/acl/OMBucketAddAclRequest.java | 5 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 5 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 5 +-
.../om/request/file/OMDirectoryCreateRequest.java | 6 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 156 +-
.../om/request/key/OMAllocateBlockRequest.java | 105 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 5 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 141 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 86 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 127 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 188 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 188 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 156 +-
.../request/s3/bucket/S3BucketDeleteRequest.java | 42 +-
.../S3InitiateMultipartUploadRequest.java | 17 +-
.../multipart/S3MultipartUploadAbortRequest.java | 1 +
.../S3MultipartUploadCommitPartRequest.java | 5 +
.../S3MultipartUploadCompleteRequest.java | 19 +-
.../om/response/bucket/OMBucketCreateResponse.java | 29 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 29 +-
.../bucket/OMBucketSetPropertyResponse.java | 29 +-
.../response/bucket/acl/OMBucketAclResponse.java | 21 +-
.../om/response/file/OMFileCreateResponse.java | 13 +-
.../om/response/key/OMAllocateBlockResponse.java | 31 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 30 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 13 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 22 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 81 +-
.../response/s3/bucket/S3BucketCreateResponse.java | 41 +-
.../response/s3/bucket/S3BucketDeleteResponse.java | 26 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 43 +-
.../hadoop/ozone/web/ozShell/OzoneAddress.java | 37 +
.../ozone/web/ozShell/s3/GetS3SecretHandler.java | 10 +-
.../web/ozShell/volume/ListVolumeHandler.java | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 11 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 34 +
.../ozone/om/TestOzoneManagerHttpServer.java | 8 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 11 +-
.../ozone/om/request/TestOMRequestUtils.java | 129 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 1 +
.../request/bucket/TestOMBucketCreateRequest.java | 48 +-
.../request/bucket/TestOMBucketDeleteRequest.java | 46 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 34 +-
.../om/request/file/TestOMFileCreateRequest.java | 42 +-
.../om/request/key/TestOMKeyCreateRequest.java | 48 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 44 +-
.../key/TestOMKeyPurgeRequestAndResponse.java | 166 +-
.../om/request/key/TestOMKeyRenameRequest.java | 106 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 1 +
.../s3/bucket/TestS3BucketCreateRequest.java | 28 +-
.../s3/bucket/TestS3BucketDeleteRequest.java | 36 +
.../om/request/s3/bucket/TestS3BucketRequest.java | 2 +-
.../ozone/om/response/TestOMResponseUtils.java | 6 +-
.../bucket/TestOMBucketCreateResponse.java | 5 +-
.../bucket/TestOMBucketDeleteResponse.java | 17 +-
.../bucket/TestOMBucketSetPropertyResponse.java | 5 +-
.../response/key/TestOMAllocateBlockResponse.java | 6 +-
.../om/response/key/TestOMKeyCreateResponse.java | 6 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 12 +-
.../om/response/key/TestOMKeyRenameResponse.java | 14 +-
.../s3/bucket/TestS3BucketDeleteResponse.java | 2 +-
.../s3/multipart/TestS3MultipartResponse.java | 11 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 21 +-
.../fs/ozone/CapableOzoneFSInputStream.java} | 35 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 19 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 6 +
.../hadoop/fs/ozone/OzoneStreamCapabilities.java} | 21 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 151 ++
.../ozone/contract/TestOzoneFSInputStreamUnit.java | 101 -
.../hadoop/ozone/recon/ConfigurationProvider.java | 17 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 11 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 31 +-
.../apache/hadoop/ozone/recon/ReconHttpServer.java | 6 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 79 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 5 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 5 +
.../ozone/recon/api/ContainerKeyService.java | 8 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 4 +-
.../ozone/recon/scm/ReconContainerManager.java | 86 +
.../recon/scm/ReconDatanodeProtocolServer.java | 31 +-
.../ReconIncrementalContainerReportHandler.java | 70 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 72 +
.../ozone/recon/scm/ReconPipelineManager.java | 139 ++
.../recon/scm/ReconPipelineReportHandler.java | 90 +
.../ReconSafeModeManager.java} | 16 +-
...conNodeManager.java => ReconStorageConfig.java} | 23 +-
.../recon/scm/ReconStorageContainerManager.java | 134 --
.../scm/ReconStorageContainerManagerFacade.java | 234 ++
.../recon/spi/ContainerDBServiceProvider.java | 5 +
.../recon/spi/StorageContainerServiceProvider.java | 31 +
.../spi/impl/ContainerDBServiceProviderImpl.java | 30 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 18 +-
.../recon/spi/impl/ReconContainerDBProvider.java | 22 +-
.../impl/StorageContainerServiceProviderImpl.java | 63 +
.../ozone/recon/tasks/ReconTaskController.java | 7 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 11 +-
.../webapps/recon/ozone-recon-web/LICENSE | 1422 ++++--------
.../webapps/recon/ozone-recon-web/api/db.json | 380 ++-
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +-
.../webapps/recon/ozone-recon-web/package.json | 2 +
.../webapps/recon/ozone-recon-web/src/App.less | 31 +
.../src/components/NavBar/NavBar.less | 21 +-
.../src/components/OverviewCard/OverviewCard.less | 4 +
.../src/components/OverviewCard/OverviewCard.tsx | 13 +-
.../src/constants/breadcrumbs.constants.tsx | 3 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 10 +
.../Overview/Overview.less => utils/common.tsx} | 4 +-
.../Datanodes/Datanodes.less} | 31 +-
.../src/views/Datanodes/Datanodes.tsx | 178 +-
.../MissingContainers.less} | 4 -
.../views/MissingContainers/MissingContainers.tsx | 218 ++
.../src/views/Overview/Overview.less | 9 +
.../src/views/Overview/Overview.tsx | 43 +-
.../Overview.less => Pipelines/Pipelines.less} | 8 +-
.../src/views/Pipelines/Pipelines.tsx | 158 ++
.../webapps/recon/ozone-recon-web/yarn.lock | 2432 +++++++++-----------
.../ozone/recon/AbstractOMMetadataManagerTest.java | 5 +-
.../scm/AbstractReconContainerManagerTest.java | 84 +
.../ozone/recon/scm/TestReconContainerManager.java | 81 +
...TestReconIncrementalContainerReportHandler.java | 121 +
.../ozone/recon/scm/TestReconPipelineManager.java | 173 ++
.../recon/scm/TestReconPipelineReportHandler.java | 101 +
.../TestStorageContainerServiceProviderImpl.java | 93 +
.../recon/tasks/TestReconTaskControllerImpl.java | 1 +
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 1 +
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 2 +-
.../ozone/admin/om/GetServiceRolesSubcommand.java | 9 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 17 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 2 +-
.../apache/hadoop/ozone/freon/FreonHttpServer.java | 2 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 44 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 3 +-
pom.xml | 4 +-
443 files changed, 18394 insertions(+), 6186 deletions(-)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/conf/RatisClientConfig.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java => hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/conf/RatisGrpcConfig.java (51%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less => hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/conf/package-info.java (88%)
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyChunkInputStream.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DFSConfigKeysLegacy.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/AbstractSpaceUsageSource.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/CachingSpaceUsageSource.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DU.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DedicatedDiskSpaceUsage.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DedicatedDiskSpaceUsageFactory.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SaveSpaceUsageToFile.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckParams.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsagePersistence.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageSource.java (58%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/package-info.java (91%)
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/LocatedContainer.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmLocatedBlock.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CRLCodec.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/Cache.java (74%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/ResourceLimitCache.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/ResourceSemaphore.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/DatanodeRatisServerConfig.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java (88%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckFactory.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageCheckParams.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java => hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsagePersistence.java (50%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/MockSpaceUsageSource.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestCachingSpaceUsageSource.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDU.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDUFactory.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDedicatedDiskSpaceUsage.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDedicatedDiskSpaceUsageFactory.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestSaveSpaceUsageToFile.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestSpaceUsageFactory.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestRatisHelper.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCRLCodec.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestResourceLimitCache.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestResourceSemaphore.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java
create mode 100644 hadoop-hdds/container-service/src/test/resources/ozone-site.xml
copy hadoop-hdds/docs/content/{interface/S3.zh.md => gdpr/GDPR in Ozone.zh.md} (52%)
create mode 100644 hadoop-hdds/docs/content/gdpr/_index.zh.md
create mode 100644 hadoop-hdds/docs/content/interface/OzoneFS.zh.md
copy hadoop-hdds/docs/content/{shell => interface}/_index.zh.md (80%)
create mode 100644 hadoop-hdds/docs/content/shell/BucketCommands.zh.md
create mode 100644 hadoop-hdds/docs/content/shell/Format.zh.md
create mode 100644 hadoop-hdds/docs/content/shell/KeyCommands.zh.md
create mode 100644 hadoop-hdds/docs/content/shell/VolumeCommands.zh.md
create mode 100644 hadoop-hdds/docs/content/start/FromSource.zh.md
create mode 100644 hadoop-hdds/docs/content/start/Kubernetes.zh.md
create mode 100644 hadoop-hdds/docs/content/start/Minikube.zh.md
create mode 100644 hadoop-hdds/docs/content/start/OnPrem.zh.md
create mode 100644 hadoop-hdds/docs/content/start/RunningViaDocker.zh.md
create mode 100644 hadoop-hdds/docs/content/start/StartFromDockerHub.zh.md
create mode 100644 hadoop-hdds/docs/content/start/_index.zh.md
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/AdminAuthorizedServlet.java (52%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{ => http}/BaseHttpServer.java (51%)
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{RatisDropwizardExports.java => http/FilterContainer.java} (54%)
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterInitializer.java (66%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HtmlQuoting.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpConfig.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLog.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLogAppender.java (51%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{ => http}/LogStreamServlet.java (97%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/NoCacheFilter.java
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{ => http}/ProfileServlet.java (99%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{ => http}/PrometheusMetricsSink.java (99%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{ => http}/PrometheusServlet.java (97%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{ => http}/RatisDropwizardExports.java (96%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/{ => http}/RatisNameRewriteSampleBuilder.java (98%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/StaticUserWebFilter.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/package-info.java (86%)
rename hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/{ => http}/TestBaseHttpServer.java (98%)
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHtmlQuoting.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLog.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java => hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java (56%)
rename hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/{ => http}/TestProfileServlet.java (90%)
rename hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/{ => http}/TestPrometheusMetricsSink.java (99%)
rename hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/{ => http}/TestRatisDropwizardExports.java (98%)
rename hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/{ => http}/TestRatisNameRewrite.java (98%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less => hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/package-info.java (88%)
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeManager.java (78%)
copy hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java => common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMReplayException.java} (59%)
copy hadoop-hdds/docs/dev-support/bin/generate-site.sh => hadoop-ozone/dist/src/main/smoketest/om-ratis/testOMAdminCmd.robot (65%)
mode change 100755 => 100644
copy hadoop-hdds/docs/dev-support/bin/generate-site.sh => hadoop-ozone/dist/src/main/smoketest/recon/recon-api.robot (57%)
mode change 100755 => 100644
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsRenameDir.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOzoneClientKeyGenerator.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestRecon.java
copy hadoop-ozone/integration-test/src/test/resources/{hdfs-site.xml => mapred-site.xml} (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
copy hadoop-ozone/integration-test/src/test/resources/{hdfs-site.xml => yarn-site.xml} (100%)
copy hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponse.java => ozonefs/src/main/java/org/apache/hadoop/fs/ozone/CapableOzoneFSInputStream.java} (56%)
copy hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java => ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneStreamCapabilities.java} (61%)
create mode 100644 hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
delete mode 100644 hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/TestOzoneFSInputStreamUnit.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconIncrementalContainerReportHandler.java (55%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineReportHandler.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/{spi/StorageContainerServiceProvider.java => scm/ReconSafeModeManager.java} (73%)
rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/{ReconNodeManager.java => ReconStorageConfig.java} (60%)
delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManager.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{views/Overview/Overview.less => utils/common.tsx} (88%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{components/OverviewCard/OverviewCard.less => views/Datanodes/Datanodes.less} (71%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{Overview/Overview.less => MissingContainers/MissingContainers.less} (94%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.tsx
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{Overview/Overview.less => Pipelines/Pipelines.less} (94%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.tsx
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineReportHandler.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestStorageContainerServiceProviderImpl.java
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 13/18: HDDS-2756. Handle pipeline creation failure
in different way when it exceeds pipeline limit
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit facd4033f5cc7bcab01e36d8893c470a884b545a
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Wed Jan 8 17:37:17 2020 +0100
HDDS-2756. Handle pipeline creation failure in different way when it exceeds pipeline limit
Closes #401
---
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 30 ++++++++++++----------
.../hdds/scm/pipeline/SCMPipelineManager.java | 1 -
2 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index bc65d14..f4a13e1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -114,10 +114,10 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
boolean meet = (nodeManager.getPipelinesCount(datanodeDetails)
- pipelineNumDeductable) < heavyNodeCriteria;
if (!meet) {
- LOG.info("Pipeline Placement: can't place more pipeline on heavy " +
- "datanode： " + datanodeDetails.getUuid().toString() + " Heaviness: " +
- nodeManager.getPipelinesCount(datanodeDetails) + " limit: " +
- heavyNodeCriteria);
+ LOG.debug("Pipeline Placement: can't place more pipeline on heavy " +
+ "datanode： " + datanodeDetails.getUuid().toString() +
+ " Heaviness: " + nodeManager.getPipelinesCount(datanodeDetails) +
+ " limit: " + heavyNodeCriteria);
}
return meet;
}
@@ -144,17 +144,19 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
int initialHealthyNodesCount = healthyNodes.size();
String msg;
if (initialHealthyNodesCount == 0) {
- msg = "No healthy node found to allocate pipeline.";
+ msg = "No healthy nodes found to allocate pipeline.";
LOG.error(msg);
throw new SCMException(msg, SCMException.ResultCodes
.FAILED_TO_FIND_HEALTHY_NODES);
}
if (initialHealthyNodesCount < nodesRequired) {
- msg = String.format("Not enough healthy nodes to allocate pipeline. %d "
+ LOG.warn("Not enough healthy nodes to allocate pipeline. %d "
+ " datanodes required. Found %d",
nodesRequired, initialHealthyNodesCount);
- LOG.error(msg);
+ msg = String.format("Pipeline creation failed due to no sufficient" +
+ " healthy datanodes. Required %d. Found %d.",
+ nodesRequired, initialHealthyNodesCount);
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
@@ -166,13 +168,15 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
.collect(Collectors.toList());
if (healthyList.size() < nodesRequired) {
- msg = String.format("Unable to find enough nodes that meet " +
+ LOG.debug("Unable to find enough nodes that meet " +
"the criteria that cannot engage in more than %d pipelines." +
" Nodes required: %d Found: %d, healthy nodes count in " +
"NodeManager: %d.",
heavyNodeCriteria, nodesRequired, healthyList.size(),
initialHealthyNodesCount);
- LOG.error(msg);
+ msg = String.format("Pipeline creation failed due to not enough" +
+ " healthy datanodes after filter. Required %d. Found %d",
+ nodesRequired, initialHealthyNodesCount);
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
@@ -229,8 +233,8 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
// First choose an anchor nodes randomly
DatanodeDetails anchor = chooseNode(healthyNodes);
if (anchor == null) {
- LOG.error("Pipeline Placement: Unable to find the first healthy nodes " +
- "that meet the criteria. Required nodes: {}, Found nodes: {}",
+ LOG.warn("Unable to find healthy nodes." +
+ " Required nodes: {}, Found nodes: {}",
nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
@@ -245,7 +249,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
healthyNodes, exclude,
nodeManager.getClusterNetworkTopologyMap(), anchor);
if (nodeOnDifferentRack == null) {
- LOG.error("Pipeline Placement: Unable to find nodes on different racks " +
+ LOG.warn("Pipeline Placement: Unable to find nodes on different racks " +
" that meet the criteria. Required nodes: {}, Found nodes: {}",
nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
@@ -269,7 +273,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
}
if (results.size() < nodesRequired) {
- LOG.error("Pipeline Placement: Unable to find the required number of " +
+ LOG.warn("Unable to find the required number of " +
"healthy nodes that meet the criteria. Required nodes: {}, " +
"Found nodes: {}", nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 11e9916..f924b41 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -187,7 +187,6 @@ public class SCMPipelineManager implements PipelineManager {
return pipeline;
} catch (IOException ex) {
metrics.incNumPipelineCreationFailed();
- LOG.error("Pipeline creation failed.", ex);
throw ex;
} finally {
lock.writeLock().unlock();
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 02/18: HDDS-1571. Create an interface for pipeline
placement policy to support network topologies. (#1395)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 5cbfb37ab6290ce39b90a82afc01440de7fc2327
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Tue Sep 10 20:15:51 2019 +0800
HDDS-1571. Create an interface for pipeline placement policy to support network topologies. (#1395)
(cherry picked from commit 753fc6703a39154ed6013e44dbae572391748906)
---
...erPlacementPolicy.java => PlacementPolicy.java} | 12 +++----
.../placement/algorithms/package-info.java | 21 -----------
.../common/src/main/resources/ozone-default.xml | 6 ++--
...onPolicy.java => SCMCommonPlacementPolicy.java} | 23 ++++++------
.../hdds/scm/container/ReplicationManager.java | 13 +++----
.../ContainerPlacementPolicyFactory.java | 18 +++++-----
.../algorithms/SCMContainerPlacementCapacity.java | 4 ++-
.../algorithms/SCMContainerPlacementRackAware.java | 12 +++----
.../algorithms/SCMContainerPlacementRandom.java | 6 ++--
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 42 +++++++++++++---------
.../hdds/scm/pipeline/RatisPipelineProvider.java | 15 ++++----
.../hdds/scm/server/StorageContainerManager.java | 4 +--
.../hdds/scm/container/TestReplicationManager.java | 7 ++--
.../algorithms/TestContainerPlacementFactory.java | 7 ++--
.../hdds/scm/node/TestContainerPlacement.java | 5 ++-
.../hdds/scm/safemode/TestSafeModeHandler.java | 5 ++-
.../hadoop/ozone/TestContainerOperations.java | 7 ++--
.../TestContainerStateMachineIdempotency.java | 5 ++-
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +--
.../hadoop/ozone/scm/TestContainerSmallFile.java | 4 +--
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 5 ++-
21 files changed, 106 insertions(+), 119 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
similarity index 80%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
index 52ce796..f6a0e8b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
@@ -15,7 +15,7 @@
* the License.
*/
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
+package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -23,14 +23,14 @@ import java.io.IOException;
import java.util.List;
/**
- * A ContainerPlacementPolicy support choosing datanodes to build replication
- * pipeline with specified constraints.
+ * A PlacementPolicy support choosing datanodes to build
+ * pipelines or containers with specified constraints.
*/
-public interface ContainerPlacementPolicy {
+public interface PlacementPolicy {
/**
- * Given the replication factor and size required, return set of datanodes
- * that satisfy the nodes and size requirement.
+ * Given an initial set of datanodes and the size required,
+ * return set of datanodes that satisfy the nodes and size requirement.
*
* @param excludedNodes - list of nodes to be excluded.
* @param favoredNodes - list of nodes preferred.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java
deleted file mode 100644
index dac4752..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-/**
- Contains container placement policy interface definition.
- **/
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index ba465c5..4144500 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -776,9 +776,11 @@
</value>
<tag>OZONE, MANAGEMENT</tag>
<description>
- The full name of class which implements org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy.
+ The full name of class which implements
+ org.apache.hadoop.hdds.scm.PlacementPolicy.
The class decides which datanode will be used to host the container replica. If not set,
- org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom will be used as default value.
+ org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom will be used as default
+ value.
</description>
</property>
<property>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
similarity index 90%
rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
index 77cdd83..25457f7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
@@ -15,7 +15,7 @@
* the License.
*/
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
+package org.apache.hadoop.hdds.scm;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
@@ -33,25 +33,25 @@ import java.util.Random;
import java.util.stream.Collectors;
/**
- * SCM CommonPolicy implements a set of invariants which are common
- * for all container placement policies, acts as the repository of helper
+ * This policy implements a set of invariants which are common
+ * for all basic placement policies, acts as the repository of helper
* functions which are common to placement policies.
*/
-public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
+public abstract class SCMCommonPlacementPolicy implements PlacementPolicy {
@VisibleForTesting
static final Logger LOG =
- LoggerFactory.getLogger(SCMCommonPolicy.class);
+ LoggerFactory.getLogger(SCMCommonPlacementPolicy.class);
private final NodeManager nodeManager;
private final Random rand;
private final Configuration conf;
/**
- * Constructs SCM Common Policy Class.
+ * Constructor.
*
* @param nodeManager NodeManager
* @param conf Configuration class.
*/
- public SCMCommonPolicy(NodeManager nodeManager, Configuration conf) {
+ public SCMCommonPlacementPolicy(NodeManager nodeManager, Configuration conf) {
this.nodeManager = nodeManager;
this.rand = new Random();
this.conf = conf;
@@ -85,7 +85,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
}
/**
- * Given the replication factor and size required, return set of datanodes
+ * Given size required, return set of datanodes
* that satisfy the nodes and size requirement.
* <p>
* Here are some invariants of container placement.
@@ -149,7 +149,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
* @param datanodeDetails DatanodeDetails
* @return true if we have enough space.
*/
- boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
+ public boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
long sizeRequired) {
SCMNodeMetric nodeMetric = nodeManager.getNodeStat(datanodeDetails);
return (nodeMetric != null) && (nodeMetric.get() != null)
@@ -164,7 +164,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
* @param nodesRequired - Nodes Required
* @param healthyNodes - List of Nodes in the result set.
* @return List of Datanodes that can be used for placement.
- * @throws SCMException
+ * @throws SCMException SCMException
*/
public List<DatanodeDetails> getResultSet(
int nodesRequired, List<DatanodeDetails> healthyNodes)
@@ -190,8 +190,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
/**
* Choose a datanode according to the policy, this function is implemented
- * by the actual policy class. For example, PlacementCapacity or
- * PlacementRandom.
+ * by the actual policy class.
*
* @param healthyNodes - Set of healthy nodes we can choose from.
* @return DatanodeDetails
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index bfa411d..509b8c2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@ -38,8 +38,9 @@ import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.metrics2.MetricsCollector;
@@ -85,7 +86,7 @@ public class ReplicationManager implements MetricsSource {
* PlacementPolicy which is used to identify where a container
* should be replicated.
*/
- private final ContainerPlacementPolicy containerPlacement;
+ private final PlacementPolicy containerPlacement;
/**
* EventPublisher to fire Replicate and Delete container events.
@@ -131,12 +132,12 @@ public class ReplicationManager implements MetricsSource {
*
* @param conf OzoneConfiguration
* @param containerManager ContainerManager
- * @param containerPlacement ContainerPlacementPolicy
+ * @param containerPlacement PlacementPolicy
* @param eventPublisher EventPublisher
*/
public ReplicationManager(final ReplicationManagerConfiguration conf,
final ContainerManager containerManager,
- final ContainerPlacementPolicy containerPlacement,
+ final PlacementPolicy containerPlacement,
final EventPublisher eventPublisher,
final LockManager<ContainerID> lockManager) {
this.containerManager = containerManager;
@@ -474,7 +475,7 @@ public class ReplicationManager implements MetricsSource {
/**
* If the given container is under replicated, identify a new set of
- * datanode(s) to replicate the container using ContainerPlacementPolicy
+ * datanode(s) to replicate the container using PlacementPolicy
* and send replicate container command to the identified datanode(s).
*
* @param container ContainerInfo
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
index 18ec2c3..adaeb87 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.container.placement.algorithms;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -34,22 +35,23 @@ public final class ContainerPlacementPolicyFactory {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerPlacementPolicyFactory.class);
- private static final Class<? extends ContainerPlacementPolicy>
+ private static final Class<? extends PlacementPolicy>
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT =
SCMContainerPlacementRandom.class;
private ContainerPlacementPolicyFactory() {
}
- public static ContainerPlacementPolicy getPolicy(Configuration conf,
- final NodeManager nodeManager, NetworkTopology clusterMap,
- final boolean fallback, SCMContainerPlacementMetrics metrics)
- throws SCMException{
- final Class<? extends ContainerPlacementPolicy> placementClass = conf
+
+ public static PlacementPolicy getPolicy(Configuration conf,
+ final NodeManager nodeManager, NetworkTopology clusterMap,
+ final boolean fallback, SCMContainerPlacementMetrics metrics)
+ throws SCMException{
+ final Class<? extends PlacementPolicy> placementClass = conf
.getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT,
- ContainerPlacementPolicy.class);
- Constructor<? extends ContainerPlacementPolicy> constructor;
+ PlacementPolicy.class);
+ Constructor<? extends PlacementPolicy> constructor;
try {
constructor = placementClass.getDeclaredConstructor(NodeManager.class,
Configuration.class, NetworkTopology.class, boolean.class,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
index 85d281c..1909344 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -65,7 +66,8 @@ import org.slf4j.LoggerFactory;
* little or no work and the cluster will achieve a balanced distribution
* over time.
*/
-public final class SCMContainerPlacementCapacity extends SCMCommonPolicy {
+public final class SCMContainerPlacementCapacity
+ extends SCMCommonPlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(SCMContainerPlacementCapacity.class);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
index 6d49459..8933fe9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetConstants;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -45,7 +46,8 @@ import java.util.List;
* recommend to use this if the network topology has more layers.
* <p>
*/
-public final class SCMContainerPlacementRackAware extends SCMCommonPolicy {
+public final class SCMContainerPlacementRackAware
+ extends SCMCommonPlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(SCMContainerPlacementRackAware.class);
@@ -271,11 +273,9 @@ public final class SCMContainerPlacementRackAware extends SCMCommonPolicy {
throw new SCMException("No satisfied datanode to meet the" +
" excludedNodes and affinityNode constrains.", null);
}
- if (hasEnoughSpace((DatanodeDetails)node, sizeRequired)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Datanode {} is chosen for container. Required size is {}",
- node.toString(), sizeRequired);
- }
+ if (super.hasEnoughSpace((DatanodeDetails)node, sizeRequired)) {
+ LOG.debug("Datanode {} is chosen. Required size is {}",
+ node.toString(), sizeRequired);
metrics.incrDatanodeChooseSuccessCount();
if (isFallbacked) {
metrics.incrDatanodeChooseFallbackCount();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
index 6b1a5c8..ce5d10d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdds.scm.container.placement.algorithms;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -37,8 +39,8 @@ import java.util.List;
* Balancer will need to support containers as a feature before this class
* can be practically used.
*/
-public final class SCMContainerPlacementRandom extends SCMCommonPolicy
- implements ContainerPlacementPolicy {
+public final class SCMContainerPlacementRandom extends SCMCommonPlacementPolicy
+ implements PlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(SCMContainerPlacementRandom.class);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index cb9954d..1983ed6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMCommonPolicy;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -47,7 +47,7 @@ import java.util.stream.Collectors;
* 3. Choose an anchor node among the viable nodes.
* 4. Choose other nodes around the anchor node based on network topology
*/
-public final class PipelinePlacementPolicy extends SCMCommonPolicy {
+public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(PipelinePlacementPolicy.class);
@@ -150,33 +150,41 @@ public final class PipelinePlacementPolicy extends SCMCommonPolicy {
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
int nodesRequired, final long sizeRequired) throws SCMException {
- // get a list of viable nodes based on criteria
+ // Get a list of viable nodes based on criteria
+ // and make sure excludedNodes are excluded from list.
List<DatanodeDetails> healthyNodes =
filterViableNodes(excludedNodes, nodesRequired);
-
- List<DatanodeDetails> results = new ArrayList<>();
-
+
// Randomly picks nodes when all nodes are equal.
// This happens when network topology is absent or
// all nodes are on the same rack.
if (checkAllNodesAreEqual(nodeManager.getClusterNetworkTopologyMap())) {
LOG.info("All nodes are considered equal. Now randomly pick nodes. " +
"Required nodes: {}", nodesRequired);
- results = super.getResultSet(nodesRequired, healthyNodes);
- if (results.size() < nodesRequired) {
- LOG.error("Unable to find the required number of healthy nodes that " +
- "meet the criteria. Required nodes: {}, Found nodes: {}",
- nodesRequired, results.size());
- throw new SCMException("Unable to find required number of nodes.",
- SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
- }
- return results;
+ return super.getResultSet(nodesRequired, healthyNodes);
+ } else {
+ // Since topology and rack awareness are available, picks nodes
+ // based on them.
+ return this.getResultSet(nodesRequired, healthyNodes);
}
+ }
+ /**
+ * Get result set based on the pipeline placement algorithm which considers
+ * network topology and rack awareness.
+ * @param nodesRequired - Nodes Required
+ * @param healthyNodes - List of Nodes in the result set.
+ * @return a list of datanodes
+ * @throws SCMException SCMException
+ */
+ @Override
+ public List<DatanodeDetails> getResultSet(
+ int nodesRequired, List<DatanodeDetails> healthyNodes)
+ throws SCMException {
+ List <DatanodeDetails> results = new ArrayList<>(nodesRequired);
// Since nodes are widely distributed, the results should be selected
// base on distance in topology, rack awareness and load balancing.
List<DatanodeDetails> exclude = new ArrayList<>();
- exclude.addAll(excludedNodes);
// First choose an anchor nodes randomly
DatanodeDetails anchor = chooseNode(healthyNodes);
if (anchor == null) {
@@ -193,7 +201,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPolicy {
// Choose the second node on different racks from anchor.
DatanodeDetails nodeOnDifferentRack = chooseNodeBasedOnRackAwareness(
- healthyNodes, excludedNodes,
+ healthyNodes, exclude,
nodeManager.getClusterNetworkTopologyMap(), anchor);
if (nodeOnDifferentRack == null) {
LOG.error("Unable to find nodes on different racks that " +
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 6b93192..dacc4ca 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -25,10 +25,9 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
.SCMContainerPlacementRandom;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -94,16 +93,16 @@ public class RatisPipelineProvider implements PipelineProvider {
* @return SCM container placement policy implementation instance.
*/
@SuppressWarnings("unchecked")
- // TODO: should we rename ContainerPlacementPolicy to PipelinePlacementPolicy?
- private static ContainerPlacementPolicy createContainerPlacementPolicy(
+ // TODO: should we rename PlacementPolicy to PipelinePlacementPolicy?
+ private static PlacementPolicy createContainerPlacementPolicy(
final NodeManager nodeManager, final Configuration conf) {
- Class<? extends ContainerPlacementPolicy> implClass =
- (Class<? extends ContainerPlacementPolicy>) conf.getClass(
+ Class<? extends PlacementPolicy> implClass =
+ (Class<? extends PlacementPolicy>) conf.getClass(
ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
SCMContainerPlacementRandom.class);
try {
- Constructor<? extends ContainerPlacementPolicy> ctor =
+ Constructor<? extends PlacementPolicy> ctor =
implClass.getDeclaredConstructor(NodeManager.class,
Configuration.class);
return ctor.newInstance(nodeManager, conf);
@@ -116,7 +115,7 @@ public class RatisPipelineProvider implements PipelineProvider {
// LOG.error("Unhandled exception occurred, Placement policy will not " +
// "be functional.");
throw new IllegalArgumentException("Unable to load " +
- "ContainerPlacementPolicy", e);
+ "PlacementPolicy", e);
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2dfde78..53adcf2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -58,7 +58,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.IncrementalContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
import org.apache.hadoop.hdds.scm.container.ReplicationManager;
@@ -391,7 +391,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
placementMetrics = SCMContainerPlacementMetrics.create();
- ContainerPlacementPolicy containerPlacementPolicy =
+ PlacementPolicy containerPlacementPolicy =
ContainerPlacementPolicyFactory.getPolicy(conf, scmNodeManager,
clusterMap, true, placementMetrics);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
index dadb309..87d7655 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
@@ -27,8 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -66,7 +65,7 @@ public class TestReplicationManager {
private ReplicationManager replicationManager;
private ContainerStateManager containerStateManager;
- private ContainerPlacementPolicy containerPlacementPolicy;
+ private PlacementPolicy containerPlacementPolicy;
private EventQueue eventQueue;
private DatanodeCommandHandler datanodeCommandHandler;
@@ -93,7 +92,7 @@ public class TestReplicationManager {
.thenAnswer(invocation -> containerStateManager
.getContainerReplicas((ContainerID)invocation.getArguments()[0]));
- containerPlacementPolicy = Mockito.mock(ContainerPlacementPolicy.class);
+ containerPlacementPolicy = Mockito.mock(PlacementPolicy.class);
Mockito.when(containerPlacementPolicy.chooseDatanodes(
Mockito.anyListOf(DatanodeDetails.class),
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
index b685ba9..a454de2 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -99,7 +100,7 @@ public class TestContainerPlacementFactory {
when(nodeManager.getNodeStat(datanodes.get(4)))
.thenReturn(new SCMNodeMetric(storageCapacity, 70L, 30L));
- ContainerPlacementPolicy policy = ContainerPlacementPolicyFactory
+ PlacementPolicy policy = ContainerPlacementPolicyFactory
.getPolicy(conf, nodeManager, cluster, true,
SCMContainerPlacementMetrics.create());
@@ -117,7 +118,7 @@ public class TestContainerPlacementFactory {
@Test
public void testDefaultPolicy() throws IOException {
- ContainerPlacementPolicy policy = ContainerPlacementPolicyFactory
+ PlacementPolicy policy = ContainerPlacementPolicyFactory
.getPolicy(conf, null, null, true, null);
Assert.assertSame(SCMContainerPlacementRandom.class, policy.getClass());
}
@@ -125,7 +126,7 @@ public class TestContainerPlacementFactory {
/**
* A dummy container placement implementation for test.
*/
- public static class DummyImpl implements ContainerPlacementPolicy {
+ public static class DummyImpl implements PlacementPolicy {
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
index 3905823..8808546 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
@@ -27,8 +27,7 @@ import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -135,7 +134,7 @@ public class TestContainerPlacement {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
testDir.getAbsolutePath());
conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
SCMNodeManager nodeManager = createNodeManager(conf);
SCMContainerManager containerManager =
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
index 5572e9a..4ad3456 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
@@ -25,8 +25,7 @@ import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ReplicationManager;
import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
@@ -70,7 +69,7 @@ public class TestSafeModeHandler {
.thenReturn(new HashSet<>());
replicationManager = new ReplicationManager(
new ReplicationManagerConfiguration(),
- containerManager, Mockito.mock(ContainerPlacementPolicy.class),
+ containerManager, Mockito.mock(PlacementPolicy.class),
eventQueue, new LockManager(configuration));
scmPipelineManager = Mockito.mock(SCMPipelineManager.class);
blockManager = Mockito.mock(BlockManagerImpl.class);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
index eadb520..17c090d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
@@ -17,11 +17,10 @@
*/
package org.apache.hadoop.ozone;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
@@ -30,7 +29,6 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.junit.Assert.assertEquals;
/**
@@ -47,8 +45,7 @@ public class TestContainerOperations {
public static void setup() throws Exception {
ozoneConf = new OzoneConfiguration();
ozoneConf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
- ozoneConf.setStorageSize(OZONE_SCM_CONTAINER_SIZE, 5, StorageUnit.GB);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(3).build();
storageClient = new ContainerOperationClient(ozoneConf);
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java
index 548f9b6..b0b3fbf 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java
@@ -29,8 +29,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.
- ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.
SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.protocolPB.
@@ -59,7 +58,7 @@ public class TestContainerStateMachineIdempotency {
public static void init() throws Exception {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster =
MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(3).build();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
index 108d204..dd29189 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.HddsDatanodeService;
@@ -84,7 +84,7 @@ public class TestDataScrubber {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.set(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, "1s");
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
ozoneConfig.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION,
false);
cluster = MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(1)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
index 9d187ff..3b6da67 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.protocolPB
@@ -60,7 +60,7 @@ public class TestContainerSmallFile {
public static void init() throws Exception {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster = MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(3)
.build();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
index cdd3f8a..0cb9329 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
@@ -31,8 +31,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.
- ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.
SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.protocolPB.
@@ -65,7 +64,7 @@ public class TestGetCommittedBlockLengthAndPutKey {
public static void init() throws Exception {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster =
MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(3).build();
cluster.waitForClusterToBeReady();
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 09/18: Revert "HDDS-2650 Fix createPipeline CLI.
(#340)"
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit b5ef117605866320b5a9aa66f8bb8231e46e5f93
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Wed Dec 18 19:47:56 2019 +0800
Revert "HDDS-2650 Fix createPipeline CLI. (#340)"
This reverts commit 7c7171082e344d51b5ef9473413db487e5267ac0.
---
.../hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
index 58a1778..edeb786 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.scm.cli.pipeline;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.SCMCLI;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import picocli.CommandLine;
@@ -29,13 +30,13 @@ import java.util.concurrent.Callable;
* Handler of createPipeline command.
*/
@CommandLine.Command(
- name = "create",
+ name = "createPipeline",
description = "create pipeline",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class CreatePipelineSubcommand implements Callable<Void> {
@CommandLine.ParentCommand
- private PipelineCommands parent;
+ private SCMCLI parent;
@CommandLine.Option(
names = {"-t", "--replicationType"},
@@ -59,7 +60,7 @@ public class CreatePipelineSubcommand implements Callable<Void> {
throw new IllegalArgumentException(type.name()
+ " is not supported yet.");
}
- try (ScmClient scmClient = parent.getParent().createScmClient()) {
+ try (ScmClient scmClient = parent.createScmClient()) {
scmClient.createReplicationPipeline(
type,
factor,
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/18: HDDS-1577. Add default pipeline placement
policy implementation. (#1366)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6ab3326806f9d148031efd672b7d2f88cff2dbd9
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Thu Sep 5 11:51:40 2019 +0800
HDDS-1577. Add default pipeline placement policy implementation. (#1366)
(cherry picked from commit b640a5f6d53830aee4b9c2a7d17bf57c987962cd)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +
.../common/src/main/resources/ozone-default.xml | 7 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 14 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 9 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 19 ++
.../hdds/scm/node/states/Node2ObjectsMap.java | 4 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 12 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 338 +++++++++++++++++++++
.../hadoop/hdds/scm/container/MockNodeManager.java | 36 ++-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 197 ++++++++++++
.../testutils/ReplicationNodeManagerMock.java | 16 +
11 files changed, 654 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index d0ac066..9fa71ad 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -292,6 +292,11 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT =
"ozone.scm.pipeline.owner.container.count";
public static final int OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT = 3;
+ // Pipeline placement policy:
+ // the max number of pipelines can a single datanode be engaged in.
+ public static final String OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT =
+ "ozone.scm.datanode.max.pipeline.engagement";
+ public static final int OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT = 5;
public static final String
OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY =
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index ecc25e9..ba465c5 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -789,6 +789,13 @@
</description>
</property>
<property>
+ <name>ozone.scm.datanode.max.pipeline.engagement</name>
+ <value>5</value>
+ <tag>OZONE, SCM, PIPELINE</tag>
+ <description>Max number of pipelines per datanode can be engaged in.
+ </description>
+ </property>
+ <property>
<name>ozone.scm.container.size</name>
<value>5GB</value>
<tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index fd8bb87..37562fe 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -118,6 +119,13 @@ public interface NodeManager extends StorageContainerNodeProtocol,
Set<PipelineID> getPipelines(DatanodeDetails datanodeDetails);
/**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param datanodeDetails DatanodeDetails
+ * @return The number of pipelines
+ */
+ int getPipelinesCount(DatanodeDetails datanodeDetails);
+
+ /**
* Add pipeline information in the NodeManager.
* @param pipeline - Pipeline to be added
*/
@@ -199,4 +207,10 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* @return the given datanode, or empty list if none found
*/
List<DatanodeDetails> getNodesByAddress(String address);
+
+ /**
+ * Get cluster map as in network topology for this node manager.
+ * @return cluster map
+ */
+ NetworkTopology getClusterNetworkTopologyMap();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index 954cb0e..9d2a9f2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@ -284,6 +284,15 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
+ * Get the count of pipelines associated to single datanode.
+ * @param datanodeDetails single datanode
+ * @return number of pipelines associated with it
+ */
+ public int getPipelinesCount(DatanodeDetails datanodeDetails) {
+ return node2PipelineMap.getPipelinesCount(datanodeDetails.getUuid());
+ }
+
+ /**
* Get information about the node.
*
* @param datanodeDetails DatanodeDetails
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index d84b75b..46534fb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -512,6 +512,16 @@ public class SCMNodeManager implements NodeManager {
}
/**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param datanodeDetails DatanodeDetails
+ * @return The number of pipelines
+ */
+ @Override
+ public int getPipelinesCount(DatanodeDetails datanodeDetails) {
+ return nodeStateManager.getPipelinesCount(datanodeDetails);
+ }
+
+ /**
* Add pipeline information in the NodeManager.
*
* @param pipeline - Pipeline to be added
@@ -645,6 +655,15 @@ public class SCMNodeManager implements NodeManager {
return results;
}
+ /**
+ * Get cluster map as in network topology for this node manager.
+ * @return cluster map
+ */
+ @Override
+ public NetworkTopology getClusterNetworkTopologyMap() {
+ return clusterMap;
+ }
+
private String nodeResolve(String hostname) {
List<String> hosts = new ArrayList<>(1);
hosts.add(hostname);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
index 37525b0..57a377d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
@@ -67,6 +67,7 @@ public class Node2ObjectsMap<T> {
* @param datanodeID -- Datanode UUID
* @param containerIDs - List of ContainerIDs.
*/
+ @VisibleForTesting
public void insertNewDatanode(UUID datanodeID, Set<T> containerIDs)
throws SCMException {
Preconditions.checkNotNull(containerIDs);
@@ -83,7 +84,8 @@ public class Node2ObjectsMap<T> {
*
* @param datanodeID - Datanode ID.
*/
- void removeDatanode(UUID datanodeID) {
+ @VisibleForTesting
+ public void removeDatanode(UUID datanodeID) {
Preconditions.checkNotNull(datanodeID);
dn2ObjectMap.computeIfPresent(datanodeID, (k, v) -> null);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
index f8633f9..714188d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
@@ -42,7 +42,7 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
}
/**
- * Returns null if there no pipelines associated with this datanode ID.
+ * Returns null if there are no pipelines associated with this datanode ID.
*
* @param datanode - UUID
* @return Set of pipelines or Null.
@@ -52,6 +52,16 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
}
/**
+ * Return 0 if there are no pipelines associated with this datanode ID.
+ * @param datanode - UUID
+ * @return Number of pipelines or 0.
+ */
+ public int getPipelinesCount(UUID datanode) {
+ Set<PipelineID> pipelines = getObjects(datanode);
+ return pipelines == null ? 0 : pipelines.size();
+ }
+
+ /**
* Adds a pipeline entry to a given dataNode in the map.
*
* @param pipeline Pipeline to be added
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
new file mode 100644
index 0000000..cb9954d
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -0,0 +1,338 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMCommonPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.Node;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Pipeline placement policy that choose datanodes based on load balancing
+ * and network topology to supply pipeline creation.
+ * <p>
+ * 1. get a list of healthy nodes
+ * 2. filter out nodes that are not too heavily engaged in other pipelines
+ * 3. Choose an anchor node among the viable nodes.
+ * 4. Choose other nodes around the anchor node based on network topology
+ */
+public final class PipelinePlacementPolicy extends SCMCommonPolicy {
+ @VisibleForTesting
+ static final Logger LOG =
+ LoggerFactory.getLogger(PipelinePlacementPolicy.class);
+ private final NodeManager nodeManager;
+ private final Configuration conf;
+ private final int heavyNodeCriteria;
+
+ /**
+ * Constructs a pipeline placement with considering network topology,
+ * load balancing and rack awareness.
+ *
+ * @param nodeManager Node Manager
+ * @param conf Configuration
+ */
+ public PipelinePlacementPolicy(
+ final NodeManager nodeManager, final Configuration conf) {
+ super(nodeManager, conf);
+ this.nodeManager = nodeManager;
+ this.conf = conf;
+ heavyNodeCriteria = conf.getInt(
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT);
+ }
+
+ /**
+ * Returns true if this node meets the criteria.
+ *
+ * @param datanodeDetails DatanodeDetails
+ * @return true if we have enough space.
+ */
+ @VisibleForTesting
+ boolean meetCriteria(DatanodeDetails datanodeDetails, long heavyNodeLimit) {
+ return (nodeManager.getPipelinesCount(datanodeDetails) <= heavyNodeLimit);
+ }
+
+ /**
+ * Filter out viable nodes based on
+ * 1. nodes that are healthy
+ * 2. nodes that are not too heavily engaged in other pipelines
+ *
+ * @param excludedNodes - excluded nodes
+ * @param nodesRequired - number of datanodes required.
+ * @return a list of viable nodes
+ * @throws SCMException when viable nodes are not enough in numbers
+ */
+ List<DatanodeDetails> filterViableNodes(
+ List<DatanodeDetails> excludedNodes, int nodesRequired)
+ throws SCMException {
+ // get nodes in HEALTHY state
+ List<DatanodeDetails> healthyNodes =
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
+ if (excludedNodes != null) {
+ healthyNodes.removeAll(excludedNodes);
+ }
+ String msg;
+ if (healthyNodes.size() == 0) {
+ msg = "No healthy node found to allocate pipeline.";
+ LOG.error(msg);
+ throw new SCMException(msg, SCMException.ResultCodes
+ .FAILED_TO_FIND_HEALTHY_NODES);
+ }
+
+ if (healthyNodes.size() < nodesRequired) {
+ msg = String.format("Not enough healthy nodes to allocate pipeline. %d "
+ + " datanodes required. Found %d",
+ nodesRequired, healthyNodes.size());
+ LOG.error(msg);
+ throw new SCMException(msg,
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ // filter nodes that meet the size and pipeline engagement criteria.
+ // Pipeline placement doesn't take node space left into account.
+ List<DatanodeDetails> healthyList = healthyNodes.stream().filter(d ->
+ meetCriteria(d, heavyNodeCriteria)).collect(Collectors.toList());
+
+ if (healthyList.size() < nodesRequired) {
+ msg = String.format("Unable to find enough nodes that meet " +
+ "the criteria that cannot engage in more than %d pipelines." +
+ " Nodes required: %d Found: %d",
+ heavyNodeCriteria, nodesRequired, healthyList.size());
+ LOG.error(msg);
+ throw new SCMException(msg,
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return healthyList;
+ }
+
+ /**
+ * Pipeline placement choose datanodes to join the pipeline.
+ *
+ * @param excludedNodes - excluded nodes
+ * @param favoredNodes - list of nodes preferred.
+ * @param nodesRequired - number of datanodes required.
+ * @param sizeRequired - size required for the container or block.
+ * @return a list of chosen datanodeDetails
+ * @throws SCMException when chosen nodes are not enough in numbers
+ */
+ @Override
+ public List<DatanodeDetails> chooseDatanodes(
+ List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
+ int nodesRequired, final long sizeRequired) throws SCMException {
+ // get a list of viable nodes based on criteria
+ List<DatanodeDetails> healthyNodes =
+ filterViableNodes(excludedNodes, nodesRequired);
+
+ List<DatanodeDetails> results = new ArrayList<>();
+
+ // Randomly picks nodes when all nodes are equal.
+ // This happens when network topology is absent or
+ // all nodes are on the same rack.
+ if (checkAllNodesAreEqual(nodeManager.getClusterNetworkTopologyMap())) {
+ LOG.info("All nodes are considered equal. Now randomly pick nodes. " +
+ "Required nodes: {}", nodesRequired);
+ results = super.getResultSet(nodesRequired, healthyNodes);
+ if (results.size() < nodesRequired) {
+ LOG.error("Unable to find the required number of healthy nodes that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return results;
+ }
+
+ // Since nodes are widely distributed, the results should be selected
+ // base on distance in topology, rack awareness and load balancing.
+ List<DatanodeDetails> exclude = new ArrayList<>();
+ exclude.addAll(excludedNodes);
+ // First choose an anchor nodes randomly
+ DatanodeDetails anchor = chooseNode(healthyNodes);
+ if (anchor == null) {
+ LOG.error("Unable to find the first healthy nodes that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ results.add(anchor);
+ exclude.add(anchor);
+ nodesRequired--;
+
+ // Choose the second node on different racks from anchor.
+ DatanodeDetails nodeOnDifferentRack = chooseNodeBasedOnRackAwareness(
+ healthyNodes, excludedNodes,
+ nodeManager.getClusterNetworkTopologyMap(), anchor);
+ if (nodeOnDifferentRack == null) {
+ LOG.error("Unable to find nodes on different racks that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ results.add(nodeOnDifferentRack);
+ exclude.add(nodeOnDifferentRack);
+ nodesRequired--;
+
+ // Then choose nodes close to anchor based on network topology
+ for (int x = 0; x < nodesRequired; x++) {
+ // invoke the choose function defined in the derived classes.
+ DatanodeDetails pick = chooseNodeFromNetworkTopology(
+ nodeManager.getClusterNetworkTopologyMap(), anchor, exclude);
+ if (pick != null) {
+ results.add(pick);
+ // exclude the picked node for next time
+ exclude.add(pick);
+ }
+ }
+
+ if (results.size() < nodesRequired) {
+ LOG.error("Unable to find the required number of healthy nodes that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return results;
+ }
+
+ /**
+ * Find a node from the healthy list and return it after removing it from the
+ * list that we are operating on.
+ *
+ * @param healthyNodes - Set of healthy nodes we can choose from.
+ * @return chosen datanodDetails
+ */
+ @Override
+ public DatanodeDetails chooseNode(
+ List<DatanodeDetails> healthyNodes) {
+ int firstNodeNdx = getRand().nextInt(healthyNodes.size());
+ int secondNodeNdx = getRand().nextInt(healthyNodes.size());
+
+ DatanodeDetails datanodeDetails;
+ // There is a possibility that both numbers will be same.
+ // if that is so, we just return the node.
+ if (firstNodeNdx == secondNodeNdx) {
+ datanodeDetails = healthyNodes.get(firstNodeNdx);
+ } else {
+ DatanodeDetails firstNodeDetails = healthyNodes.get(firstNodeNdx);
+ DatanodeDetails secondNodeDetails = healthyNodes.get(secondNodeNdx);
+ SCMNodeMetric firstNodeMetric =
+ nodeManager.getNodeStat(firstNodeDetails);
+ SCMNodeMetric secondNodeMetric =
+ nodeManager.getNodeStat(secondNodeDetails);
+ datanodeDetails = firstNodeMetric.isGreater(secondNodeMetric.get())
+ ? firstNodeDetails : secondNodeDetails;
+ }
+ // the pick is decided and it should be removed from candidates.
+ healthyNodes.remove(datanodeDetails);
+ return datanodeDetails;
+ }
+
+ /**
+ * Choose node on different racks as anchor is on based on rack awareness.
+ * If a node on different racks cannot be found, then return a random node.
+ * @param healthyNodes healthy nodes
+ * @param excludedNodes excluded nodes
+ * @param networkTopology network topology
+ * @param anchor anchor node
+ * @return a node on different rack
+ */
+ @VisibleForTesting
+ protected DatanodeDetails chooseNodeBasedOnRackAwareness(
+ List<DatanodeDetails> healthyNodes, List<DatanodeDetails> excludedNodes,
+ NetworkTopology networkTopology, DatanodeDetails anchor) {
+ Preconditions.checkArgument(networkTopology != null);
+ if (checkAllNodesAreEqual(networkTopology)) {
+ return null;
+ }
+
+ for (DatanodeDetails node : healthyNodes) {
+ if (excludedNodes.contains(node)
+ || networkTopology.isSameParent(anchor, node)) {
+ continue;
+ } else {
+ // the pick is decided and it should be removed from candidates.
+ healthyNodes.remove(node);
+ return node;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Check if all nodes are equal in topology.
+ * They are equal when network topology is absent or there are on
+ * the same rack.
+ * @param topology network topology
+ * @return true when all nodes are equal
+ */
+ private boolean checkAllNodesAreEqual(NetworkTopology topology) {
+ if (topology == null) {
+ return true;
+ }
+ return (topology.getNumOfNodes(topology.getMaxLevel() - 1) == 1);
+ }
+
+ /**
+ * Choose node based on network topology.
+ * @param networkTopology network topology
+ * @param anchor anchor datanode to start with
+ * @param excludedNodes excluded datanodes
+ * @return chosen datanode
+ */
+ @VisibleForTesting
+ protected DatanodeDetails chooseNodeFromNetworkTopology(
+ NetworkTopology networkTopology, DatanodeDetails anchor,
+ List<DatanodeDetails> excludedNodes) {
+ Preconditions.checkArgument(networkTopology != null);
+
+ Collection<Node> excluded = new ArrayList<>();
+ if (excludedNodes != null && excludedNodes.size() != 0) {
+ excluded.addAll(excludedNodes);
+ }
+ excluded.add(anchor);
+
+ Node pick = networkTopology.chooseRandom(
+ anchor.getNetworkLocation(), excluded);
+ DatanodeDetails pickedNode = (DatanodeDetails) pick;
+ // exclude the picked node for next time
+ if (excludedNodes != null) {
+ excludedNodes.add(pickedNode);
+ }
+ return pickedNode;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 06dc675..bca4189 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -17,10 +17,12 @@
package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.net.NetConstants;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.net.Node;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
@@ -86,7 +88,7 @@ public class MockNodeManager implements NodeManager {
private final SCMNodeStat aggregateStat;
private boolean safemode;
private final Map<UUID, List<SCMCommand>> commandMap;
- private final Node2PipelineMap node2PipelineMap;
+ private Node2PipelineMap node2PipelineMap;
private final Node2ContainerMap node2ContainerMap;
private NetworkTopology clusterMap;
private ConcurrentMap<String, Set<String>> dnsToUuidMap;
@@ -100,6 +102,7 @@ public class MockNodeManager implements NodeManager {
this.node2ContainerMap = new Node2ContainerMap();
this.dnsToUuidMap = new ConcurrentHashMap<>();
aggregateStat = new SCMNodeStat();
+ clusterMap = new NetworkTopologyImpl(new Configuration());
if (initializeFakeNodes) {
for (int x = 0; x < nodeCount; x++) {
DatanodeDetails dd = MockDatanodeDetails.randomDatanodeDetails();
@@ -251,6 +254,16 @@ public class MockNodeManager implements NodeManager {
}
/**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param datanodeDetails DatanodeDetails
+ * @return The number of pipelines
+ */
+ @Override
+ public int getPipelinesCount(DatanodeDetails datanodeDetails) {
+ return node2PipelineMap.getPipelinesCount(datanodeDetails.getUuid());
+ }
+
+ /**
* Add pipeline information in the NodeManager.
* @param pipeline - Pipeline to be added
*/
@@ -260,6 +273,22 @@ public class MockNodeManager implements NodeManager {
}
/**
+ * Get the entire Node2PipelineMap.
+ * @return Node2PipelineMap
+ */
+ public Node2PipelineMap getNode2PipelineMap() {
+ return node2PipelineMap;
+ }
+
+ /**
+ * Set the Node2PipelineMap.
+ * @param node2PipelineMap Node2PipelineMap
+ */
+ public void setNode2PipelineMap(Node2PipelineMap node2PipelineMap) {
+ this.node2PipelineMap = node2PipelineMap;
+ }
+
+ /**
* Remove a pipeline information from the NodeManager.
* @param pipeline - Pipeline to be removed
*/
@@ -517,6 +546,11 @@ public class MockNodeManager implements NodeManager {
return results;
}
+ @Override
+ public NetworkTopology getClusterNetworkTopologyMap() {
+ return clusterMap;
+ }
+
public void setNetworkTopology(NetworkTopology topology) {
this.clusterMap = topology;
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
new file mode 100644
index 0000000..2e0d0b1
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.*;
+import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Test for PipelinePlacementPolicy.
+ */
+public class TestPipelinePlacementPolicy {
+ private MockNodeManager nodeManager;
+ private PipelinePlacementPolicy placementPolicy;
+ private static final int PIPELINE_PLACEMENT_MAX_NODES_COUNT = 10;
+
+ @Before
+ public void init() throws Exception {
+ nodeManager = new MockNodeManager(true,
+ PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+ placementPolicy =
+ new PipelinePlacementPolicy(nodeManager, new OzoneConfiguration());
+ }
+
+ @Test
+ public void testChooseNodeBasedOnNetworkTopology() {
+ List<DatanodeDetails> healthyNodes =
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
+ DatanodeDetails anchor = placementPolicy.chooseNode(healthyNodes);
+ // anchor should be removed from healthyNodes after being chosen.
+ Assert.assertFalse(healthyNodes.contains(anchor));
+
+ List<DatanodeDetails> excludedNodes =
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+ DatanodeDetails nextNode = placementPolicy.chooseNodeFromNetworkTopology(
+ nodeManager.getClusterNetworkTopologyMap(), anchor, excludedNodes);
+ // excludedNodes should contain nextNode after being chosen.
+ Assert.assertTrue(excludedNodes.contains(nextNode));
+ // nextNode should not be the same as anchor.
+ Assert.assertTrue(anchor.getUuid() != nextNode.getUuid());
+ }
+
+ @Test
+ public void testChooseNodeBasedOnRackAwareness() {
+ List<DatanodeDetails> healthyNodes = overWriteLocationInNodes(
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY));
+ DatanodeDetails anchor = placementPolicy.chooseNode(healthyNodes);
+ NetworkTopology topologyWithDifRacks =
+ createNetworkTopologyOnDifRacks();
+ DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnRackAwareness(
+ healthyNodes, new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ topologyWithDifRacks, anchor);
+ Assert.assertFalse(topologyWithDifRacks.isSameParent(anchor, nextNode));
+ }
+
+ private final static Node[] NODES = new NodeImpl[] {
+ new NodeImpl("h1", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h2", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h3", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h4", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h5", "/r2", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h6", "/r2", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h7", "/r2", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h8", "/r2", NetConstants.NODE_COST_DEFAULT),
+ };
+
+
+ private NetworkTopology createNetworkTopologyOnDifRacks() {
+ NetworkTopology topology = new NetworkTopologyImpl(new Configuration());
+ for (Node n : NODES) {
+ topology.add(n);
+ }
+ return topology;
+ }
+
+ private List<DatanodeDetails> overWriteLocationInNodes(
+ List<DatanodeDetails> datanodes) {
+ List<DatanodeDetails> results = new ArrayList<>(datanodes.size());
+ for (int i = 0; i < datanodes.size(); i++) {
+ DatanodeDetails datanode = datanodes.get(i);
+ DatanodeDetails result = DatanodeDetails.newBuilder()
+ .setUuid(datanode.getUuidString())
+ .setHostName(datanode.getHostName())
+ .setIpAddress(datanode.getIpAddress())
+ .addPort(datanode.getPort(DatanodeDetails.Port.Name.STANDALONE))
+ .addPort(datanode.getPort(DatanodeDetails.Port.Name.RATIS))
+ .addPort(datanode.getPort(DatanodeDetails.Port.Name.REST))
+ .setNetworkLocation(NODES[i].getNetworkLocation()).build();
+ results.add(result);
+ }
+ return results;
+ }
+
+ @Test
+ public void testHeavyNodeShouldBeExcluded() throws SCMException{
+ List<DatanodeDetails> healthyNodes =
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
+ int nodesRequired = healthyNodes.size()/2;
+ // only minority of healthy NODES are heavily engaged in pipelines.
+ int minorityHeavy = healthyNodes.size()/2 - 1;
+ List<DatanodeDetails> pickedNodes1 = placementPolicy.chooseDatanodes(
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ nodesRequired, 0);
+ // modify node to pipeline mapping.
+ insertHeavyNodesIntoNodeManager(healthyNodes, minorityHeavy);
+ // NODES should be sufficient.
+ Assert.assertEquals(nodesRequired, pickedNodes1.size());
+ // make sure pipeline placement policy won't select duplicated NODES.
+ Assert.assertTrue(checkDuplicateNodesUUID(pickedNodes1));
+
+ // majority of healthy NODES are heavily engaged in pipelines.
+ int majorityHeavy = healthyNodes.size()/2 + 2;
+ insertHeavyNodesIntoNodeManager(healthyNodes, majorityHeavy);
+ boolean thrown = false;
+ List<DatanodeDetails> pickedNodes2 = null;
+ try {
+ pickedNodes2 = placementPolicy.chooseDatanodes(
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ nodesRequired, 0);
+ } catch (SCMException e) {
+ Assert.assertFalse(thrown);
+ thrown = true;
+ }
+ // NODES should NOT be sufficient and exception should be thrown.
+ Assert.assertNull(pickedNodes2);
+ Assert.assertTrue(thrown);
+ }
+
+ private boolean checkDuplicateNodesUUID(List<DatanodeDetails> nodes) {
+ HashSet<UUID> uuids = nodes.stream().
+ map(DatanodeDetails::getUuid).
+ collect(Collectors.toCollection(HashSet::new));
+ return uuids.size() == nodes.size();
+ }
+
+ private Set<PipelineID> mockPipelineIDs(int count) {
+ Set<PipelineID> pipelineIDs = new HashSet<>(count);
+ for (int i = 0; i < count; i++) {
+ pipelineIDs.add(PipelineID.randomId());
+ }
+ return pipelineIDs;
+ }
+
+ private void insertHeavyNodesIntoNodeManager(
+ List<DatanodeDetails> nodes, int heavyNodeCount) throws SCMException{
+ if (nodes == null) {
+ throw new SCMException("",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ int considerHeavyCount =
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT + 1;
+
+ Node2PipelineMap mockMap = new Node2PipelineMap();
+ for (DatanodeDetails node : nodes) {
+ // mock heavy node
+ if (heavyNodeCount > 0) {
+ mockMap.insertNewDatanode(
+ node.getUuid(), mockPipelineIDs(considerHeavyCount));
+ heavyNodeCount--;
+ } else {
+ mockMap.insertNewDatanode(node.getUuid(), mockPipelineIDs(1));
+ }
+ }
+ nodeManager.setNode2PipelineMap(mockMap);
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 0ecff3f..7e8ec52 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -20,6 +20,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -166,6 +167,16 @@ public class ReplicationNodeManagerMock implements NodeManager {
}
/**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param dnId DatanodeDetails
+ * @return The number of pipelines
+ */
+ @Override
+ public int getPipelinesCount(DatanodeDetails dnId) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
* Add pipeline information in the NodeManager.
* @param pipeline - Pipeline to be added
*/
@@ -327,4 +338,9 @@ public class ReplicationNodeManagerMock implements NodeManager {
public List<DatanodeDetails> getNodesByAddress(String address) {
return new LinkedList<>();
}
+
+ @Override
+ public NetworkTopology getClusterNetworkTopologyMap() {
+ return null;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 06/18: Rebase Fix
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit e98c12f512101585e9402b32ccd0ab42e87a7610
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Tue Dec 3 16:32:08 2019 +0800
Rebase Fix
---
.../apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java | 6 +++---
.../org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java | 6 ++++--
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
index 7513cad..ff52470 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
@@ -36,9 +36,9 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
private boolean autoOpenPipeline;
public MockRatisPipelineProvider(NodeManager nodeManager,
- PipelineStateManager stateManager,
- Configuration conf, boolean autoOpen) {
- super(nodeManager, stateManager, conf, null);
+ PipelineStateManager stateManager, Configuration conf,
+ EventPublisher eventPublisher, boolean autoOpen) {
+ super(nodeManager, stateManager, conf, eventPublisher);
autoOpenPipeline = autoOpen;
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index 2df851d..491e289 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -383,10 +383,12 @@ public class TestSCMPipelineManager {
OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, -1,
TimeUnit.MILLISECONDS);
+ EventQueue eventQueue = new EventQueue();
final SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, new EventQueue(), null);
+ new SCMPipelineManager(conf, nodeManager, eventQueue);
final PipelineProvider ratisProvider = new MockRatisPipelineProvider(
- nodeManager, pipelineManager.getStateManager(), conf, false);
+ nodeManager, pipelineManager.getStateManager(), conf, eventQueue,
+ false);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
ratisProvider);
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 10/18: HDDS-2650 Fix createPipeline CLI and make it
message based. (#370)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6cdfdcb4dc16cc28f6c4f419c455913fa316e965
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Wed Dec 18 22:27:04 2019 +0800
HDDS-2650 Fix createPipeline CLI and make it message based. (#370)
---
...inerLocationProtocolServerSideTranslatorPB.java | 26 ++++++++++++++++++++++
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 7 +++---
2 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 5ff75e7..f2e4253 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ActivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ActivatePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ClosePipelineRequestProto;
@@ -72,6 +73,9 @@ import com.google.protobuf.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto.Error.errorPipelineAlreadyExists;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto.Error.success;
+
/**
* This class is the server-side translator that forwards requests received on
* {@link StorageContainerLocationProtocolPB} to the
@@ -160,6 +164,12 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
.setScmCloseContainerResponse(closeContainer(
request.getScmCloseContainerRequest()))
.build();
+ case AllocatePipeline:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setPipelineResponse(allocatePipeline(request.getPipelineRequest()))
+ .build();
case ListPipelines:
return ScmContainerLocationResponse.newBuilder()
.setCmdType(request.getCmdType())
@@ -327,6 +337,22 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
return SCMCloseContainerResponseProto.newBuilder().build();
}
+ public PipelineResponseProto allocatePipeline(
+ StorageContainerLocationProtocolProtos.PipelineRequestProto request)
+ throws IOException {
+ Pipeline pipeline = impl.createReplicationPipeline(
+ request.getReplicationType(), request.getReplicationFactor(),
+ HddsProtos.NodePool.getDefaultInstance());
+ if (pipeline == null) {
+ return PipelineResponseProto.newBuilder()
+ .setErrorCode(errorPipelineAlreadyExists).build();
+ }
+ PipelineResponseProto response = PipelineResponseProto.newBuilder()
+ .setErrorCode(success)
+ .setPipeline(pipeline.getProtobufMessage()).build();
+ return response;
+ }
+
public ListPipelineResponseProto listPipelines(
ListPipelineRequestProto request)
throws IOException {
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
index edeb786..58a1778 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.cli.pipeline;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.cli.SCMCLI;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import picocli.CommandLine;
@@ -30,13 +29,13 @@ import java.util.concurrent.Callable;
* Handler of createPipeline command.
*/
@CommandLine.Command(
- name = "createPipeline",
+ name = "create",
description = "create pipeline",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class CreatePipelineSubcommand implements Callable<Void> {
@CommandLine.ParentCommand
- private SCMCLI parent;
+ private PipelineCommands parent;
@CommandLine.Option(
names = {"-t", "--replicationType"},
@@ -60,7 +59,7 @@ public class CreatePipelineSubcommand implements Callable<Void> {
throw new IllegalArgumentException(type.name()
+ " is not supported yet.");
}
- try (ScmClient scmClient = parent.createScmClient()) {
+ try (ScmClient scmClient = parent.getParent().createScmClient()) {
scmClient.createReplicationPipeline(
type,
factor,
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 04/18: HDDS-1569 Support creating multiple pipelines
with same datanode. Contributed by Li Cheng.
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 1ef5e6df3ed15e93296a9c0f3c2460cdec423e00
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Tue Oct 29 12:46:00 2019 +0800
HDDS-1569 Support creating multiple pipelines with same datanode. Contributed by Li Cheng.
This closes #28
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../common/src/main/resources/ozone-default.xml | 15 ++-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 5 +
.../ContainerPlacementPolicyFactory.java | 8 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 2 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 1 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 89 +++++++++----
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 5 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 137 ++++++++++++---------
.../hdds/scm/pipeline/RatisPipelineUtils.java | 96 +++++++++++++++
.../hdds/scm/pipeline/SCMPipelineManager.java | 13 +-
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 8 ++
.../scm/safemode/HealthyPipelineSafeModeRule.java | 13 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 3 +
.../scm/pipeline/TestPipelinePlacementPolicy.java | 15 ++-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 17 +--
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 3 +
.../hdds/scm/pipeline/TestPipelineClose.java | 1 +
.../TestRatisPipelineCreateAndDestroy.java | 24 +++-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 5 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 3 +
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 12 ++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 4 +
.../ozone/client/rpc/Test2WayCommitInRatis.java | 1 +
.../ozone/client/rpc/TestBlockOutputStream.java | 1 +
.../rpc/TestBlockOutputStreamWithFailures.java | 7 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 1 +
.../rpc/TestContainerReplicationEndToEnd.java | 5 +-
.../client/rpc/TestContainerStateMachine.java | 5 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 12 +-
.../client/rpc/TestFailureHandlingByClient.java | 4 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 3 +-
.../ozone/client/rpc/TestKeyInputStream.java | 1 +
.../rpc/TestMultiBlockWritesWithDnFailures.java | 8 +-
.../rpc/TestOzoneClientRetriesOnException.java | 1 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 1 +
.../ozone/client/rpc/TestWatchForCommit.java | 3 +
.../TestCloseContainerByPipeline.java | 5 +
.../hadoop/ozone/freon/TestDataValidate.java | 2 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 1 +
.../TestSCMContainerPlacementPolicyMetrics.java | 1 +
.../hadoop/ozone/scm/node/TestQueryNode.java | 3 +
42 files changed, 419 insertions(+), 135 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 9fa71ad..15a47f3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -296,7 +296,15 @@ public final class ScmConfigKeys {
// the max number of pipelines can a single datanode be engaged in.
public static final String OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT =
"ozone.scm.datanode.max.pipeline.engagement";
- public static final int OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT = 5;
+ // Setting to zero by default means this limit doesn't take effect.
+ public static final int OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT = 0;
+
+ // Upper limit for how many pipelines can be created.
+ // Only for test purpose now.
+ public static final String OZONE_SCM_PIPELINE_NUMBER_LIMIT =
+ "ozone.scm.pipeline.number.limit";
+ // Setting to zero by default means this limit doesn't take effect.
+ public static final int OZONE_SCM_PIPELINE_NUMBER_LIMIT_DEFAULT = 0;
public static final String
OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY =
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 4144500..8bc2ea1 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -791,10 +791,19 @@
</description>
</property>
<property>
- <name>ozone.scm.datanode.max.pipeline.engagement</name>
- <value>5</value>
+ <name>ozone.scm.datanode.max.pipeline.engagement</name>
+ <value>0</value>
+ <tag>OZONE, SCM, PIPELINE</tag>
+ <description>Max number of pipelines per datanode can be engaged in.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.pipeline.number.limit</name>
+ <value>0</value>
<tag>OZONE, SCM, PIPELINE</tag>
- <description>Max number of pipelines per datanode can be engaged in.
+ <description>Upper limit for how many pipelines can be OPEN in SCM.
+ 0 as default means there is no limit. Otherwise, the number is the limit
+ of max amount of pipelines which are OPEN.
</description>
</property>
<property>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index b7a7525..cdc3878 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -197,8 +197,13 @@ public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean {
// TODO: #CLUTIL Remove creation logic when all replication types and
// factors are handled by pipeline creator
pipeline = pipelineManager.createPipeline(type, factor);
+
// wait until pipeline is ready
pipelineManager.waitPipelineReady(pipeline.getId(), 0);
+ } catch (SCMException se) {
+ LOG.warn("Pipeline creation failed for type:{} factor:{}. " +
+ "Datanodes may be used up.", type, factor, se);
+ break;
} catch (IOException e) {
LOG.warn("Pipeline creation failed for type:{} factor:{}. Retrying " +
"get pipelines call once.", type, factor, e);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
index adaeb87..74431f9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
@@ -43,10 +43,10 @@ public final class ContainerPlacementPolicyFactory {
}
- public static PlacementPolicy getPolicy(Configuration conf,
- final NodeManager nodeManager, NetworkTopology clusterMap,
- final boolean fallback, SCMContainerPlacementMetrics metrics)
- throws SCMException{
+ public static PlacementPolicy getPolicy(
+ Configuration conf, final NodeManager nodeManager,
+ NetworkTopology clusterMap, final boolean fallback,
+ SCMContainerPlacementMetrics metrics) throws SCMException{
final Class<? extends PlacementPolicy> placementClass = conf
.getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
index 714188d..18809ed 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
@@ -80,7 +80,7 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
dn2ObjectMap.computeIfPresent(dnId,
(k, v) -> {
v.remove(pipeline.getId());
- return v;
+ return v.isEmpty() ? null : v;
});
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index 3006987..b663f2a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -115,6 +115,7 @@ class BackgroundPipelineCreator {
if (scheduler.isClosed()) {
break;
}
+
pipelineManager.createPipeline(type, factor);
} catch (IOException ioe) {
break;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index 1983ed6..23eb574 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -52,6 +53,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
static final Logger LOG =
LoggerFactory.getLogger(PipelinePlacementPolicy.class);
private final NodeManager nodeManager;
+ private final PipelineStateManager stateManager;
private final Configuration conf;
private final int heavyNodeCriteria;
@@ -59,15 +61,17 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
* Constructs a pipeline placement with considering network topology,
* load balancing and rack awareness.
*
- * @param nodeManager Node Manager
+ * @param nodeManager NodeManager
+ * @param stateManager PipelineStateManager
* @param conf Configuration
*/
- public PipelinePlacementPolicy(
- final NodeManager nodeManager, final Configuration conf) {
+ public PipelinePlacementPolicy(final NodeManager nodeManager,
+ final PipelineStateManager stateManager, final Configuration conf) {
super(nodeManager, conf);
this.nodeManager = nodeManager;
this.conf = conf;
- heavyNodeCriteria = conf.getInt(
+ this.stateManager = stateManager;
+ this.heavyNodeCriteria = conf.getInt(
ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT);
}
@@ -76,11 +80,46 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
* Returns true if this node meets the criteria.
*
* @param datanodeDetails DatanodeDetails
+ * @param nodesRequired nodes required count
* @return true if we have enough space.
*/
@VisibleForTesting
- boolean meetCriteria(DatanodeDetails datanodeDetails, long heavyNodeLimit) {
- return (nodeManager.getPipelinesCount(datanodeDetails) <= heavyNodeLimit);
+ boolean meetCriteria(DatanodeDetails datanodeDetails, int nodesRequired) {
+ if (heavyNodeCriteria == 0) {
+ // no limit applied.
+ return true;
+ }
+ // Datanodes from pipeline in some states can also be considered available
+ // for pipeline allocation. Thus the number of these pipeline shall be
+ // deducted from total heaviness calculation.
+ int pipelineNumDeductable = 0;
+ Set<PipelineID> pipelines = nodeManager.getPipelines(datanodeDetails);
+ for (PipelineID pid : pipelines) {
+ Pipeline pipeline;
+ try {
+ pipeline = stateManager.getPipeline(pid);
+ } catch (PipelineNotFoundException e) {
+ LOG.error("Pipeline not found in pipeline state manager during" +
+ " pipeline creation. PipelineID: " + pid +
+ " exception: " + e.getMessage());
+ continue;
+ }
+ if (pipeline != null &&
+ pipeline.getFactor().getNumber() == nodesRequired &&
+ pipeline.getType() == HddsProtos.ReplicationType.RATIS &&
+ pipeline.getPipelineState() == Pipeline.PipelineState.CLOSED) {
+ pipelineNumDeductable++;
+ }
+ }
+ boolean meet = (nodeManager.getPipelinesCount(datanodeDetails)
+ - pipelineNumDeductable) < heavyNodeCriteria;
+ if (!meet) {
+ LOG.info("Pipeline Placement: can't place more pipeline on heavy " +
+ "datanode： " + datanodeDetails.getUuid().toString() + " Heaviness: " +
+ nodeManager.getPipelinesCount(datanodeDetails) + " limit: " +
+ heavyNodeCriteria);
+ }
+ return meet;
}
/**
@@ -102,18 +141,19 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
if (excludedNodes != null) {
healthyNodes.removeAll(excludedNodes);
}
+ int initialHealthyNodesCount = healthyNodes.size();
String msg;
- if (healthyNodes.size() == 0) {
+ if (initialHealthyNodesCount == 0) {
msg = "No healthy node found to allocate pipeline.";
LOG.error(msg);
throw new SCMException(msg, SCMException.ResultCodes
.FAILED_TO_FIND_HEALTHY_NODES);
}
- if (healthyNodes.size() < nodesRequired) {
+ if (initialHealthyNodesCount < nodesRequired) {
msg = String.format("Not enough healthy nodes to allocate pipeline. %d "
+ " datanodes required. Found %d",
- nodesRequired, healthyNodes.size());
+ nodesRequired, initialHealthyNodesCount);
LOG.error(msg);
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
@@ -121,14 +161,17 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
// filter nodes that meet the size and pipeline engagement criteria.
// Pipeline placement doesn't take node space left into account.
- List<DatanodeDetails> healthyList = healthyNodes.stream().filter(d ->
- meetCriteria(d, heavyNodeCriteria)).collect(Collectors.toList());
+ List<DatanodeDetails> healthyList = healthyNodes.stream()
+ .filter(d -> meetCriteria(d, nodesRequired)).limit(nodesRequired)
+ .collect(Collectors.toList());
if (healthyList.size() < nodesRequired) {
msg = String.format("Unable to find enough nodes that meet " +
"the criteria that cannot engage in more than %d pipelines." +
- " Nodes required: %d Found: %d",
- heavyNodeCriteria, nodesRequired, healthyList.size());
+ " Nodes required: %d Found: %d, healthy nodes count in " +
+ "NodeManager: %d.",
+ heavyNodeCriteria, nodesRequired, healthyList.size(),
+ initialHealthyNodesCount);
LOG.error(msg);
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
@@ -154,13 +197,11 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
// and make sure excludedNodes are excluded from list.
List<DatanodeDetails> healthyNodes =
filterViableNodes(excludedNodes, nodesRequired);
-
- // Randomly picks nodes when all nodes are equal.
+
+ // Randomly picks nodes when all nodes are equal or factor is ONE.
// This happens when network topology is absent or
// all nodes are on the same rack.
if (checkAllNodesAreEqual(nodeManager.getClusterNetworkTopologyMap())) {
- LOG.info("All nodes are considered equal. Now randomly pick nodes. " +
- "Required nodes: {}", nodesRequired);
return super.getResultSet(nodesRequired, healthyNodes);
} else {
// Since topology and rack awareness are available, picks nodes
@@ -188,8 +229,8 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
// First choose an anchor nodes randomly
DatanodeDetails anchor = chooseNode(healthyNodes);
if (anchor == null) {
- LOG.error("Unable to find the first healthy nodes that " +
- "meet the criteria. Required nodes: {}, Found nodes: {}",
+ LOG.error("Pipeline Placement: Unable to find the first healthy nodes " +
+ "that meet the criteria. Required nodes: {}, Found nodes: {}",
nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
@@ -204,8 +245,8 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
healthyNodes, exclude,
nodeManager.getClusterNetworkTopologyMap(), anchor);
if (nodeOnDifferentRack == null) {
- LOG.error("Unable to find nodes on different racks that " +
- "meet the criteria. Required nodes: {}, Found nodes: {}",
+ LOG.error("Pipeline Placement: Unable to find nodes on different racks " +
+ " that meet the criteria. Required nodes: {}, Found nodes: {}",
nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
@@ -228,9 +269,9 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
}
if (results.size() < nodesRequired) {
- LOG.error("Unable to find the required number of healthy nodes that " +
- "meet the criteria. Required nodes: {}, Found nodes: {}",
- nodesRequired, results.size());
+ LOG.error("Pipeline Placement: Unable to find the required number of " +
+ "healthy nodes that meet the criteria. Required nodes: {}, " +
+ "Found nodes: {}", nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
index 443378c..8e0f32d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -52,8 +53,8 @@ class PipelineStateMap {
PipelineStateMap() {
// TODO: Use TreeMap for range operations?
- pipelineMap = new HashMap<>();
- pipeline2container = new HashMap<>();
+ pipelineMap = new ConcurrentHashMap<>();
+ pipeline2container = new ConcurrentHashMap<>();
query2OpenPipelines = new HashMap<>();
initializeQueryMap();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index dacc4ca..23b02ed 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -20,14 +20,12 @@ package org.apache.hadoop.hdds.scm.pipeline;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementRandom;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -38,8 +36,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -60,6 +56,9 @@ public class RatisPipelineProvider implements PipelineProvider {
private final PipelineStateManager stateManager;
private final Configuration conf;
private final EventPublisher eventPublisher;
+ private final PipelinePlacementPolicy placementPolicy;
+ private int pipelineNumberLimit;
+ private int maxPipelinePerDatanode;
// Set parallelism at 3, as now in Ratis we create 1 and 3 node pipelines.
private final int parallelismForPool = 3;
@@ -82,65 +81,93 @@ public class RatisPipelineProvider implements PipelineProvider {
this.stateManager = stateManager;
this.conf = conf;
this.eventPublisher = eventPublisher;
+ this.placementPolicy =
+ new PipelinePlacementPolicy(nodeManager, stateManager, conf);
+ this.pipelineNumberLimit = conf.getInt(
+ ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT,
+ ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT_DEFAULT);
+ this.maxPipelinePerDatanode = conf.getInt(
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT);
}
+ private List<DatanodeDetails> pickNodesNeverUsed(ReplicationFactor factor)
+ throws SCMException {
+ Set<DatanodeDetails> dnsUsed = new HashSet<>();
+ stateManager.getPipelines(ReplicationType.RATIS, factor)
+ .stream().filter(
+ p -> p.getPipelineState().equals(PipelineState.OPEN) ||
+ p.getPipelineState().equals(PipelineState.DORMANT) ||
+ p.getPipelineState().equals(PipelineState.ALLOCATED))
+ .forEach(p -> dnsUsed.addAll(p.getNodes()));
- /**
- * Create pluggable container placement policy implementation instance.
- *
- * @param nodeManager - SCM node manager.
- * @param conf - configuration.
- * @return SCM container placement policy implementation instance.
- */
- @SuppressWarnings("unchecked")
- // TODO: should we rename PlacementPolicy to PipelinePlacementPolicy?
- private static PlacementPolicy createContainerPlacementPolicy(
- final NodeManager nodeManager, final Configuration conf) {
- Class<? extends PlacementPolicy> implClass =
- (Class<? extends PlacementPolicy>) conf.getClass(
- ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementRandom.class);
+ // Get list of healthy nodes
+ List<DatanodeDetails> dns = nodeManager
+ .getNodes(HddsProtos.NodeState.HEALTHY)
+ .parallelStream()
+ .filter(dn -> !dnsUsed.contains(dn))
+ .limit(factor.getNumber())
+ .collect(Collectors.toList());
+ if (dns.size() < factor.getNumber()) {
+ String e = String
+ .format("Cannot create pipeline of factor %d using %d nodes." +
+ " Used %d nodes. Healthy nodes %d", factor.getNumber(),
+ dns.size(), dnsUsed.size(),
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY).size());
+ throw new SCMException(e,
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return dns;
+ }
- try {
- Constructor<? extends PlacementPolicy> ctor =
- implClass.getDeclaredConstructor(NodeManager.class,
- Configuration.class);
- return ctor.newInstance(nodeManager, conf);
- } catch (RuntimeException e) {
- throw e;
- } catch (InvocationTargetException e) {
- throw new RuntimeException(implClass.getName()
- + " could not be constructed.", e.getCause());
- } catch (Exception e) {
-// LOG.error("Unhandled exception occurred, Placement policy will not " +
-// "be functional.");
- throw new IllegalArgumentException("Unable to load " +
- "PlacementPolicy", e);
+ private boolean exceedPipelineNumberLimit(ReplicationFactor factor) {
+ if (factor != ReplicationFactor.THREE) {
+ // Only put limits for Factor THREE pipelines.
+ return false;
+ }
+ // Per datanode limit
+ if (maxPipelinePerDatanode > 0) {
+ return (stateManager.getPipelines(ReplicationType.RATIS, factor).size() -
+ stateManager.getPipelines(ReplicationType.RATIS, factor,
+ Pipeline.PipelineState.CLOSED).size()) > maxPipelinePerDatanode *
+ nodeManager.getNodeCount(HddsProtos.NodeState.HEALTHY) /
+ factor.getNumber();
+ }
+
+ // Global limit
+ if (pipelineNumberLimit > 0) {
+ return (stateManager.getPipelines(ReplicationType.RATIS,
+ ReplicationFactor.THREE).size() - stateManager.getPipelines(
+ ReplicationType.RATIS, ReplicationFactor.THREE,
+ Pipeline.PipelineState.CLOSED).size()) >
+ (pipelineNumberLimit - stateManager.getPipelines(
+ ReplicationType.RATIS, ReplicationFactor.ONE).size());
}
+
+ return false;
}
@Override
public Pipeline create(ReplicationFactor factor) throws IOException {
- // Get set of datanodes already used for ratis pipeline
- Set<DatanodeDetails> dnsUsed = new HashSet<>();
- stateManager.getPipelines(ReplicationType.RATIS, factor).stream().filter(
- p -> p.getPipelineState().equals(PipelineState.OPEN) ||
- p.getPipelineState().equals(PipelineState.DORMANT) ||
- p.getPipelineState().equals(PipelineState.ALLOCATED))
- .forEach(p -> dnsUsed.addAll(p.getNodes()));
+ if (exceedPipelineNumberLimit(factor)) {
+ throw new SCMException("Ratis pipeline number meets the limit: " +
+ pipelineNumberLimit + " factor : " +
+ factor.getNumber(),
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
- // Get list of healthy nodes
- List<DatanodeDetails> dns =
- nodeManager.getNodes(NodeState.HEALTHY)
- .parallelStream()
- .filter(dn -> !dnsUsed.contains(dn))
- .limit(factor.getNumber())
- .collect(Collectors.toList());
- if (dns.size() < factor.getNumber()) {
- String e = String
- .format("Cannot create pipeline of factor %d using %d nodes.",
- factor.getNumber(), dns.size());
- throw new InsufficientDatanodesException(e);
+ List<DatanodeDetails> dns;
+
+ switch(factor) {
+ case ONE:
+ dns = pickNodesNeverUsed(ReplicationFactor.ONE);
+ break;
+ case THREE:
+ dns = placementPolicy.chooseDatanodes(null,
+ null, factor.getNumber(), 0);
+ break;
+ default:
+ throw new IllegalStateException("Unknown factor: " + factor.name());
}
Pipeline pipeline = Pipeline.newBuilder()
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
new file mode 100644
index 0000000..b8cdf06
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.ratis.RatisHelper;
+import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.grpc.GrpcTlsConfig;
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.protocol.RaftGroupId;
+import org.apache.ratis.protocol.RaftPeer;
+import org.apache.ratis.retry.RetryPolicy;
+import org.apache.ratis.rpc.SupportedRpcType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Utility class for Ratis pipelines. Contains methods to create and destroy
+ * ratis pipelines.
+ */
+public final class RatisPipelineUtils {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RatisPipelineUtils.class);
+
+ private RatisPipelineUtils() {
+ }
+ /**
+ * Removes pipeline from SCM. Sends ratis command to destroy pipeline on all
+ * the datanodes.
+ *
+ * @param pipeline - Pipeline to be destroyed
+ * @param ozoneConf - Ozone configuration
+ * @param grpcTlsConfig
+ * @throws IOException
+ */
+ public static void destroyPipeline(Pipeline pipeline, Configuration ozoneConf,
+ GrpcTlsConfig grpcTlsConfig) {
+ final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
+ }
+ for (DatanodeDetails dn : pipeline.getNodes()) {
+ try {
+ destroyPipeline(dn, pipeline.getId(), ozoneConf, grpcTlsConfig);
+ } catch (IOException e) {
+ LOG.warn("Pipeline destroy failed for pipeline={} dn={} exception={}",
+ pipeline.getId(), dn, e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Sends ratis command to destroy pipeline on the given datanode.
+ *
+ * @param dn - Datanode on which pipeline needs to be destroyed
+ * @param pipelineID - ID of pipeline to be destroyed
+ * @param ozoneConf - Ozone configuration
+ * @param grpcTlsConfig - grpc tls configuration
+ * @throws IOException
+ */
+ static void destroyPipeline(DatanodeDetails dn, PipelineID pipelineID,
+ Configuration ozoneConf, GrpcTlsConfig grpcTlsConfig) throws IOException {
+ final String rpcType = ozoneConf
+ .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
+ ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
+ final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
+ final RaftPeer p = RatisHelper.toRaftPeer(dn);
+ try(RaftClient client = RatisHelper
+ .newRaftClient(SupportedRpcType.valueOfIgnoreCase(rpcType), p,
+ retryPolicy, grpcTlsConfig, ozoneConf)) {
+ client.groupRemove(RaftGroupId.valueOf(pipelineID.getId()),
+ true, p.getId());
+ }
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 32aa7b6..035d604 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -55,10 +55,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import static org.apache.hadoop.hdds.scm
- .ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm
- .ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_PIPELINE_DB;
/**
@@ -109,8 +105,8 @@ public class SCMPipelineManager implements PipelineManager {
scheduler = new Scheduler("RatisPipelineUtilsThread", false, 1);
this.backgroundPipelineCreator =
new BackgroundPipelineCreator(this, scheduler, conf);
- int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
+ int cacheSize = conf.getInt(ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB,
+ ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
final File pipelineDBPath = getPipelineDBPath(conf);
this.pipelineStore =
MetadataStoreBuilder.newBuilder()
@@ -176,10 +172,9 @@ public class SCMPipelineManager implements PipelineManager {
metrics.createPerPipelineMetrics(pipeline);
}
return pipeline;
- } catch (InsufficientDatanodesException idEx) {
- throw idEx;
} catch (IOException ex) {
metrics.incNumPipelineCreationFailed();
+ LOG.error("Pipeline creation failed.", ex);
throw ex;
} finally {
lock.writeLock().unlock();
@@ -188,7 +183,7 @@ public class SCMPipelineManager implements PipelineManager {
@Override
public Pipeline createPipeline(ReplicationType type, ReplicationFactor factor,
- List<DatanodeDetails> nodes) {
+ List<DatanodeDetails> nodes) {
// This will mostly be used to create dummy pipeline for SimplePipelines.
// We don't update the metrics for SimplePipelines.
lock.writeLock().lock();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
index 40a6f29..8c348ed 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
@@ -135,6 +135,14 @@ public final class SCMPipelineMetrics implements MetricsSource {
}
/**
+ * Get the number of pipeline created.
+ * @return number of pipeline
+ */
+ long getNumPipelineCreated() {
+ return numPipelineCreated.value();
+ }
+
+ /**
* Increments number of failed pipeline creation count.
*/
void incNumPipelineCreationFailed() {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
index 33936d5..1a03c34 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
@@ -17,11 +17,14 @@
*/
package org.apache.hadoop.hdds.scm.safemode;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -46,6 +49,8 @@ public class HealthyPipelineSafeModeRule
private int healthyPipelineThresholdCount;
private int currentHealthyPipelineCount = 0;
private final double healthyPipelinesPercent;
+ private final Set<PipelineID> processedPipelineIDs =
+ new HashSet<>();
HealthyPipelineSafeModeRule(String ruleName, EventQueue eventQueue,
PipelineManager pipelineManager,
@@ -117,8 +122,11 @@ public class HealthyPipelineSafeModeRule
Preconditions.checkNotNull(pipeline);
if (pipeline.getType() == HddsProtos.ReplicationType.RATIS &&
pipeline.getFactor() == HddsProtos.ReplicationFactor.THREE) {
- getSafeModeMetrics().incCurrentHealthyPipelinesCount();
- currentHealthyPipelineCount++;
+ if (!processedPipelineIDs.contains(pipeline.getId())) {
+ getSafeModeMetrics().incCurrentHealthyPipelinesCount();
+ currentHealthyPipelineCount++;
+ processedPipelineIDs.add(pipeline.getId());
+ }
}
if (scmInSafeMode()) {
@@ -131,6 +139,7 @@ public class HealthyPipelineSafeModeRule
@Override
protected void cleanup() {
+ processedPipelineIDs.clear();
}
@VisibleForTesting
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 594ea5c..977038e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -66,6 +66,8 @@ import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+
/**
* Test DeadNodeHandler.
*/
@@ -87,6 +89,7 @@ public class TestDeadNodeHandler {
storageDir = GenericTestUtils.getTempPath(
TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 0);
eventQueue = new EventQueue();
scm = HddsTestUtils.getScm(conf);
nodeManager = (SCMNodeManager) scm.getScmNodeManager();
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index 2e0d0b1..1e34039 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -34,11 +34,14 @@ import org.junit.Test;
import java.util.*;
import java.util.stream.Collectors;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+
/**
* Test for PipelinePlacementPolicy.
*/
public class TestPipelinePlacementPolicy {
private MockNodeManager nodeManager;
+ private OzoneConfiguration conf;
private PipelinePlacementPolicy placementPolicy;
private static final int PIPELINE_PLACEMENT_MAX_NODES_COUNT = 10;
@@ -46,8 +49,10 @@ public class TestPipelinePlacementPolicy {
public void init() throws Exception {
nodeManager = new MockNodeManager(true,
PIPELINE_PLACEMENT_MAX_NODES_COUNT);
- placementPolicy =
- new PipelinePlacementPolicy(nodeManager, new OzoneConfiguration());
+ conf = new OzoneConfiguration();
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 5);
+ placementPolicy = new PipelinePlacementPolicy(
+ nodeManager, new PipelineStateManager(conf), conf);
}
@Test
@@ -123,7 +128,7 @@ public class TestPipelinePlacementPolicy {
public void testHeavyNodeShouldBeExcluded() throws SCMException{
List<DatanodeDetails> healthyNodes =
nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
- int nodesRequired = healthyNodes.size()/2;
+ int nodesRequired = HddsProtos.ReplicationFactor.THREE.getNumber();
// only minority of healthy NODES are heavily engaged in pipelines.
int minorityHeavy = healthyNodes.size()/2 - 1;
List<DatanodeDetails> pickedNodes1 = placementPolicy.chooseDatanodes(
@@ -179,7 +184,9 @@ public class TestPipelinePlacementPolicy {
}
int considerHeavyCount =
- ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT + 1;
+ conf.getInt(
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT) + 1;
Node2PipelineMap mockMap = new Node2PipelineMap();
for (DatanodeDetails node : nodes) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index 81723e1..08f5185 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
@@ -34,12 +35,13 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.PipelineReportFromDatanode;
-import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.test.GenericTestUtils;
@@ -59,6 +61,7 @@ public class TestSCMPipelineManager {
@Before
public void setUp() throws Exception {
conf = new OzoneConfiguration();
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 1);
testDir = GenericTestUtils
.getTestDir(TestSCMPipelineManager.class.getSimpleName());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
@@ -253,10 +256,10 @@ public class TestSCMPipelineManager {
pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
Assert.fail();
- } catch (InsufficientDatanodesException idEx) {
- Assert.assertEquals(
- "Cannot create pipeline of factor 3 using 1 nodes.",
- idEx.getMessage());
+ } catch (SCMException ioe) {
+ // pipeline creation failed this time.
+ Assert.assertEquals(SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE,
+ ioe.getResult());
}
metrics = getMetrics(
@@ -266,7 +269,7 @@ public class TestSCMPipelineManager {
numPipelineCreateFailed = getLongCounter(
"NumPipelineCreationFailed", metrics);
- Assert.assertTrue(numPipelineCreateFailed == 0);
+ Assert.assertTrue(numPipelineCreateFailed == 1);
// clean up
pipelineManager.close();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
index 23d7833..acc4031 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
@@ -97,6 +97,7 @@ public class TestOzoneFsHAURLs {
conf.setTimeDuration(
OMConfigKeys.OZONE_OM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 3);
OMStorage omStore = new OMStorage(conf);
omStore.setClusterId(clusterId);
@@ -106,6 +107,8 @@ public class TestOzoneFsHAURLs {
// Start the cluster
cluster = MiniOzoneCluster.newHABuilder(conf)
+ .setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
index 21fa7bd..aba9cae 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
@@ -170,6 +170,7 @@ public class TestPipelineClose {
pipelineActionHandler
.onMessage(pipelineActionsFromDatanode, new EventQueue());
Thread.sleep(5000);
+
OzoneContainer ozoneContainer =
cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
.getContainer();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java
index 3590e43..fc90ee9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.scm.pipeline;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -36,6 +37,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
/**
@@ -51,9 +53,12 @@ public class TestRatisPipelineCreateAndDestroy {
public void init(int numDatanodes) throws Exception {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
GenericTestUtils.getRandomizedTempPath());
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(numDatanodes)
- .setHbInterval(1000)
+ .setTotalPipelineNumLimit(numDatanodes + numDatanodes/3)
+ .setHbInterval(2000)
.setHbProcessorInterval(1000)
.build();
cluster.waitForClusterToBeReady();
@@ -134,7 +139,9 @@ public class TestRatisPipelineCreateAndDestroy {
} catch (IOException ioe) {
// As now all datanodes are shutdown, they move to stale state, there
// will be no sufficient datanodes to create the pipeline.
- Assert.assertTrue(ioe instanceof InsufficientDatanodesException);
+ Assert.assertTrue(ioe instanceof SCMException);
+ Assert.assertEquals(SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE,
+ ((SCMException) ioe).getResult());
}
// make sure pipelines is destroyed
@@ -147,9 +154,14 @@ public class TestRatisPipelineCreateAndDestroy {
for (Pipeline pipeline : pipelines) {
pipelineManager.finalizeAndDestroyPipeline(pipeline, false);
}
- // make sure pipelines is created after node start
- pipelineManager.triggerPipelineCreation();
- waitForPipelines(1);
+
+ if (cluster.getStorageContainerManager()
+ .getScmNodeManager().getNodeCount(HddsProtos.NodeState.HEALTHY) >=
+ HddsProtos.ReplicationFactor.THREE.getNumber()) {
+ // make sure pipelines is created after node start
+ pipelineManager.triggerPipelineCreation();
+ waitForPipelines(1);
+ }
}
private void waitForPipelines(int numPipelines)
@@ -157,6 +169,6 @@ public class TestRatisPipelineCreateAndDestroy {
GenericTestUtils.waitFor(() -> pipelineManager
.getPipelines(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE, Pipeline.PipelineState.OPEN)
- .size() == numPipelines, 100, 40000);
+ .size() >= numPipelines, 100, 40000);
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
index 459a67a..baeee6a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
@@ -57,8 +57,11 @@ public class TestSCMRestart {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
+ int numOfNodes = 4;
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(4)
+ .setNumDatanodes(numOfNodes)
+ // allow only one FACTOR THREE pipeline.
+ .setTotalPipelineNumLimit(numOfNodes + 1)
.setHbInterval(1000)
.setHbProcessorInterval(1000)
.build();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
index 866d0b0..4b35317 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
@@ -39,6 +39,7 @@ import org.junit.rules.TemporaryFolder;
import java.util.List;
import java.util.concurrent.TimeoutException;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
import static org.junit.Assert.fail;
/**
@@ -64,6 +65,8 @@ public class TestSCMSafeModeWithPipelineRules {
true);
conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "10s");
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 50);
+
clusterBuilder = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(numDatanodes)
.setHbInterval(1000)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index de27d5a..0042363 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -251,6 +251,7 @@ public interface MiniOzoneCluster {
protected static final int DEFAULT_HB_INTERVAL_MS = 1000;
protected static final int DEFAULT_HB_PROCESSOR_INTERVAL_MS = 100;
protected static final int ACTIVE_OMS_NOT_SET = -1;
+ protected static final int DEFAULT_PIPELIME_LIMIT = 3;
protected final OzoneConfiguration conf;
protected String path;
@@ -278,6 +279,7 @@ public interface MiniOzoneCluster {
protected int numOfDatanodes = 3;
protected boolean startDataNodes = true;
protected CertificateClient certClient;
+ protected int pipelineNumLimit = DEFAULT_PIPELIME_LIMIT;
protected Builder(OzoneConfiguration conf) {
this.conf = conf;
@@ -365,6 +367,16 @@ public interface MiniOzoneCluster {
}
/**
+ * Sets the total number of pipelines to create.
+ * @param val number of pipelines
+ * @return MiniOzoneCluster.Builder
+ */
+ public Builder setTotalPipelineNumLimit(int val) {
+ pipelineNumLimit = val;
+ return this;
+ }
+
+ /**
* Sets the number of HeartBeat Interval of Datanodes, the value should be
* in MilliSeconds.
*
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index c2e196a..9bfa8bd 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -575,6 +575,10 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
streamBufferMaxSize.get(), streamBufferSizeUnit.get());
conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
streamBufferSizeUnit.get());
+ // MiniOzoneCluster should have global pipeline upper limit.
+ conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT,
+ pipelineNumLimit == DEFAULT_PIPELIME_LIMIT ?
+ 2 * numOfDatanodes : pipelineNumLimit);
configureTrace();
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java
index c8bf36b..64ded12 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java
@@ -83,6 +83,7 @@ public class Test2WayCommitInRatis {
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
.setBlockSize(blockSize)
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
index 96226d8..fa7783c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
@@ -84,6 +84,7 @@ public class TestBlockOutputStream {
StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
.setBlockSize(blockSize)
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java
index e236b85..07e306e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientMetrics;
import org.apache.hadoop.hdds.scm.XceiverClientRatis;
@@ -91,9 +92,11 @@ public class TestBlockOutputStreamWithFailures {
conf.setQuietMode(false);
conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
StorageUnit.MB);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 3);
+
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7)
- .setBlockSize(blockSize).setChunkSize(chunkSize)
- .setStreamBufferFlushSize(flushSize)
+ .setTotalPipelineNumLimit(10).setBlockSize(blockSize)
+ .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize)
.setStreamBufferMaxSize(maxFlushSize)
.setStreamBufferSizeUnit(StorageUnit.BYTES).build();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java
index 4a5f528..16f50c6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCommitWatcher.java
@@ -97,6 +97,7 @@ public class TestCommitWatcher {
StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
.setBlockSize(blockSize)
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java
index 36f720b..439287e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java
@@ -57,6 +57,7 @@ import java.util.function.Predicate;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
/**
* Tests delete key operation with a slow follower in the datanode
@@ -107,10 +108,12 @@ public class TestContainerReplicationEndToEnd {
1000, TimeUnit.SECONDS);
conf.setLong("hdds.scm.replication.thread.interval",
containerReportInterval);
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
conf.setQuietMode(false);
cluster =
- MiniOzoneCluster.newBuilder(conf).setNumDatanodes(4).setHbInterval(200)
+ MiniOzoneCluster.newBuilder(conf).setNumDatanodes(4)
+ .setTotalPipelineNumLimit(6).setHbInterval(200)
.build();
cluster.waitForClusterToBeReady();
cluster.getStorageContainerManager().getReplicationManager().start();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
index 6bef060..ba5ed9f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
@@ -53,8 +53,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.*;
/**
* Tests the containerStateMachine failure handling.
@@ -83,7 +82,7 @@ public class TestContainerStateMachine {
baseDir.mkdirs();
conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);
- // conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ // conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 200,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
index cf96a74..da2d656 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
@@ -83,6 +83,7 @@ public class TestDeleteWithSlowFollower {
private static String bucketName;
private static String path;
private static XceiverClientManager xceiverClientManager;
+ private static final int FACTOR_THREE_PIPELINE_COUNT = 1;
/**
* Create a MiniDFSCluster for testing.
@@ -120,10 +121,13 @@ public class TestDeleteWithSlowFollower {
1000, TimeUnit.SECONDS);
conf.setTimeDuration(OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
1, TimeUnit.SECONDS);
-
conf.setQuietMode(false);
- cluster =
- MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).setHbInterval(100)
+ int numOfDatanodes = 3;
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(numOfDatanodes)
+ .setTotalPipelineNumLimit(
+ numOfDatanodes + FACTOR_THREE_PIPELINE_COUNT)
+ .setHbInterval(100)
.build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
@@ -185,7 +189,7 @@ public class TestDeleteWithSlowFollower {
cluster.getStorageContainerManager().getPipelineManager()
.getPipelines(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
- Assert.assertTrue(pipelineList.size() == 1);
+ Assert.assertTrue(pipelineList.size() >= FACTOR_THREE_PIPELINE_COUNT);
Pipeline pipeline = pipelineList.get(0);
for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
if (ContainerTestHelper.isRatisFollower(dn, pipeline)) {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index 21b51e7..d4e5d7d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -98,6 +99,7 @@ public class TestFailureHandlingByClient {
1, TimeUnit.SECONDS);
conf.setBoolean(
OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, true);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
conf.setQuietMode(false);
conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
@@ -106,7 +108,7 @@ public class TestFailureHandlingByClient {
Collections.singleton(HddsUtils.getHostName(conf))).get(0),
"/rack1");
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(10).build();
+ .setNumDatanodes(10).setTotalPipelineNumLimit(15).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
client = OzoneClientFactory.getClient(conf);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java
index 47a716e..75af061 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java
@@ -67,7 +67,8 @@ public class TestHybridPipelineOnDatanode {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).build();
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3)
+ .setTotalPipelineNumLimit(5).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
client = OzoneClientFactory.getClient(conf);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
index bb7b6f0..5892083 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
@@ -82,6 +82,7 @@ public class TestKeyInputStream {
StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3)
+ .setTotalPipelineNumLimit(5)
.setBlockSize(blockSize)
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java
index 281ad4a..9b62923 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java
@@ -49,8 +49,7 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.*;
/**
* Tests MultiBlock Writes with Dn failures by Ozone Client.
@@ -88,10 +87,13 @@ public class TestMultiBlockWritesWithDnFailures {
conf.setTimeDuration(
OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
1, TimeUnit.SECONDS);
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(datanodes).build();
+ .setNumDatanodes(datanodes)
+ .setTotalPipelineNumLimit(0)
+ .build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
client = OzoneClientFactory.getClient(conf);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnException.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnException.java
index 5aefcc8..1bf2ea3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnException.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnException.java
@@ -93,6 +93,7 @@ public class TestOzoneClientRetriesOnException {
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
.setBlockSize(blockSize)
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 0027dca..72dfc41 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -165,6 +165,7 @@ public abstract class TestOzoneRpcClientAbstract {
static void startCluster(OzoneConfiguration conf) throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3)
+ .setTotalPipelineNumLimit(10)
.setScmId(scmId)
.build();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
index d2007ce..b84e61c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
@@ -58,6 +58,7 @@ import java.util.concurrent.TimeoutException;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
/**
* This class verifies the watchForCommit Handling by xceiverClient.
@@ -95,10 +96,12 @@ public class TestWatchForCommit {
conf.setTimeDuration(
OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY,
1, TimeUnit.SECONDS);
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 5);
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
.setBlockSize(blockSize)
.setChunkSize(chunkSize)
.setStreamBufferFlushSize(flushSize)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index c65ce95..8ee47a9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -53,6 +53,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.concurrent.TimeoutException;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+
/**
* Test container closing.
*/
@@ -75,8 +77,11 @@ public class TestCloseContainerByPipeline {
public static void init() throws Exception {
conf = new OzoneConfiguration();
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, "1");
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
+ .setTotalPipelineNumLimit(15)
.build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
index 7857e1f..0c875c9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
@@ -40,7 +40,7 @@ public abstract class TestDataValidate {
*/
static void startCluster(OzoneConfiguration conf) throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(5).build();
+ .setNumDatanodes(5).setTotalPipelineNumLimit(8).build();
cluster.waitForClusterToBeReady();
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
index 80ef246..5150fd4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
@@ -55,6 +55,7 @@ public class TestFreonWithPipelineDestroy {
.setHbProcessorInterval(1000)
.setHbInterval(1000)
.setNumDatanodes(3)
+ .setTotalPipelineNumLimit(8)
.build();
cluster.waitForClusterToBeReady();
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java
index 1f9d9fb..4025aca 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java
@@ -85,6 +85,7 @@ public class TestSCMContainerPlacementPolicyMetrics {
"/rack1");
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(4)
+ .setTotalPipelineNumLimit(10)
.build();
cluster.waitForClusterToBeReady();
metrics = getMetrics(SCMContainerPlacementMetrics.class.getSimpleName());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
index 841fd85..1ca3110 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.scm.node;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -78,9 +79,11 @@ public class TestQueryNode {
conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 3);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(numOfDatanodes)
+ .setTotalPipelineNumLimit(numOfDatanodes + numOfDatanodes/2)
.build();
cluster.waitForClusterToBeReady();
scmClient = new ContainerOperationClient(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 03/18: HDDS-2089: Add createPipeline CLI. (#1418)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 3d6e5b19ed001c015074b65f9b82d05172e0f07b
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Fri Sep 13 07:01:16 2019 +0800
HDDS-2089: Add createPipeline CLI. (#1418)
(cherry picked from commit 326b5acd4a63fe46821919322867f5daff30750c)
---
.../org/apache/hadoop/ozone/audit/SCMAction.java | 1 +
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 8 +--
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 71 ++++++++++++++++++++++
.../hdds/scm/cli/pipeline/PipelineCommands.java | 1 +
5 files changed, 78 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
index c3e9440..fada2d8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
@@ -31,6 +31,7 @@ public enum SCMAction implements AuditAction {
GET_CONTAINER,
GET_CONTAINER_WITH_PIPELINE,
LIST_CONTAINER,
+ CREATE_PIPELINE,
LIST_PIPELINE,
CLOSE_PIPELINE,
ACTIVATE_PIPELINE,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
index 00cb7ae..a772a97 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
@@ -48,7 +48,7 @@ public class SimplePipelineProvider implements PipelineProvider {
String e = String
.format("Cannot create pipeline of factor %d using %d nodes.",
factor.getNumber(), dns.size());
- throw new IOException(e);
+ throw new InsufficientDatanodesException(e);
}
Collections.shuffle(dns);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index b16f7a5..dad1622 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -395,10 +395,10 @@ public class SCMClientProtocolServer implements
public Pipeline createReplicationPipeline(HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor, HddsProtos.NodePool nodePool)
throws IOException {
- // TODO: will be addressed in future patch.
- // This is needed only for debugging purposes to make sure cluster is
- // working correctly.
- return null;
+ Pipeline result = scm.getPipelineManager().createPipeline(type, factor);
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.CREATE_PIPELINE, null));
+ return result;
}
@Override
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
new file mode 100644
index 0000000..edeb786
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.cli.pipeline;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.SCMCLI;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Handler of createPipeline command.
+ */
+@CommandLine.Command(
+ name = "createPipeline",
+ description = "create pipeline",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class CreatePipelineSubcommand implements Callable<Void> {
+ @CommandLine.ParentCommand
+ private SCMCLI parent;
+
+ @CommandLine.Option(
+ names = {"-t", "--replicationType"},
+ description = "Replication type (STAND_ALONE, RATIS)",
+ defaultValue = "STAND_ALONE"
+ )
+ private HddsProtos.ReplicationType type
+ = HddsProtos.ReplicationType.STAND_ALONE;
+
+ @CommandLine.Option(
+ names = {"-f", "--replicationFactor"},
+ description = "Replication factor (ONE, THREE)",
+ defaultValue = "ONE"
+ )
+ private HddsProtos.ReplicationFactor factor
+ = HddsProtos.ReplicationFactor.ONE;
+
+ @Override
+ public Void call() throws Exception {
+ if (type == HddsProtos.ReplicationType.CHAINED) {
+ throw new IllegalArgumentException(type.name()
+ + " is not supported yet.");
+ }
+ try (ScmClient scmClient = parent.createScmClient()) {
+ scmClient.createReplicationPipeline(
+ type,
+ factor,
+ HddsProtos.NodePool.getDefaultInstance());
+ return null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java
index 948a51a..0bdbc19 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/PipelineCommands.java
@@ -37,6 +37,7 @@ import java.util.concurrent.Callable;
ListPipelinesSubcommand.class,
ActivatePipelineSubcommand.class,
DeactivatePipelineSubcommand.class,
+ CreatePipelineSubcommand.class,
ClosePipelineSubcommand.class
})
public class PipelineCommands implements Callable<Void> {
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 15/18: HDDS-2772 Better management for pipeline
creation limitation. (#410)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit b63a745a50ffe9f196f454cc0914ab5079e9e853
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Fri Jan 10 10:50:38 2020 +0800
HDDS-2772 Better management for pipeline creation limitation. (#410)
---
.../common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +-
hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +-
.../org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 +-
.../test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java | 3 +++
4 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 5f52e92..980a710 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -297,7 +297,7 @@ public final class ScmConfigKeys {
public static final String OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT =
"ozone.scm.datanode.max.pipeline.engagement";
// Setting to zero by default means this limit doesn't take effect.
- public static final int OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT = 0;
+ public static final int OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT = 2;
// Upper limit for how many pipelines can be created.
// Only for test purpose now.
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 93ef0a5..31fe046 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -792,7 +792,7 @@
</property>
<property>
<name>ozone.scm.datanode.max.pipeline.engagement</name>
- <value>0</value>
+ <value>2</value>
<tag>OZONE, SCM, PIPELINE</tag>
<description>Max number of pipelines per datanode can be engaged in.
</description>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index f4a13e1..8c6e5c7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -147,7 +147,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
msg = "No healthy nodes found to allocate pipeline.";
LOG.error(msg);
throw new SCMException(msg, SCMException.ResultCodes
- .FAILED_TO_FIND_HEALTHY_NODES);
+ .FAILED_TO_FIND_SUITABLE_NODE);
}
if (initialHealthyNodesCount < nodesRequired) {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
index 077886c..ce27eed 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.commons.lang3.RandomStringUtils;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
@@ -79,6 +81,7 @@ public class TestOzoneManagerRestart {
conf.setBoolean(OZONE_ACL_ENABLED, true);
conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
+ conf.setInt(OZONE_SCM_PIPELINE_NUMBER_LIMIT, 10);
cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 14/18: HDDS-2115 Add acceptance test for
createPipeline CLI and datanode list CLI (#375)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit cb5486bfae6de2e15f9cd57f28f56ba1d05f1c9f
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Thu Jan 9 05:19:34 2020 +0800
HDDS-2115 Add acceptance test for createPipeline CLI and datanode list CLI (#375)
* HDDS-2115 Add acceptance test for createPipeline CLI and datanode list CLI.
---
.../hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java | 9 ++++++++-
.../src/main/smoketest/scmcli/{pipeline.robot => datanode.robot} | 7 ++++---
hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot | 7 ++++++-
3 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
index 58a1778..e0bdddb 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdds.scm.cli.pipeline;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import picocli.CommandLine;
import java.util.concurrent.Callable;
@@ -60,10 +61,16 @@ public class CreatePipelineSubcommand implements Callable<Void> {
+ " is not supported yet.");
}
try (ScmClient scmClient = parent.getParent().createScmClient()) {
- scmClient.createReplicationPipeline(
+ Pipeline pipeline = scmClient.createReplicationPipeline(
type,
factor,
HddsProtos.NodePool.getDefaultInstance());
+
+ if (pipeline != null) {
+ System.out.println(pipeline.getId().toString() +
+ " is created. Factor: " + pipeline.getFactor() +
+ ", Type: " + pipeline.getType());
+ }
return null;
}
}
diff --git a/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot b/hadoop-ozone/dist/src/main/smoketest/scmcli/datanode.robot
similarity index 82%
copy from hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot
copy to hadoop-ozone/dist/src/main/smoketest/scmcli/datanode.robot
index 6a6f0b0..ed1173d 100644
--- a/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/scmcli/datanode.robot
@@ -23,6 +23,7 @@ Resource ../commonlib.robot
*** Test Cases ***
-Run list pipeline
- ${output} = Execute ozone scmcli pipeline list
- Should contain ${output} Type:RATIS, Factor:ONE, State:OPEN
\ No newline at end of file
+Run list datanodes
+ ${output} = Execute ozone scmcli datanode list
+ Should contain ${output} Datanode:
+ Should contain ${output} Related pipelines:
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot b/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot
index 6a6f0b0..f411e0c 100644
--- a/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/scmcli/pipeline.robot
@@ -25,4 +25,9 @@ Resource ../commonlib.robot
*** Test Cases ***
Run list pipeline
${output} = Execute ozone scmcli pipeline list
- Should contain ${output} Type:RATIS, Factor:ONE, State:OPEN
\ No newline at end of file
+ Should contain ${output} Type:
+ Should contain ${output} Factor:ONE, State:
+
+Run create pipeline
+ ${output} = Execute ozone scmcli pipeline create
+ Should contain ${output} is created. Factor: ONE, Type: STAND_ALONE
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 16/18: HDDS-2913 Update config names and CLI for
multi-raft feature. (#462)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 8b42cdcfc1e6a880f41f79842b2db0e5bd1b456d
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Tue Jan 28 02:19:33 2020 +0800
HDDS-2913 Update config names and CLI for multi-raft feature. (#462)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 18 +++++-----
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 19 +++++++----
.../common/src/main/resources/ozone-default.xml | 9 ++---
.../hdds/scm/node/states/Node2PipelineMap.java | 5 ++-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 38 ++++++++++------------
.../hdds/scm/pipeline/RatisPipelineProvider.java | 8 ++---
.../hdds/scm/pipeline/RatisPipelineUtils.java | 16 +++------
.../hdds/scm/pipeline/SCMPipelineManager.java | 30 ++++++++++-------
.../container/TestCloseContainerEventHandler.java | 2 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 4 +--
.../TestPipelineDatanodesIntersection.java | 32 ++++++++++--------
.../scm/pipeline/TestPipelinePlacementPolicy.java | 8 ++---
.../scm/pipeline/TestRatisPipelineProvider.java | 2 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 4 +--
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 11 +++----
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 11 +++----
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 4 +--
.../safemode/TestSCMSafeModeWithPipelineRules.java | 4 +--
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 6 ++--
.../rpc/TestBlockOutputStreamWithFailures.java | 2 +-
.../rpc/TestContainerReplicationEndToEnd.java | 4 +--
.../client/rpc/TestFailureHandlingByClient.java | 2 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 2 +-
.../ozone/client/rpc/TestWatchForCommit.java | 4 +--
.../TestCloseContainerByPipeline.java | 4 +--
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 4 +--
.../hadoop/ozone/scm/node/TestQueryNode.java | 2 +-
28 files changed, 129 insertions(+), 128 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 980a710..eb2a9e5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -293,18 +293,18 @@ public final class ScmConfigKeys {
"ozone.scm.pipeline.owner.container.count";
public static final int OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT = 3;
// Pipeline placement policy:
- // the max number of pipelines can a single datanode be engaged in.
- public static final String OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT =
- "ozone.scm.datanode.max.pipeline.engagement";
- // Setting to zero by default means this limit doesn't take effect.
- public static final int OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT = 2;
+ // Upper limit for how many pipelines a datanode can engage in.
+ public static final String OZONE_DATANODE_PIPELINE_LIMIT =
+ "ozone.datanode.pipeline.limit";
+ public static final int OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT = 2;
- // Upper limit for how many pipelines can be created.
+ // Upper limit for how many pipelines can be created
+ // across the cluster nodes managed by SCM.
// Only for test purpose now.
- public static final String OZONE_SCM_PIPELINE_NUMBER_LIMIT =
- "ozone.scm.pipeline.number.limit";
+ public static final String OZONE_SCM_RATIS_PIPELINE_LIMIT =
+ "ozone.scm.ratis.pipeline.limit";
// Setting to zero by default means this limit doesn't take effect.
- public static final int OZONE_SCM_PIPELINE_NUMBER_LIMIT_DEFAULT = 0;
+ public static final int OZONE_SCM_RATIS_PIPELINE_LIMIT_DEFAULT = 0;
public static final String
OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY =
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 1dc2373..6849494 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
import java.io.IOException;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -57,7 +58,7 @@ public final class Pipeline {
// Current reported Leader for the pipeline
private UUID leaderId;
// Timestamp for pipeline upon creation
- private Long creationTimestamp;
+ private Instant creationTimestamp;
// Only valid for Ratis THREE pipeline. No need persist.
private int nodeIdsHash;
@@ -74,7 +75,7 @@ public final class Pipeline {
this.factor = factor;
this.state = state;
this.nodeStatus = nodeStatus;
- this.creationTimestamp = System.currentTimeMillis();
+ this.creationTimestamp = Instant.now();
this.nodeIdsHash = 0;
}
@@ -119,7 +120,7 @@ public final class Pipeline {
*
* @return Creation Timestamp
*/
- public Long getCreationTimestamp() {
+ public Instant getCreationTimestamp() {
return creationTimestamp;
}
@@ -128,7 +129,7 @@ public final class Pipeline {
*
* @param creationTimestamp
*/
- void setCreationTimestamp(Long creationTimestamp) {
+ void setCreationTimestamp(Instant creationTimestamp) {
this.creationTimestamp = creationTimestamp;
}
@@ -253,7 +254,7 @@ public final class Pipeline {
.setFactor(factor)
.setState(PipelineState.getProtobuf(state))
.setLeaderID(leaderId != null ? leaderId.toString() : "")
- .setCreationTimeStamp(creationTimestamp)
+ .setCreationTimeStamp(creationTimestamp.toEpochMilli())
.addAllMembers(nodeStatus.keySet().stream()
.map(DatanodeDetails::getProtoBufMessage)
.collect(Collectors.toList()));
@@ -289,6 +290,7 @@ public final class Pipeline {
.setNodes(pipeline.getMembersList().stream()
.map(DatanodeDetails::getFromProtoBuf).collect(Collectors.toList()))
.setNodesInOrder(pipeline.getMemberOrdersList())
+ .setCreateTimestamp(pipeline.getCreationTimeStamp())
.build();
}
@@ -357,7 +359,7 @@ public final class Pipeline {
private List<Integer> nodeOrder = null;
private List<DatanodeDetails> nodesInOrder = null;
private UUID leaderId = null;
- private Long creationTimestamp = null;
+ private Instant creationTimestamp = null;
private int nodeIdsHash = 0;
public Builder() {}
@@ -410,6 +412,11 @@ public final class Pipeline {
return this;
}
+ public Builder setCreateTimestamp(long createTimestamp) {
+ this.creationTimestamp = Instant.ofEpochMilli(createTimestamp);
+ return this;
+ }
+
public Builder setNodeIdsHash(int nodeIdsHash1) {
this.nodeIdsHash = nodeIdsHash1;
return this;
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 31fe046..875f79f 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -791,14 +791,14 @@
</description>
</property>
<property>
- <name>ozone.scm.datanode.max.pipeline.engagement</name>
+ <name>ozone.datanode.pipeline.limit</name>
<value>2</value>
<tag>OZONE, SCM, PIPELINE</tag>
<description>Max number of pipelines per datanode can be engaged in.
</description>
</property>
<property>
- <name>ozone.scm.pipeline.number.limit</name>
+ <name>ozone.scm.ratis.pipeline.limit</name>
<value>0</value>
<tag>OZONE, SCM, PIPELINE</tag>
<description>Upper limit for how many pipelines can be OPEN in SCM.
@@ -813,8 +813,9 @@
<description>
Timeout for every pipeline to stay in ALLOCATED stage. When pipeline is created,
it should be at OPEN stage once pipeline report is successfully received by SCM.
- If a pipeline stays at ALLOCATED for too long, it should be scrubbed so that new
- pipeline can be created. This timeout is for how long pipeline can stay at ALLOCATED
+ If a pipeline stays at ALLOCATED longer than the specified period of time,
+ it should be scrubbed so that new pipeline can be created.
+ This timeout is for how long pipeline can stay at ALLOCATED
stage until it gets scrubbed.
</description>
</property>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
index 18809ed..6533cb8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
@@ -57,8 +57,7 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
* @return Number of pipelines or 0.
*/
public int getPipelinesCount(UUID datanode) {
- Set<PipelineID> pipelines = getObjects(datanode);
- return pipelines == null ? 0 : pipelines.size();
+ return getObjects(datanode).size();
}
/**
@@ -80,7 +79,7 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
dn2ObjectMap.computeIfPresent(dnId,
(k, v) -> {
v.remove(pipeline.getId());
- return v.isEmpty() ? null : v;
+ return v;
});
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index 8c6e5c7..4261a87 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -72,8 +72,8 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
this.conf = conf;
this.stateManager = stateManager;
this.heavyNodeCriteria = conf.getInt(
- ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
- ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT);
+ ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT,
+ ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT);
}
/**
@@ -113,7 +113,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
}
boolean meet = (nodeManager.getPipelinesCount(datanodeDetails)
- pipelineNumDeductable) < heavyNodeCriteria;
- if (!meet) {
+ if (!meet && LOG.isDebugEnabled()) {
LOG.debug("Pipeline Placement: can't place more pipeline on heavy " +
"datanode： " + datanodeDetails.getUuid().toString() +
" Heaviness: " + nodeManager.getPipelinesCount(datanodeDetails) +
@@ -143,17 +143,11 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
}
int initialHealthyNodesCount = healthyNodes.size();
String msg;
- if (initialHealthyNodesCount == 0) {
- msg = "No healthy nodes found to allocate pipeline.";
- LOG.error(msg);
- throw new SCMException(msg, SCMException.ResultCodes
- .FAILED_TO_FIND_SUITABLE_NODE);
- }
if (initialHealthyNodesCount < nodesRequired) {
- LOG.warn("Not enough healthy nodes to allocate pipeline. %d "
- + " datanodes required. Found %d",
- nodesRequired, initialHealthyNodesCount);
+ LOG.warn("Not enough healthy nodes to allocate pipeline." +
+ nodesRequired + " datanodes required. Found: " +
+ initialHealthyNodesCount);
msg = String.format("Pipeline creation failed due to no sufficient" +
" healthy datanodes. Required %d. Found %d.",
nodesRequired, initialHealthyNodesCount);
@@ -168,15 +162,17 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
.collect(Collectors.toList());
if (healthyList.size() < nodesRequired) {
- LOG.debug("Unable to find enough nodes that meet " +
- "the criteria that cannot engage in more than %d pipelines." +
- " Nodes required: %d Found: %d, healthy nodes count in " +
- "NodeManager: %d.",
- heavyNodeCriteria, nodesRequired, healthyList.size(),
- initialHealthyNodesCount);
- msg = String.format("Pipeline creation failed due to not enough" +
- " healthy datanodes after filter. Required %d. Found %d",
- nodesRequired, initialHealthyNodesCount);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to find enough nodes that meet the criteria that" +
+ " cannot engage in more than" + heavyNodeCriteria +
+ " pipelines. Nodes required: " + nodesRequired + " Found:" +
+ healthyList.size() + " healthy nodes count in NodeManager: " +
+ initialHealthyNodesCount);
+ }
+ msg = String.format("Pipeline creation failed because nodes are engaged" +
+ " in other pipelines and every node can only be engaged in" +
+ " max %d pipelines. Required %d. Found %d",
+ heavyNodeCriteria, nodesRequired, healthyList.size());
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 9585907..4865074 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -84,11 +84,11 @@ public class RatisPipelineProvider implements PipelineProvider {
this.placementPolicy =
new PipelinePlacementPolicy(nodeManager, stateManager, conf);
this.pipelineNumberLimit = conf.getInt(
- ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT,
- ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT_DEFAULT);
+ ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT,
+ ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT_DEFAULT);
this.maxPipelinePerDatanode = conf.getInt(
- ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
- ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT);
+ ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT,
+ ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT);
}
private List<DatanodeDetails> pickNodesNeverUsed(ReplicationFactor factor)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
index f9f2011..7fe1cc1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -106,27 +106,21 @@ public final class RatisPipelineUtils {
}
/**
- * Return first existed pipeline which share the same set of datanodes
+ * Return the list of pipelines who share the same set of datanodes
* with the input pipeline.
* @param stateManager PipelineStateManager
* @param pipeline input pipeline
* @return first matched pipeline
*/
- static Pipeline checkPipelineContainSameDatanodes(
+ static List<Pipeline> checkPipelineContainSameDatanodes(
PipelineStateManager stateManager, Pipeline pipeline) {
- List<Pipeline> matchedPipelines = stateManager.getPipelines(
+ return stateManager.getPipelines(
HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE)
.stream().filter(p -> !p.getId().equals(pipeline.getId()) &&
(// For all OPEN or ALLOCATED pipelines
- p.getPipelineState() == Pipeline.PipelineState.OPEN ||
- p.getPipelineState() == Pipeline.PipelineState.ALLOCATED) &&
- p.getNodeIdsHash() == pipeline.getNodeIdsHash())
+ p.getPipelineState() != Pipeline.PipelineState.CLOSED &&
+ p.getNodeIdsHash() == pipeline.getNodeIdsHash()))
.collect(Collectors.toList());
- if (matchedPipelines.size() == 0) {
- return null;
- } else {
- return matchedPipelines.stream().findFirst().get();
- }
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index f924b41..88c4329 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -45,6 +45,8 @@ import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
import java.io.File;
import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -172,17 +174,20 @@ public class SCMPipelineManager implements PipelineManager {
metrics.incNumPipelineCreated();
metrics.createPerPipelineMetrics(pipeline);
}
- Pipeline overlapPipeline = RatisPipelineUtils
+ List<Pipeline> overlapPipelines = RatisPipelineUtils
.checkPipelineContainSameDatanodes(stateManager, pipeline);
- if (overlapPipeline != null) {
+ if (!overlapPipelines.isEmpty()) {
+ // Count 1 overlap at a time.
metrics.incNumPipelineContainSameDatanodes();
//TODO remove until pipeline allocation is proved equally distributed.
- LOG.info("Pipeline: " + pipeline.getId().toString() +
- " contains same datanodes as previous pipeline: " +
- overlapPipeline.getId().toString() + " nodeIds: " +
- pipeline.getNodes().get(0).getUuid().toString() +
- ", " + pipeline.getNodes().get(1).getUuid().toString() +
- ", " + pipeline.getNodes().get(2).getUuid().toString());
+ for (Pipeline overlapPipeline : overlapPipelines) {
+ LOG.info("Pipeline: " + pipeline.getId().toString() +
+ " contains same datanodes as previous pipelines: " +
+ overlapPipeline.getId().toString() + " nodeIds: " +
+ pipeline.getNodes().get(0).getUuid().toString() +
+ ", " + pipeline.getNodes().get(1).getUuid().toString() +
+ ", " + pipeline.getNodes().get(2).getUuid().toString());
+ }
}
return pipeline;
} catch (IOException ex) {
@@ -381,20 +386,21 @@ public class SCMPipelineManager implements PipelineManager {
// Only srub pipeline for RATIS THREE pipeline
return;
}
- Long currentTime = System.currentTimeMillis();
+ Instant currentTime = Instant.now();
Long pipelineScrubTimeoutInMills = conf.getTimeDuration(
ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT,
ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
List<Pipeline> needToSrubPipelines = stateManager.getPipelines(type, factor,
Pipeline.PipelineState.ALLOCATED).stream()
- .filter(p -> (currentTime - p.getCreationTimestamp()
- >= pipelineScrubTimeoutInMills))
+ .filter(p -> currentTime.toEpochMilli() - p.getCreationTimestamp()
+ .toEpochMilli() >= pipelineScrubTimeoutInMills)
.collect(Collectors.toList());
for (Pipeline p : needToSrubPipelines) {
LOG.info("srubbing pipeline: id: " + p.getId().toString() +
" since it stays at ALLOCATED stage for " +
- (currentTime - p.getCreationTimestamp())/60000 + " mins.");
+ Duration.between(currentTime, p.getCreationTimestamp()).toMinutes() +
+ " mins.");
finalizeAndDestroyPipeline(p, false);
}
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
index f35bfe2..10c38a8 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
@@ -68,7 +68,7 @@ public class TestCloseContainerEventHandler {
.getTestDir(TestCloseContainerEventHandler.class.getSimpleName());
configuration
.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
- configuration.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT, 16);
+ configuration.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 16);
nodeManager = new MockNodeManager(true, 10);
eventQueue = new EventQueue();
pipelineManager =
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 977038e..4cdc46f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -66,7 +66,7 @@ import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
/**
* Test DeadNodeHandler.
@@ -89,7 +89,7 @@ public class TestDeadNodeHandler {
storageDir = GenericTestUtils.getTempPath(
TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 0);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 0);
eventQueue = new EventQueue();
scm = HddsTestUtils.getScm(conf);
nodeManager = (SCMNodeManager) scm.getScmNodeManager();
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
index 87e8cf4..41eea3d 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
@@ -34,8 +34,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.List;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE;
/**
@@ -77,7 +78,7 @@ public class TestPipelineDatanodesIntersection {
@Test
public void testPipelineDatanodesIntersection() {
NodeManager nodeManager= new MockNodeManager(true, nodeCount);
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, nodeHeaviness);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, nodeHeaviness);
conf.setBoolean(OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, false);
PipelineStateManager stateManager = new PipelineStateManager();
PipelineProvider provider = new MockRatisPipelineProvider(nodeManager,
@@ -92,20 +93,23 @@ public class TestPipelineDatanodesIntersection {
Pipeline pipeline = provider.create(HddsProtos.ReplicationFactor.THREE);
stateManager.addPipeline(pipeline);
nodeManager.addPipeline(pipeline);
- Pipeline overlapPipeline = RatisPipelineUtils
+ List<Pipeline> overlapPipelines = RatisPipelineUtils
.checkPipelineContainSameDatanodes(stateManager, pipeline);
- if (overlapPipeline != null){
+
+ if (overlapPipelines.isEmpty()){
intersectionCount++;
- LOG.info("This pipeline: " + pipeline.getId().toString() +
- " overlaps with previous pipeline: " + overlapPipeline.getId() +
- ". They share same set of datanodes as: " +
- pipeline.getNodesInOrder().get(0).getUuid() + "/" +
- pipeline.getNodesInOrder().get(1).getUuid() + "/" +
- pipeline.getNodesInOrder().get(2).getUuid() + " and " +
- overlapPipeline.getNodesInOrder().get(0).getUuid() + "/" +
- overlapPipeline.getNodesInOrder().get(1).getUuid() + "/" +
- overlapPipeline.getNodesInOrder().get(2).getUuid() +
- " is the same.");
+ for (Pipeline overlapPipeline : overlapPipelines) {
+ LOG.info("This pipeline: " + pipeline.getId().toString() +
+ " overlaps with previous pipeline: " + overlapPipeline.getId() +
+ ". They share same set of datanodes as: " +
+ pipeline.getNodesInOrder().get(0).getUuid() + "/" +
+ pipeline.getNodesInOrder().get(1).getUuid() + "/" +
+ pipeline.getNodesInOrder().get(2).getUuid() + " and " +
+ overlapPipeline.getNodesInOrder().get(0).getUuid() + "/" +
+ overlapPipeline.getNodesInOrder().get(1).getUuid() + "/" +
+ overlapPipeline.getNodesInOrder().get(2).getUuid() +
+ " is the same.");
+ }
}
createdPipelineCount++;
} catch(SCMException e) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index 2fe67f9..2fff7d9 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -34,7 +34,7 @@ import org.junit.Test;
import java.util.*;
import java.util.stream.Collectors;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
/**
* Test for PipelinePlacementPolicy.
@@ -50,7 +50,7 @@ public class TestPipelinePlacementPolicy {
nodeManager = new MockNodeManager(true,
PIPELINE_PLACEMENT_MAX_NODES_COUNT);
conf = new OzoneConfiguration();
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 5);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 5);
placementPolicy = new PipelinePlacementPolicy(
nodeManager, new PipelineStateManager(), conf);
}
@@ -185,8 +185,8 @@ public class TestPipelinePlacementPolicy {
int considerHeavyCount =
conf.getInt(
- ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
- ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT) + 1;
+ ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT,
+ ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT) + 1;
Node2PipelineMap mockMap = new Node2PipelineMap();
for (DatanodeDetails node : nodes) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index 5623359..a17fc08 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -61,7 +61,7 @@ public class TestRatisPipelineProvider {
public void init() throws Exception {
nodeManager = new MockNodeManager(true, 10);
conf = new OzoneConfiguration();
- conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT,
maxPipelinePerNode);
stateManager = new PipelineStateManager();
provider = new MockRatisPipelineProvider(nodeManager,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index e6bf7a0..deba91b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
import static org.apache.commons.collections.CollectionUtils.intersection;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
@@ -65,7 +65,7 @@ public class TestSCMPipelineManager {
@Before
public void setUp() throws Exception {
conf = new OzoneConfiguration();
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 1);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 1);
testDir = GenericTestUtils
.getTestDir(TestSCMPipelineManager.class.getSimpleName());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
index dcd8402..badfadc 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.cli.datanode;
+import com.google.common.base.Strings;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -62,15 +63,15 @@ public class ListInfoSubcommand implements Callable<Void> {
public Void call() throws Exception {
try (ScmClient scmClient = parent.getParent().createScmClient()) {
pipelines = scmClient.listPipelines();
- if (isNullOrEmpty(ipaddress) && isNullOrEmpty(uuid)) {
+ if (Strings.isNullOrEmpty(ipaddress) && Strings.isNullOrEmpty(uuid)) {
getAllNodes(scmClient).stream().forEach(p -> printDatanodeInfo(p));
} else {
Stream<DatanodeDetails> allNodes = getAllNodes(scmClient).stream();
- if (!isNullOrEmpty(ipaddress)) {
+ if (!Strings.isNullOrEmpty(ipaddress)) {
allNodes = allNodes.filter(p -> p.getIpAddress()
.compareToIgnoreCase(ipaddress) == 0);
}
- if (!isNullOrEmpty(uuid)) {
+ if (!Strings.isNullOrEmpty(uuid)) {
allNodes = allNodes.filter(p -> p.getUuid().toString().equals(uuid));
}
allNodes.forEach(p -> printDatanodeInfo(p));
@@ -117,8 +118,4 @@ public class ListInfoSubcommand implements Callable<Void> {
+ datanode.getHostName() + "/" + relatedPipelineNum +
" pipelines) \n" + "Related pipelines: \n" + pipelineListInfo);
}
-
- protected static boolean isNullOrEmpty(String str) {
- return ((str == null) || str.trim().isEmpty());
- }
}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java
index 8b3b1b3..f8ac1d4 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ListPipelinesSubcommand.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.cli.pipeline;
+import com.google.common.base.Strings;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import picocli.CommandLine;
@@ -53,13 +54,13 @@ public class ListPipelinesSubcommand implements Callable<Void> {
@Override
public Void call() throws Exception {
try (ScmClient scmClient = parent.getParent().createScmClient()) {
- if (isNullOrEmpty(factor) && isNullOrEmpty(state)) {
+ if (Strings.isNullOrEmpty(factor) && Strings.isNullOrEmpty(state)) {
scmClient.listPipelines().forEach(System.out::println);
} else {
scmClient.listPipelines().stream()
- .filter(p -> ((isNullOrEmpty(factor) ||
+ .filter(p -> ((Strings.isNullOrEmpty(factor) ||
(p.getFactor().toString().compareToIgnoreCase(factor) == 0))
- && (isNullOrEmpty(state) ||
+ && (Strings.isNullOrEmpty(state) ||
(p.getPipelineState().toString().compareToIgnoreCase(state)
== 0))))
.forEach(System.out::println);
@@ -67,8 +68,4 @@ public class ListPipelinesSubcommand implements Callable<Void> {
return null;
}
}
-
- protected static boolean isNullOrEmpty(String str) {
- return ((str == null) || str.trim().isEmpty());
- }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
index acc4031..7a6143c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
@@ -97,7 +97,7 @@ public class TestOzoneFsHAURLs {
conf.setTimeDuration(
OMConfigKeys.OZONE_OM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS);
- conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 3);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 3);
OMStorage omStore = new OMStorage(conf);
omStore.setClusterId(clusterId);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java
index fc90ee9..bd677db 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java
@@ -37,7 +37,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
/**
@@ -53,7 +53,7 @@ public class TestRatisPipelineCreateAndDestroy {
public void init(int numDatanodes) throws Exception {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
GenericTestUtils.getRandomizedTempPath());
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 2);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(numDatanodes)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
index 4b35317..39b67ac 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
@@ -39,7 +39,7 @@ import org.junit.rules.TemporaryFolder;
import java.util.List;
import java.util.concurrent.TimeoutException;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
import static org.junit.Assert.fail;
/**
@@ -65,7 +65,7 @@ public class TestSCMSafeModeWithPipelineRules {
true);
conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "10s");
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 50);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 50);
clusterBuilder = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(numDatanodes)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 9bfa8bd..7758c3c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -576,9 +576,9 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
streamBufferSizeUnit.get());
// MiniOzoneCluster should have global pipeline upper limit.
- conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT,
- pipelineNumLimit == DEFAULT_PIPELIME_LIMIT ?
- 2 * numOfDatanodes : pipelineNumLimit);
+ conf.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT,
+ pipelineNumLimit >= DEFAULT_PIPELIME_LIMIT ?
+ pipelineNumLimit : DEFAULT_PIPELIME_LIMIT);
configureTrace();
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java
index 07e306e..1b6b7dc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java
@@ -92,7 +92,7 @@ public class TestBlockOutputStreamWithFailures {
conf.setQuietMode(false);
conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
StorageUnit.MB);
- conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 3);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 3);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7)
.setTotalPipelineNumLimit(10).setBlockSize(blockSize)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java
index 439287e..6917ab2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java
@@ -57,7 +57,7 @@ import java.util.function.Predicate;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
/**
* Tests delete key operation with a slow follower in the datanode
@@ -108,7 +108,7 @@ public class TestContainerReplicationEndToEnd {
1000, TimeUnit.SECONDS);
conf.setLong("hdds.scm.replication.thread.interval",
containerReportInterval);
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 2);
conf.setQuietMode(false);
cluster =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index d4e5d7d..a84e16e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -99,7 +99,7 @@ public class TestFailureHandlingByClient {
1, TimeUnit.SECONDS);
conf.setBoolean(
OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, true);
- conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 2);
conf.setQuietMode(false);
conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java
index 9b62923..7d31499 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java
@@ -87,7 +87,7 @@ public class TestMultiBlockWritesWithDnFailures {
conf.setTimeDuration(
OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
1, TimeUnit.SECONDS);
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 2);
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
index b84e61c..95dcedc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
@@ -58,7 +58,7 @@ import java.util.concurrent.TimeoutException;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
/**
* This class verifies the watchForCommit Handling by xceiverClient.
@@ -96,7 +96,7 @@ public class TestWatchForCommit {
conf.setTimeDuration(
OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY,
1, TimeUnit.SECONDS);
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 5);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 5);
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index 8ee47a9..869f091 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -53,7 +53,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.concurrent.TimeoutException;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
/**
* Test container closing.
@@ -77,7 +77,7 @@ public class TestCloseContainerByPipeline {
public static void init() throws Exception {
conf = new OzoneConfiguration();
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, "1");
- conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 2);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
index ce27eed..6058fad 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.commons.lang3.RandomStringUtils;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
@@ -81,7 +81,7 @@ public class TestOzoneManagerRestart {
conf.setBoolean(OZONE_ACL_ENABLED, true);
conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
- conf.setInt(OZONE_SCM_PIPELINE_NUMBER_LIMIT, 10);
+ conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
index 1ca3110..14660d6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java
@@ -79,7 +79,7 @@ public class TestQueryNode {
conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
- conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 3);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 3);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(numOfDatanodes)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 12/18: Resolve rebase conflict.
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 4d463d6964535406a69da60e171bc2952c4ae172
Author: Li Cheng <ti...@tencent.com>
AuthorDate: Mon Dec 23 15:52:37 2019 +0800
Resolve rebase conflict.
---
.../hdds/scm/pipeline/TestPipelineDatanodesIntersection.java | 2 +-
.../hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java | 2 +-
.../hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java | 11 +++++------
3 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
index 45f85ef..87e8cf4 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
@@ -79,7 +79,7 @@ public class TestPipelineDatanodesIntersection {
NodeManager nodeManager= new MockNodeManager(true, nodeCount);
conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, nodeHeaviness);
conf.setBoolean(OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, false);
- PipelineStateManager stateManager = new PipelineStateManager(conf);
+ PipelineStateManager stateManager = new PipelineStateManager();
PipelineProvider provider = new MockRatisPipelineProvider(nodeManager,
stateManager, conf);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index 1e34039..2fe67f9 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -52,7 +52,7 @@ public class TestPipelinePlacementPolicy {
conf = new OzoneConfiguration();
conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 5);
placementPolicy = new PipelinePlacementPolicy(
- nodeManager, new PipelineStateManager(conf), conf);
+ nodeManager, new PipelineStateManager(), conf);
}
@Test
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index 46fd8c8..5623359 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -39,8 +39,6 @@ import java.util.UUID;
import java.util.stream.Collectors;
import static org.apache.commons.collections.CollectionUtils.intersection;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
@@ -57,12 +55,14 @@ public class TestRatisPipelineProvider {
private PipelineProvider provider;
private PipelineStateManager stateManager;
private OzoneConfiguration conf;
+ private int maxPipelinePerNode = 2;
@Before
public void init() throws Exception {
nodeManager = new MockNodeManager(true, 10);
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
+ conf = new OzoneConfiguration();
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
+ maxPipelinePerNode);
stateManager = new PipelineStateManager();
provider = new MockRatisPipelineProvider(nodeManager,
stateManager, conf);
@@ -194,8 +194,7 @@ public class TestRatisPipelineProvider {
// Use up first 3 DNs for an open pipeline.
List<DatanodeDetails> dns = healthyNodes.subList(0, 3);
- for (int i = 0; i < conf.getInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
- OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT); i++) {
+ for (int i = 0; i < maxPipelinePerNode; i++) {
// Saturate pipeline counts on all the 1st 3 DNs.
addPipeline(dns, factor, Pipeline.PipelineState.OPEN, REPLICATION_TYPE);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 08/18: HDDS-2035 Implement datanode level CLI to
reveal pipeline relation. (#348)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 81ed72784861fbcff833ead124f1af034788c39e
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Wed Dec 18 19:05:52 2019 +0800
HDDS-2035 Implement datanode level CLI to reveal pipeline relation. (#348)
---
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 2 +
.../hdds/scm/cli/datanode/DatanodeCommands.java | 52 +++++++++
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 124 +++++++++++++++++++++
.../hadoop/hdds/scm/cli/datanode/package-info.java | 22 ++++
4 files changed, 200 insertions(+)
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
index 8c0fb03..20a35a8 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands;
+import org.apache.hadoop.hdds.scm.cli.datanode.DatanodeCommands;
import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands;
import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
@@ -59,6 +60,7 @@ import picocli.CommandLine.Option;
SafeModeCommands.class,
ContainerCommands.class,
PipelineCommands.class,
+ DatanodeCommands.class,
TopologySubcommand.class,
ReplicationManagerCommands.class
},
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
new file mode 100644
index 0000000..94763d3
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.cli.MissingSubcommandException;
+import org.apache.hadoop.hdds.scm.cli.SCMCLI;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Subcommand for datanode related operations.
+ */
+@CommandLine.Command(
+ name = "datanode",
+ description = "Datanode specific operations",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class,
+ subcommands = {
+ ListInfoSubcommand.class
+ })
+public class DatanodeCommands implements Callable<Void> {
+
+ @CommandLine.ParentCommand
+ private SCMCLI parent;
+
+ public SCMCLI getParent() {
+ return parent;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ throw new MissingSubcommandException(
+ this.parent.getCmd().getSubcommands().get("datanode"));
+ }
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
new file mode 100644
index 0000000..dcd8402
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Handler of list datanodes info command.
+ */
+@CommandLine.Command(
+ name = "list",
+ description = "List info of datanodes",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class ListInfoSubcommand implements Callable<Void> {
+
+ @CommandLine.ParentCommand
+ private DatanodeCommands parent;
+
+ @CommandLine.Option(names = {"--ip"},
+ description = "Show info by ip address.",
+ defaultValue = "",
+ required = false)
+ private String ipaddress;
+
+ @CommandLine.Option(names = {"--id"},
+ description = "Show info by datanode UUID.",
+ defaultValue = "",
+ required = false)
+ private String uuid;
+
+ private List<Pipeline> pipelines;
+
+
+ @Override
+ public Void call() throws Exception {
+ try (ScmClient scmClient = parent.getParent().createScmClient()) {
+ pipelines = scmClient.listPipelines();
+ if (isNullOrEmpty(ipaddress) && isNullOrEmpty(uuid)) {
+ getAllNodes(scmClient).stream().forEach(p -> printDatanodeInfo(p));
+ } else {
+ Stream<DatanodeDetails> allNodes = getAllNodes(scmClient).stream();
+ if (!isNullOrEmpty(ipaddress)) {
+ allNodes = allNodes.filter(p -> p.getIpAddress()
+ .compareToIgnoreCase(ipaddress) == 0);
+ }
+ if (!isNullOrEmpty(uuid)) {
+ allNodes = allNodes.filter(p -> p.getUuid().toString().equals(uuid));
+ }
+ allNodes.forEach(p -> printDatanodeInfo(p));
+ }
+ return null;
+ }
+ }
+
+ private List<DatanodeDetails> getAllNodes(ScmClient scmClient)
+ throws IOException {
+ List<HddsProtos.Node> nodes = scmClient.queryNode(
+ HddsProtos.NodeState.HEALTHY, HddsProtos.QueryScope.CLUSTER, "");
+
+ return nodes.stream()
+ .map(p -> DatanodeDetails.getFromProtoBuf(p.getNodeID()))
+ .collect(Collectors.toList());
+ }
+
+ private void printDatanodeInfo(DatanodeDetails datanode) {
+ StringBuilder pipelineListInfo = new StringBuilder();
+ int relatedPipelineNum = 0;
+ if (!pipelines.isEmpty()) {
+ List<Pipeline> relatedPipelines = pipelines.stream().filter(
+ p -> p.getNodes().contains(datanode)).collect(Collectors.toList());
+ if (relatedPipelines.isEmpty()) {
+ pipelineListInfo.append("No related pipelines" +
+ " or the node is not in Healthy state.");
+ } else {
+ relatedPipelineNum = relatedPipelines.size();
+ relatedPipelines.stream().forEach(
+ p -> pipelineListInfo.append(p.getId().getId().toString())
+ .append("/").append(p.getFactor().toString()).append("/")
+ .append(p.getType().toString()).append("/")
+ .append(p.getPipelineState().toString()).append("/")
+ .append(datanode.getUuid().equals(p.getLeaderId()) ?
+ "Leader" : "Follower")
+ .append(System.getProperty("line.separator")));
+ }
+ } else {
+ pipelineListInfo.append("No pipelines in cluster.");
+ }
+ System.out.println("Datanode: " + datanode.getUuid().toString() +
+ " (" + datanode.getIpAddress() + "/"
+ + datanode.getHostName() + "/" + relatedPipelineNum +
+ " pipelines) \n" + "Related pipelines: \n" + pipelineListInfo);
+ }
+
+ protected static boolean isNullOrEmpty(String str) {
+ return ((str == null) || str.trim().isEmpty());
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/package-info.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/package-info.java
new file mode 100644
index 0000000..f4c45cf
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains all of the datanode related scm commands.
+ */
+package org.apache.hadoop.hdds.scm.cli.datanode;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 05/18: HDDS-1572 Implement a Pipeline scrubber to
clean up non-OPEN pipeline. (#237)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 4cbf554a7b3ffb4b241858b340ec75ba337146a1
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Wed Nov 27 20:18:50 2019 +0800
HDDS-1572 Implement a Pipeline scrubber to clean up non-OPEN pipeline. (#237)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 7 ++++
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 31 +++++++++++++++-
hadoop-hdds/common/src/main/proto/hdds.proto | 1 +
.../common/src/main/resources/ozone-default.xml | 12 ++++++
.../scm/pipeline/BackgroundPipelineCreator.java | 8 +++-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 3 ++
.../hdds/scm/pipeline/SCMPipelineManager.java | 26 +++++++++++++
.../scm/pipeline/MockRatisPipelineProvider.java | 28 ++++++++++++++
.../scm/pipeline/TestRatisPipelineProvider.java | 10 ++++-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 43 +++++++++++++++++++++-
10 files changed, 165 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 15a47f3..5f52e92 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -310,6 +310,13 @@ public final class ScmConfigKeys {
OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY =
"ozone.scm.keyvalue.container.deletion-choosing.policy";
+ // Max timeout for pipeline to stay at ALLOCATED state before scrubbed.
+ public static final String OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT =
+ "ozone.scm.pipeline.allocated.timeout";
+
+ public static final String OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT_DEFAULT =
+ "5m";
+
public static final String OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT =
"ozone.scm.container.creation.lease.timeout";
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 594fcf7..295156d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -56,6 +56,8 @@ public final class Pipeline {
private ThreadLocal<List<DatanodeDetails>> nodesInOrder = new ThreadLocal<>();
// Current reported Leader for the pipeline
private UUID leaderId;
+ // Timestamp for pipeline upon creation
+ private Long creationTimestamp;
/**
* The immutable properties of pipeline object is used in
@@ -70,6 +72,7 @@ public final class Pipeline {
this.factor = factor;
this.state = state;
this.nodeStatus = nodeStatus;
+ this.creationTimestamp = System.currentTimeMillis();
}
/**
@@ -109,6 +112,24 @@ public final class Pipeline {
}
/**
+ * Return the creation time of pipeline.
+ *
+ * @return Creation Timestamp
+ */
+ public Long getCreationTimestamp() {
+ return creationTimestamp;
+ }
+
+ /**
+ * Set the creation timestamp. Only for protobuf now.
+ *
+ * @param creationTimestamp
+ */
+ void setCreationTimestamp(Long creationTimestamp) {
+ this.creationTimestamp = creationTimestamp;
+ }
+
+ /**
* Return the pipeline leader's UUID.
*
* @return DatanodeDetails.UUID.
@@ -221,6 +242,7 @@ public final class Pipeline {
.setFactor(factor)
.setState(PipelineState.getProtobuf(state))
.setLeaderID(leaderId != null ? leaderId.toString() : "")
+ .setCreationTimeStamp(creationTimestamp)
.addAllMembers(nodeStatus.keySet().stream()
.map(DatanodeDetails::getProtoBufMessage)
.collect(Collectors.toList()));
@@ -299,7 +321,8 @@ public final class Pipeline {
b.append(", Factor:").append(getFactor());
b.append(", State:").append(getPipelineState());
b.append(", leaderId:").append(getLeaderId());
- b.append(" ]");
+ b.append(", CreationTimestamp").append(getCreationTimestamp());
+ b.append("]");
return b.toString();
}
@@ -323,6 +346,7 @@ public final class Pipeline {
private List<Integer> nodeOrder = null;
private List<DatanodeDetails> nodesInOrder = null;
private UUID leaderId = null;
+ private Long creationTimestamp = null;
public Builder() {}
@@ -334,6 +358,7 @@ public final class Pipeline {
this.nodeStatus = pipeline.nodeStatus;
this.nodesInOrder = pipeline.nodesInOrder.get();
this.leaderId = pipeline.getLeaderId();
+ this.creationTimestamp = pipeline.getCreationTimestamp();
}
public Builder setId(PipelineID id1) {
@@ -380,6 +405,10 @@ public final class Pipeline {
Preconditions.checkNotNull(nodeStatus);
Pipeline pipeline = new Pipeline(id, type, factor, state, nodeStatus);
pipeline.setLeaderId(leaderId);
+ // overwrite with original creationTimestamp
+ if (creationTimestamp != null) {
+ pipeline.setCreationTimestamp(creationTimestamp);
+ }
if (nodeOrder != null && !nodeOrder.isEmpty()) {
// This branch is for build from ProtoBuf
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 8da3576..c78175c 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -75,6 +75,7 @@ message Pipeline {
required PipelineID id = 5;
optional string leaderID = 6;
repeated uint32 memberOrders = 7;
+ optional uint64 creationTimeStamp = 8;
}
message KeyValue {
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 8bc2ea1..93ef0a5 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -807,6 +807,18 @@
</description>
</property>
<property>
+ <name>ozone.scm.pipeline.allocated.timeout</name>
+ <value>5m</value>
+ <tag>OZONE, SCM, PIPELINE</tag>
+ <description>
+ Timeout for every pipeline to stay in ALLOCATED stage. When pipeline is created,
+ it should be at OPEN stage once pipeline report is successfully received by SCM.
+ If a pipeline stays at ALLOCATED for too long, it should be scrubbed so that new
+ pipeline can be created. This timeout is for how long pipeline can stay at ALLOCATED
+ stage until it gets scrubbed.
+ </description>
+ </property>
+ <property>
<name>ozone.scm.container.size</name>
<value>5GB</value>
<tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index b663f2a..8e4ec6a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -110,12 +110,18 @@ class BackgroundPipelineCreator {
// Skip this iteration for creating pipeline
continue;
}
+
+ try {
+ pipelineManager.scrubPipeline(type, factor);
+ } catch (IOException e) {
+ LOG.error("Error while scrubbing pipelines {}", e);
+ }
+
while (true) {
try {
if (scheduler.isClosed()) {
break;
}
-
pipelineManager.createPipeline(type, factor);
} catch (IOException ioe) {
break;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
index 0855295..635e032 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
@@ -77,6 +77,9 @@ public interface PipelineManager extends Closeable, PipelineManagerMXBean {
void finalizeAndDestroyPipeline(Pipeline pipeline, boolean onTimeout)
throws IOException;
+ void scrubPipeline(ReplicationType type, ReplicationFactor factor)
+ throws IOException;
+
void startPipelineCreator();
void triggerPipelineCreation();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 035d604..01af465 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -54,6 +54,7 @@ import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_PIPELINE_DB;
@@ -363,6 +364,31 @@ public class SCMPipelineManager implements PipelineManager {
}
@Override
+ public void scrubPipeline(ReplicationType type, ReplicationFactor factor)
+ throws IOException{
+ if (type != ReplicationType.RATIS || factor != ReplicationFactor.THREE) {
+ // Only srub pipeline for RATIS THREE pipeline
+ return;
+ }
+ Long currentTime = System.currentTimeMillis();
+ Long pipelineScrubTimeoutInMills = conf.getTimeDuration(
+ ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT,
+ ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ List<Pipeline> needToSrubPipelines = stateManager.getPipelines(type, factor,
+ Pipeline.PipelineState.ALLOCATED).stream()
+ .filter(p -> (currentTime - p.getCreationTimestamp()
+ >= pipelineScrubTimeoutInMills))
+ .collect(Collectors.toList());
+ for (Pipeline p : needToSrubPipelines) {
+ LOG.info("srubbing pipeline: id: " + p.getId().toString() +
+ " since it stays at ALLOCATED stage for " +
+ (currentTime - p.getCreationTimestamp())/60000 + " mins.");
+ finalizeAndDestroyPipeline(p, false);
+ }
+ }
+
+ @Override
public Map<String, Integer> getPipelineInfo() {
final Map<String, Integer> pipelineInfo = new HashMap<>();
for (Pipeline.PipelineState state : Pipeline.PipelineState.values()) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
index 25b0adc..7513cad 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
@@ -33,6 +33,15 @@ import java.util.List;
*/
public class MockRatisPipelineProvider extends RatisPipelineProvider {
+ private boolean autoOpenPipeline;
+
+ public MockRatisPipelineProvider(NodeManager nodeManager,
+ PipelineStateManager stateManager,
+ Configuration conf, boolean autoOpen) {
+ super(nodeManager, stateManager, conf, null);
+ autoOpenPipeline = autoOpen;
+ }
+
public MockRatisPipelineProvider(NodeManager nodeManager,
PipelineStateManager stateManager,
Configuration conf) {
@@ -43,6 +52,7 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
PipelineStateManager stateManager, Configuration conf,
EventPublisher eventPublisher) {
super(nodeManager, stateManager, conf, eventPublisher);
+ autoOpenPipeline = true;
}
protected void initializePipeline(Pipeline pipeline) throws IOException {
@@ -50,6 +60,24 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
}
@Override
+ public Pipeline create(HddsProtos.ReplicationFactor factor)
+ throws IOException {
+ if (autoOpenPipeline) {
+ return super.create(factor);
+ } else {
+ Pipeline initialPipeline = super.create(factor);
+ return Pipeline.newBuilder()
+ .setId(initialPipeline.getId())
+ // overwrite pipeline state to main ALLOCATED
+ .setState(Pipeline.PipelineState.ALLOCATED)
+ .setType(initialPipeline.getType())
+ .setFactor(factor)
+ .setNodes(initialPipeline.getNodes())
+ .build();
+ }
+ }
+
+ @Override
public void shutdown() {
// Do nothing.
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index f5e3f84..66991e4 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.junit.Assume;
@@ -53,9 +54,11 @@ public class TestRatisPipelineProvider {
@Before
public void init() throws Exception {
nodeManager = new MockNodeManager(true, 10);
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 1);
stateManager = new PipelineStateManager();
provider = new MockRatisPipelineProvider(nodeManager,
- stateManager, new OzoneConfiguration());
+ stateManager, conf);
}
private void createPipelineAndAssertions(
@@ -64,6 +67,7 @@ public class TestRatisPipelineProvider {
assertPipelineProperties(pipeline, factor, REPLICATION_TYPE,
Pipeline.PipelineState.ALLOCATED);
stateManager.addPipeline(pipeline);
+ nodeManager.addPipeline(pipeline);
Pipeline pipeline1 = provider.create(factor);
assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE,
@@ -149,6 +153,9 @@ public class TestRatisPipelineProvider {
Pipeline pipeline = provider.create(factor);
assertPipelineProperties(pipeline, factor, REPLICATION_TYPE,
Pipeline.PipelineState.ALLOCATED);
+ nodeManager.addPipeline(pipeline);
+ stateManager.addPipeline(pipeline);
+
List<DatanodeDetails> nodes = pipeline.getNodes();
@@ -184,5 +191,6 @@ public class TestRatisPipelineProvider {
.build();
stateManager.addPipeline(openPipeline);
+ nodeManager.addPipeline(openPipeline);
}
}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index 08f5185..2df851d 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
@@ -28,6 +29,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
@@ -270,7 +272,7 @@ public class TestSCMPipelineManager {
numPipelineCreateFailed = getLongCounter(
"NumPipelineCreationFailed", metrics);
Assert.assertTrue(numPipelineCreateFailed == 1);
-
+
// clean up
pipelineManager.close();
}
@@ -374,6 +376,45 @@ public class TestSCMPipelineManager {
pipelineManager.close();
}
+ @Test
+ public void testScrubPipeline() throws IOException {
+ // No timeout for pipeline scrubber.
+ conf.setTimeDuration(
+ OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, -1,
+ TimeUnit.MILLISECONDS);
+
+ final SCMPipelineManager pipelineManager =
+ new SCMPipelineManager(conf, nodeManager, new EventQueue(), null);
+ final PipelineProvider ratisProvider = new MockRatisPipelineProvider(
+ nodeManager, pipelineManager.getStateManager(), conf, false);
+
+ pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
+ ratisProvider);
+
+ Pipeline pipeline = pipelineManager
+ .createPipeline(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE);
+ // At this point, pipeline is not at OPEN stage.
+ Assert.assertEquals(pipeline.getPipelineState(),
+ Pipeline.PipelineState.ALLOCATED);
+
+ // pipeline should be seen in pipelineManager as ALLOCATED.
+ Assert.assertTrue(pipelineManager
+ .getPipelines(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE,
+ Pipeline.PipelineState.ALLOCATED).contains(pipeline));
+ pipelineManager.scrubPipeline(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE);
+
+ // pipeline should be scrubbed.
+ Assert.assertFalse(pipelineManager
+ .getPipelines(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE,
+ Pipeline.PipelineState.ALLOCATED).contains(pipeline));
+
+ pipelineManager.close();
+ }
+
private void sendPipelineReport(DatanodeDetails dn,
Pipeline pipeline, PipelineReportHandler pipelineReportHandler,
boolean isLeader, EventQueue eventQueue) {
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 17/18: HDDS-2924. Fix Pipeline#nodeIdsHash collision
issue. (#478)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit dd2b7e8cec022ccfc1f4abf193dbb62cc5e883ec
Author: Xiaoyu Yao <xy...@apache.org>
AuthorDate: Mon Jan 27 12:45:58 2020 -0800
HDDS-2924. Fix Pipeline#nodeIdsHash collision issue. (#478)
---
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 38 ++++++++++----------
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 40 ++++++++++------------
.../hdds/scm/pipeline/PipelineStateManager.java | 7 ----
.../hdds/scm/pipeline/RatisPipelineProvider.java | 8 -----
.../hdds/scm/pipeline/RatisPipelineUtils.java | 16 ++-------
.../scm/pipeline/MockRatisPipelineProvider.java | 4 ---
.../scm/pipeline/TestPipelinePlacementPolicy.java | 7 ++--
.../scm/pipeline/TestRatisPipelineProvider.java | 36 ++-----------------
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 17 ++++-----
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 4 +--
.../compose/ozone-topology/docker-compose.yaml | 28 +++++++++++++++
11 files changed, 84 insertions(+), 121 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 6849494..5a28e42 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -21,11 +21,13 @@ package org.apache.hadoop.hdds.scm.pipeline;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -59,8 +61,6 @@ public final class Pipeline {
private UUID leaderId;
// Timestamp for pipeline upon creation
private Instant creationTimestamp;
- // Only valid for Ratis THREE pipeline. No need persist.
- private int nodeIdsHash;
/**
* The immutable properties of pipeline object is used in
@@ -76,7 +76,6 @@ public final class Pipeline {
this.state = state;
this.nodeStatus = nodeStatus;
this.creationTimestamp = Instant.now();
- this.nodeIdsHash = 0;
}
/**
@@ -133,14 +132,6 @@ public final class Pipeline {
this.creationTimestamp = creationTimestamp;
}
- public int getNodeIdsHash() {
- return nodeIdsHash;
- }
-
- void setNodeIdsHash(int nodeIdsHash) {
- this.nodeIdsHash = nodeIdsHash;
- }
-
/**
* Return the pipeline leader's UUID.
*
@@ -167,6 +158,23 @@ public final class Pipeline {
}
/**
+ * Return an immutable set of nodes which form this pipeline.
+ * @return Set of DatanodeDetails
+ */
+ public Set<DatanodeDetails> getNodeSet() {
+ return Collections.unmodifiableSet(nodeStatus.keySet());
+ }
+
+ /**
+ * Check if the input pipeline share the same set of datanodes.
+ * @param pipeline
+ * @return true if the input pipeline shares the same set of datanodes.
+ */
+ public boolean sameDatanodes(Pipeline pipeline) {
+ return getNodeSet().equals(pipeline.getNodeSet());
+ }
+
+ /**
* Returns the leader if found else defaults to closest node.
*
* @return {@link DatanodeDetails}
@@ -360,7 +368,6 @@ public final class Pipeline {
private List<DatanodeDetails> nodesInOrder = null;
private UUID leaderId = null;
private Instant creationTimestamp = null;
- private int nodeIdsHash = 0;
public Builder() {}
@@ -373,7 +380,6 @@ public final class Pipeline {
this.nodesInOrder = pipeline.nodesInOrder.get();
this.leaderId = pipeline.getLeaderId();
this.creationTimestamp = pipeline.getCreationTimestamp();
- this.nodeIdsHash = 0;
}
public Builder setId(PipelineID id1) {
@@ -417,11 +423,6 @@ public final class Pipeline {
return this;
}
- public Builder setNodeIdsHash(int nodeIdsHash1) {
- this.nodeIdsHash = nodeIdsHash1;
- return this;
- }
-
public Pipeline build() {
Preconditions.checkNotNull(id);
Preconditions.checkNotNull(type);
@@ -430,7 +431,6 @@ public final class Pipeline {
Preconditions.checkNotNull(nodeStatus);
Pipeline pipeline = new Pipeline(id, type, factor, state, nodeStatus);
pipeline.setLeaderId(leaderId);
- pipeline.setNodeIdsHash(nodeIdsHash);
// overwrite with original creationTimestamp
if (creationTimestamp != null) {
pipeline.setCreationTimestamp(creationTimestamp);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index 4261a87..9d78063 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -145,12 +145,10 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
String msg;
if (initialHealthyNodesCount < nodesRequired) {
- LOG.warn("Not enough healthy nodes to allocate pipeline." +
- nodesRequired + " datanodes required. Found: " +
- initialHealthyNodesCount);
msg = String.format("Pipeline creation failed due to no sufficient" +
" healthy datanodes. Required %d. Found %d.",
nodesRequired, initialHealthyNodesCount);
+ LOG.warn(msg);
throw new SCMException(msg,
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
@@ -229,42 +227,49 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
// First choose an anchor nodes randomly
DatanodeDetails anchor = chooseNode(healthyNodes);
if (anchor == null) {
- LOG.warn("Unable to find healthy nodes." +
+ LOG.warn("Unable to find healthy node for anchor(first) node." +
" Required nodes: {}, Found nodes: {}",
nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("First node chosen: {}", anchor);
+ }
results.add(anchor);
exclude.add(anchor);
- nodesRequired--;
// Choose the second node on different racks from anchor.
DatanodeDetails nodeOnDifferentRack = chooseNodeBasedOnRackAwareness(
healthyNodes, exclude,
nodeManager.getClusterNetworkTopologyMap(), anchor);
if (nodeOnDifferentRack == null) {
- LOG.warn("Pipeline Placement: Unable to find nodes on different racks " +
- " that meet the criteria. Required nodes: {}, Found nodes: {}",
- nodesRequired, results.size());
+ LOG.warn("Pipeline Placement: Unable to find 2nd node on different " +
+ "racks that meets the criteria. Required nodes: {}, Found nodes:" +
+ " {}", nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.",
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Second node chosen: {}", nodeOnDifferentRack);
+ }
results.add(nodeOnDifferentRack);
exclude.add(nodeOnDifferentRack);
- nodesRequired--;
// Then choose nodes close to anchor based on network topology
- for (int x = 0; x < nodesRequired; x++) {
+ int nodesToFind = nodesRequired - results.size();
+ for (int x = 0; x < nodesToFind; x++) {
// invoke the choose function defined in the derived classes.
DatanodeDetails pick = chooseNodeFromNetworkTopology(
nodeManager.getClusterNetworkTopologyMap(), anchor, exclude);
if (pick != null) {
results.add(pick);
- // exclude the picked node for next time
exclude.add(pick);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remaining node chosen: {}", pick);
+ }
}
}
@@ -306,9 +311,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
datanodeDetails = firstNodeMetric.isGreater(secondNodeMetric.get())
? firstNodeDetails : secondNodeDetails;
}
- // the pick is decided and it should be removed from candidates.
healthyNodes.remove(datanodeDetails);
-
return datanodeDetails;
}
@@ -331,12 +334,10 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
}
for (DatanodeDetails node : healthyNodes) {
- if (excludedNodes.contains(node)
- || networkTopology.isSameParent(anchor, node)) {
+ if (excludedNodes.contains(node) ||
+ anchor.getNetworkLocation().equals(node.getNetworkLocation())) {
continue;
} else {
- // the pick is decided and it should be removed from candidates.
- healthyNodes.remove(node);
return node;
}
}
@@ -374,15 +375,10 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
if (excludedNodes != null && excludedNodes.size() != 0) {
excluded.addAll(excludedNodes);
}
- excluded.add(anchor);
Node pick = networkTopology.chooseRandom(
anchor.getNetworkLocation(), excluded);
DatanodeDetails pickedNode = (DatanodeDetails) pick;
- // exclude the picked node for next time
- if (excludedNodes != null) {
- excludedNodes.add(pickedNode);
- }
return pickedNode;
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
index 051202b..bb56a03 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
@@ -132,13 +132,6 @@ public class PipelineStateManager {
pipeline = pipelineStateMap
.updatePipelineState(pipelineId, PipelineState.OPEN);
}
- // Amend nodeIdsHash if needed.
- if (pipeline.getType() == ReplicationType.RATIS &&
- pipeline.getFactor() == ReplicationFactor.THREE &&
- pipeline.getNodeIdsHash() == 0) {
- pipeline.setNodeIdsHash(RatisPipelineUtils
- .encodeNodeIdsOfFactorThreePipeline(pipeline.getNodes()));
- }
return pipeline;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 4865074..13c3b6a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -157,7 +157,6 @@ public class RatisPipelineProvider implements PipelineProvider {
}
List<DatanodeDetails> dns;
- int nodeIdHash = 0;
switch(factor) {
case ONE:
@@ -166,7 +165,6 @@ public class RatisPipelineProvider implements PipelineProvider {
case THREE:
dns = placementPolicy.chooseDatanodes(null,
null, factor.getNumber(), 0);
- nodeIdHash = RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(dns);
break;
default:
throw new IllegalStateException("Unknown factor: " + factor.name());
@@ -178,7 +176,6 @@ public class RatisPipelineProvider implements PipelineProvider {
.setType(ReplicationType.RATIS)
.setFactor(factor)
.setNodes(dns)
- .setNodeIdsHash(nodeIdHash)
.build();
// Send command to datanodes to create pipeline
@@ -199,17 +196,12 @@ public class RatisPipelineProvider implements PipelineProvider {
@Override
public Pipeline create(ReplicationFactor factor,
List<DatanodeDetails> nodes) {
- int nodeIdHash = 0;
- if (factor == ReplicationFactor.THREE) {
- nodeIdHash = RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes);
- }
return Pipeline.newBuilder()
.setId(PipelineID.randomId())
.setState(PipelineState.ALLOCATED)
.setType(ReplicationType.RATIS)
.setFactor(factor)
.setNodes(nodes)
- .setNodeIdsHash(nodeIdHash)
.build();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
index 7fe1cc1..552ae7d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -96,21 +96,12 @@ public final class RatisPipelineUtils {
}
}
- static int encodeNodeIdsOfFactorThreePipeline(List<DatanodeDetails> nodes) {
- if (nodes.size() != HddsProtos.ReplicationFactor.THREE.getNumber()) {
- return 0;
- }
- return nodes.get(0).getUuid().hashCode() ^
- nodes.get(1).getUuid().hashCode() ^
- nodes.get(2).getUuid().hashCode();
- }
-
/**
* Return the list of pipelines who share the same set of datanodes
* with the input pipeline.
* @param stateManager PipelineStateManager
* @param pipeline input pipeline
- * @return first matched pipeline
+ * @return list of matched pipeline
*/
static List<Pipeline> checkPipelineContainSameDatanodes(
PipelineStateManager stateManager, Pipeline pipeline) {
@@ -118,9 +109,8 @@ public final class RatisPipelineUtils {
HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE)
.stream().filter(p -> !p.getId().equals(pipeline.getId()) &&
- (// For all OPEN or ALLOCATED pipelines
- p.getPipelineState() != Pipeline.PipelineState.CLOSED &&
- p.getNodeIdsHash() == pipeline.getNodeIdsHash()))
+ (p.getPipelineState() != Pipeline.PipelineState.CLOSED &&
+ p.sameDatanodes(pipeline)))
.collect(Collectors.toList());
}
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
index 3eb146a..ff52470 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
@@ -73,8 +73,6 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
.setType(initialPipeline.getType())
.setFactor(factor)
.setNodes(initialPipeline.getNodes())
- .setNodeIdsHash(RatisPipelineUtils
- .encodeNodeIdsOfFactorThreePipeline(initialPipeline.getNodes()))
.build();
}
}
@@ -93,8 +91,6 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
.setType(HddsProtos.ReplicationType.RATIS)
.setFactor(factor)
.setNodes(nodes)
- .setNodeIdsHash(RatisPipelineUtils
- .encodeNodeIdsOfFactorThreePipeline(nodes))
.build();
}
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index 2fff7d9..b9aa9af 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -65,10 +65,10 @@ public class TestPipelinePlacementPolicy {
List<DatanodeDetails> excludedNodes =
new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+ excludedNodes.add(anchor);
DatanodeDetails nextNode = placementPolicy.chooseNodeFromNetworkTopology(
nodeManager.getClusterNetworkTopologyMap(), anchor, excludedNodes);
- // excludedNodes should contain nextNode after being chosen.
- Assert.assertTrue(excludedNodes.contains(nextNode));
+ Assert.assertFalse(excludedNodes.contains(nextNode));
// nextNode should not be the same as anchor.
Assert.assertTrue(anchor.getUuid() != nextNode.getUuid());
}
@@ -83,7 +83,8 @@ public class TestPipelinePlacementPolicy {
DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnRackAwareness(
healthyNodes, new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
topologyWithDifRacks, anchor);
- Assert.assertFalse(topologyWithDifRacks.isSameParent(anchor, nextNode));
+ Assert.assertFalse(anchor.getNetworkLocation().equals(
+ nextNode.getNetworkLocation()));
}
private final static Node[] NODES = new NodeImpl[] {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index a17fc08..86d54b3 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -35,7 +35,6 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.UUID;
import java.util.stream.Collectors;
import static org.apache.commons.collections.CollectionUtils.intersection;
@@ -84,7 +83,7 @@ public class TestRatisPipelineProvider {
intersection(pipeline.getNodes(), pipeline1.getNodes())
.size() < factor.getNumber());
if (pipeline.getFactor() == HddsProtos.ReplicationFactor.THREE) {
- assertNotEquals(pipeline.getNodeIdsHash(), pipeline1.getNodeIdsHash());
+ assertNotEquals(pipeline.getNodeSet(), pipeline1.getNodeSet());
}
stateManager.addPipeline(pipeline1);
nodeManager.addPipeline(pipeline1);
@@ -105,7 +104,7 @@ public class TestRatisPipelineProvider {
stateManager.addPipeline(pipeline1);
// With enough pipeline quote on datanodes, they should not share
// the same set of datanodes.
- assertNotEquals(pipeline.getNodeIdsHash(), pipeline1.getNodeIdsHash());
+ assertNotEquals(pipeline.getNodeSet(), pipeline1.getNodeSet());
}
@Test
@@ -141,33 +140,6 @@ public class TestRatisPipelineProvider {
}
@Test
- public void testComputeNodeIdsHash() {
- int total = HddsProtos.ReplicationFactor.THREE.getNumber();
- List<DatanodeDetails> nodes1 = new ArrayList<>();
- for (int i = 0; i < total; i++) {
- nodes1.add(MockDatanodeDetails.createDatanodeDetails(
- UUID.fromString("00000-11000-00000-00000-0000" + (i + 1))));
- }
-
- Assert.assertEquals(total, nodes1.size());
- Assert.assertNotEquals(0,
- RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes1));
-
- List<DatanodeDetails> nodes2 = new ArrayList<>();
- for (int i = 0; i < total; i++) {
- nodes2.add(MockDatanodeDetails.createDatanodeDetails(
- UUID.fromString("00000-11000-00000-00000-0000" + (total - i))));
- }
- Assert.assertEquals(total, nodes2.size());
- Assert.assertNotEquals(0,
- RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes2));
-
- Assert.assertEquals(
- RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes1),
- RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes2));
- }
-
- @Test
public void testCreateFactorTHREEPipelineWithSameDatanodes() {
List<DatanodeDetails> healthyNodes = nodeManager
.getNodes(HddsProtos.NodeState.HEALTHY).stream()
@@ -178,9 +150,7 @@ public class TestRatisPipelineProvider {
Pipeline pipeline2 = provider.create(
HddsProtos.ReplicationFactor.THREE, healthyNodes);
- Assert.assertTrue(pipeline1.getNodes().parallelStream()
- .allMatch(pipeline2.getNodes()::contains));
- Assert.assertEquals(pipeline1.getNodeIdsHash(), pipeline2.getNodeIdsHash());
+ Assert.assertEquals(pipeline1.getNodeSet(), pipeline2.getNodeSet());
}
@Test
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index deba91b..ab23153 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import static org.apache.commons.collections.CollectionUtils.intersection;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
@@ -116,15 +115,13 @@ public class TestSCMPipelineManager {
List<Pipeline> pipelineList =
pipelineManager.getPipelines(HddsProtos.ReplicationType.RATIS);
Assert.assertEquals(pipelines, new HashSet<>(pipelineList));
- // All NodeIdsHash from original pipeline list
- List<Integer> originalPipelineHash = pipelineList.stream()
- .map(Pipeline::getNodeIdsHash).collect(Collectors.toList());
- // All NodeIdsHash from reloaded pipeline list
- List<Integer> reloadedPipelineHash = pipelines.stream()
- .map(Pipeline::getNodeIdsHash).collect(Collectors.toList());
- // Original NodeIdsHash list should contain same items from reloaded one.
- Assert.assertEquals(pipelineNum,
- intersection(originalPipelineHash, reloadedPipelineHash).size());
+
+ Set<Set<DatanodeDetails>> originalPipelines = pipelineList.stream()
+ .map(Pipeline::getNodeSet).collect(Collectors.toSet());
+ Set<Set<DatanodeDetails>> reloadedPipelineHash = pipelines.stream()
+ .map(Pipeline::getNodeSet).collect(Collectors.toSet());
+ Assert.assertEquals(reloadedPipelineHash, originalPipelines);
+ Assert.assertEquals(pipelineNum, originalPipelines.size());
// clean up
for (Pipeline pipeline : pipelines) {
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
index badfadc..e4060b3 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
@@ -114,8 +114,8 @@ public class ListInfoSubcommand implements Callable<Void> {
pipelineListInfo.append("No pipelines in cluster.");
}
System.out.println("Datanode: " + datanode.getUuid().toString() +
- " (" + datanode.getIpAddress() + "/"
- + datanode.getHostName() + "/" + relatedPipelineNum +
+ " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress()
+ + "/" + datanode.getHostName() + "/" + relatedPipelineNum +
" pipelines) \n" + "Related pipelines: \n" + pipelineListInfo);
}
}
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
index 69611fa..ccd131c 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
@@ -72,6 +72,34 @@ services:
networks:
net:
ipv4_address: 10.5.0.7
+ datanode_5:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ privileged: true #required by the profiler
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9864
+ - 9882
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ networks:
+ net:
+ ipv4_address: 10.5.0.8
+ datanode_6:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ privileged: true #required by the profiler
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9864
+ - 9882
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ networks:
+ net:
+ ipv4_address: 10.5.0.9
om:
image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
privileged: true #required by the profiler
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 11/18: HDDS-1574 Average out pipeline allocation on
datanodes and add metrcs/test (#291)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit d6b9ec0a5667322f2eb2c820953684c9f6ab1cc1
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Thu Dec 19 06:00:35 2019 +0800
HDDS-1574 Average out pipeline allocation on datanodes and add metrcs/test (#291)
---
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 19 +++
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 3 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 7 ++
.../hdds/scm/pipeline/RatisPipelineProvider.java | 8 ++
.../hdds/scm/pipeline/RatisPipelineUtils.java | 38 +++++-
.../hdds/scm/pipeline/SCMPipelineManager.java | 12 ++
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 9 ++
.../hadoop/hdds/scm/container/MockNodeManager.java | 14 ++-
.../container/TestCloseContainerEventHandler.java | 7 +-
.../scm/pipeline/MockRatisPipelineProvider.java | 4 +
.../TestPipelineDatanodesIntersection.java | 129 +++++++++++++++++++++
.../scm/pipeline/TestRatisPipelineProvider.java | 73 ++++++++++--
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 15 ++-
.../testutils/ReplicationNodeManagerMock.java | 4 +-
14 files changed, 324 insertions(+), 18 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 295156d..1dc2373 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -58,6 +58,8 @@ public final class Pipeline {
private UUID leaderId;
// Timestamp for pipeline upon creation
private Long creationTimestamp;
+ // Only valid for Ratis THREE pipeline. No need persist.
+ private int nodeIdsHash;
/**
* The immutable properties of pipeline object is used in
@@ -73,6 +75,7 @@ public final class Pipeline {
this.state = state;
this.nodeStatus = nodeStatus;
this.creationTimestamp = System.currentTimeMillis();
+ this.nodeIdsHash = 0;
}
/**
@@ -129,6 +132,14 @@ public final class Pipeline {
this.creationTimestamp = creationTimestamp;
}
+ public int getNodeIdsHash() {
+ return nodeIdsHash;
+ }
+
+ void setNodeIdsHash(int nodeIdsHash) {
+ this.nodeIdsHash = nodeIdsHash;
+ }
+
/**
* Return the pipeline leader's UUID.
*
@@ -347,6 +358,7 @@ public final class Pipeline {
private List<DatanodeDetails> nodesInOrder = null;
private UUID leaderId = null;
private Long creationTimestamp = null;
+ private int nodeIdsHash = 0;
public Builder() {}
@@ -359,6 +371,7 @@ public final class Pipeline {
this.nodesInOrder = pipeline.nodesInOrder.get();
this.leaderId = pipeline.getLeaderId();
this.creationTimestamp = pipeline.getCreationTimestamp();
+ this.nodeIdsHash = 0;
}
public Builder setId(PipelineID id1) {
@@ -397,6 +410,11 @@ public final class Pipeline {
return this;
}
+ public Builder setNodeIdsHash(int nodeIdsHash1) {
+ this.nodeIdsHash = nodeIdsHash1;
+ return this;
+ }
+
public Pipeline build() {
Preconditions.checkNotNull(id);
Preconditions.checkNotNull(type);
@@ -405,6 +423,7 @@ public final class Pipeline {
Preconditions.checkNotNull(nodeStatus);
Pipeline pipeline = new Pipeline(id, type, factor, state, nodeStatus);
pipeline.setLeaderId(leaderId);
+ pipeline.setNodeIdsHash(nodeIdsHash);
// overwrite with original creationTimestamp
if (creationTimestamp != null) {
pipeline.setCreationTimestamp(creationTimestamp);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index 23eb574..bc65d14 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -162,7 +162,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
// filter nodes that meet the size and pipeline engagement criteria.
// Pipeline placement doesn't take node space left into account.
List<DatanodeDetails> healthyList = healthyNodes.stream()
- .filter(d -> meetCriteria(d, nodesRequired)).limit(nodesRequired)
+ .filter(d -> meetCriteria(d, nodesRequired))
.collect(Collectors.toList());
if (healthyList.size() < nodesRequired) {
@@ -308,6 +308,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
}
// the pick is decided and it should be removed from candidates.
healthyNodes.remove(datanodeDetails);
+
return datanodeDetails;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
index bb56a03..051202b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
@@ -132,6 +132,13 @@ public class PipelineStateManager {
pipeline = pipelineStateMap
.updatePipelineState(pipelineId, PipelineState.OPEN);
}
+ // Amend nodeIdsHash if needed.
+ if (pipeline.getType() == ReplicationType.RATIS &&
+ pipeline.getFactor() == ReplicationFactor.THREE &&
+ pipeline.getNodeIdsHash() == 0) {
+ pipeline.setNodeIdsHash(RatisPipelineUtils
+ .encodeNodeIdsOfFactorThreePipeline(pipeline.getNodes()));
+ }
return pipeline;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 23b02ed..9585907 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -157,6 +157,7 @@ public class RatisPipelineProvider implements PipelineProvider {
}
List<DatanodeDetails> dns;
+ int nodeIdHash = 0;
switch(factor) {
case ONE:
@@ -165,6 +166,7 @@ public class RatisPipelineProvider implements PipelineProvider {
case THREE:
dns = placementPolicy.chooseDatanodes(null,
null, factor.getNumber(), 0);
+ nodeIdHash = RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(dns);
break;
default:
throw new IllegalStateException("Unknown factor: " + factor.name());
@@ -176,6 +178,7 @@ public class RatisPipelineProvider implements PipelineProvider {
.setType(ReplicationType.RATIS)
.setFactor(factor)
.setNodes(dns)
+ .setNodeIdsHash(nodeIdHash)
.build();
// Send command to datanodes to create pipeline
@@ -196,12 +199,17 @@ public class RatisPipelineProvider implements PipelineProvider {
@Override
public Pipeline create(ReplicationFactor factor,
List<DatanodeDetails> nodes) {
+ int nodeIdHash = 0;
+ if (factor == ReplicationFactor.THREE) {
+ nodeIdHash = RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes);
+ }
return Pipeline.newBuilder()
.setId(PipelineID.randomId())
.setState(PipelineState.ALLOCATED)
.setType(ReplicationType.RATIS)
.setFactor(factor)
.setNodes(nodes)
+ .setNodeIdsHash(nodeIdHash)
.build();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
index b8cdf06..f9f2011 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -18,9 +18,12 @@
package org.apache.hadoop.hdds.scm.pipeline;
import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.ratis.client.RaftClient;
@@ -33,7 +36,6 @@ import org.apache.ratis.rpc.SupportedRpcType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* Utility class for Ratis pipelines. Contains methods to create and destroy
* ratis pipelines.
@@ -93,4 +95,38 @@ public final class RatisPipelineUtils {
true, p.getId());
}
}
+
+ static int encodeNodeIdsOfFactorThreePipeline(List<DatanodeDetails> nodes) {
+ if (nodes.size() != HddsProtos.ReplicationFactor.THREE.getNumber()) {
+ return 0;
+ }
+ return nodes.get(0).getUuid().hashCode() ^
+ nodes.get(1).getUuid().hashCode() ^
+ nodes.get(2).getUuid().hashCode();
+ }
+
+ /**
+ * Return first existed pipeline which share the same set of datanodes
+ * with the input pipeline.
+ * @param stateManager PipelineStateManager
+ * @param pipeline input pipeline
+ * @return first matched pipeline
+ */
+ static Pipeline checkPipelineContainSameDatanodes(
+ PipelineStateManager stateManager, Pipeline pipeline) {
+ List<Pipeline> matchedPipelines = stateManager.getPipelines(
+ HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE)
+ .stream().filter(p -> !p.getId().equals(pipeline.getId()) &&
+ (// For all OPEN or ALLOCATED pipelines
+ p.getPipelineState() == Pipeline.PipelineState.OPEN ||
+ p.getPipelineState() == Pipeline.PipelineState.ALLOCATED) &&
+ p.getNodeIdsHash() == pipeline.getNodeIdsHash())
+ .collect(Collectors.toList());
+ if (matchedPipelines.size() == 0) {
+ return null;
+ } else {
+ return matchedPipelines.stream().findFirst().get();
+ }
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 01af465..11e9916 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -172,6 +172,18 @@ public class SCMPipelineManager implements PipelineManager {
metrics.incNumPipelineCreated();
metrics.createPerPipelineMetrics(pipeline);
}
+ Pipeline overlapPipeline = RatisPipelineUtils
+ .checkPipelineContainSameDatanodes(stateManager, pipeline);
+ if (overlapPipeline != null) {
+ metrics.incNumPipelineContainSameDatanodes();
+ //TODO remove until pipeline allocation is proved equally distributed.
+ LOG.info("Pipeline: " + pipeline.getId().toString() +
+ " contains same datanodes as previous pipeline: " +
+ overlapPipeline.getId().toString() + " nodeIds: " +
+ pipeline.getNodes().get(0).getUuid().toString() +
+ ", " + pipeline.getNodes().get(1).getUuid().toString() +
+ ", " + pipeline.getNodes().get(2).getUuid().toString());
+ }
return pipeline;
} catch (IOException ex) {
metrics.incNumPipelineCreationFailed();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
index 8c348ed..1cf8d3a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
@@ -54,6 +54,7 @@ public final class SCMPipelineMetrics implements MetricsSource {
private @Metric MutableCounterLong numPipelineDestroyFailed;
private @Metric MutableCounterLong numPipelineReportProcessed;
private @Metric MutableCounterLong numPipelineReportProcessingFailed;
+ private @Metric MutableCounterLong numPipelineContainSameDatanodes;
private Map<PipelineID, MutableCounterLong> numBlocksAllocated;
/** Private constructor. */
@@ -92,6 +93,7 @@ public final class SCMPipelineMetrics implements MetricsSource {
numPipelineDestroyFailed.snapshot(recordBuilder, true);
numPipelineReportProcessed.snapshot(recordBuilder, true);
numPipelineReportProcessingFailed.snapshot(recordBuilder, true);
+ numPipelineContainSameDatanodes.snapshot(recordBuilder, true);
numBlocksAllocated
.forEach((pid, metric) -> metric.snapshot(recordBuilder, true));
}
@@ -176,4 +178,11 @@ public final class SCMPipelineMetrics implements MetricsSource {
void incNumPipelineReportProcessingFailed() {
numPipelineReportProcessingFailed.incr();
}
+
+ /**
+ * Increments number of pipeline who contains same set of datanodes.
+ */
+ void incNumPipelineContainSameDatanodes() {
+ numPipelineContainSameDatanodes.incr();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index bca4189..cbeef7f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.net.NetConstants;
@@ -93,7 +93,8 @@ public class MockNodeManager implements NodeManager {
private NetworkTopology clusterMap;
private ConcurrentMap<String, Set<String>> dnsToUuidMap;
- public MockNodeManager(boolean initializeFakeNodes, int nodeCount) {
+ public MockNodeManager(NetworkTopologyImpl clusterMap,
+ boolean initializeFakeNodes, int nodeCount) {
this.healthyNodes = new LinkedList<>();
this.staleNodes = new LinkedList<>();
this.deadNodes = new LinkedList<>();
@@ -101,8 +102,8 @@ public class MockNodeManager implements NodeManager {
this.node2PipelineMap = new Node2PipelineMap();
this.node2ContainerMap = new Node2ContainerMap();
this.dnsToUuidMap = new ConcurrentHashMap<>();
- aggregateStat = new SCMNodeStat();
- clusterMap = new NetworkTopologyImpl(new Configuration());
+ this.aggregateStat = new SCMNodeStat();
+ this.clusterMap = clusterMap;
if (initializeFakeNodes) {
for (int x = 0; x < nodeCount; x++) {
DatanodeDetails dd = MockDatanodeDetails.randomDatanodeDetails();
@@ -114,6 +115,11 @@ public class MockNodeManager implements NodeManager {
this.commandMap = new HashMap<>();
}
+ public MockNodeManager(boolean initializeFakeNodes, int nodeCount) {
+ this(new NetworkTopologyImpl(new OzoneConfiguration()),
+ initializeFakeNodes, nodeCount);
+ }
+
/**
* Invoked from ctor to create some node Metrics.
*
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
index 612bf5d..f35bfe2 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
@@ -67,13 +68,14 @@ public class TestCloseContainerEventHandler {
.getTestDir(TestCloseContainerEventHandler.class.getSimpleName());
configuration
.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
+ configuration.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_NUMBER_LIMIT, 16);
nodeManager = new MockNodeManager(true, 10);
eventQueue = new EventQueue();
pipelineManager =
new SCMPipelineManager(configuration, nodeManager, eventQueue);
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
- pipelineManager.getStateManager(), configuration);
+ pipelineManager.getStateManager(), configuration, eventQueue);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
containerManager = new SCMContainerManager(configuration, pipelineManager);
@@ -91,6 +93,9 @@ public class TestCloseContainerEventHandler {
if (containerManager != null) {
containerManager.close();
}
+ if (pipelineManager != null) {
+ pipelineManager.close();
+ }
FileUtil.fullyDelete(testDir);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
index ff52470..3eb146a 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
@@ -73,6 +73,8 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
.setType(initialPipeline.getType())
.setFactor(factor)
.setNodes(initialPipeline.getNodes())
+ .setNodeIdsHash(RatisPipelineUtils
+ .encodeNodeIdsOfFactorThreePipeline(initialPipeline.getNodes()))
.build();
}
}
@@ -91,6 +93,8 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider {
.setType(HddsProtos.ReplicationType.RATIS)
.setFactor(factor)
.setNodes(nodes)
+ .setNodeIdsHash(RatisPipelineUtils
+ .encodeNodeIdsOfFactorThreePipeline(nodes))
.build();
}
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
new file mode 100644
index 0000000..45f85ef
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE;
+
+/**
+ * Test for pipeline datanodes intersection.
+ */
+@RunWith(Parameterized.class)
+public class TestPipelineDatanodesIntersection {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestPipelineDatanodesIntersection.class.getName());
+
+ private int nodeCount;
+ private int nodeHeaviness;
+ private OzoneConfiguration conf;
+ private boolean end;
+
+ @Before
+ public void initialize() {
+ conf = new OzoneConfiguration();
+ end = false;
+ }
+
+ public TestPipelineDatanodesIntersection(int nodeCount, int nodeHeaviness) {
+ this.nodeCount = nodeCount;
+ this.nodeHeaviness = nodeHeaviness;
+ }
+
+ @Parameterized.Parameters
+ public static Collection inputParams() {
+ return Arrays.asList(new Object[][] {
+ {4, 5},
+ {10, 5},
+ {20, 5},
+ {50, 5},
+ {100, 5},
+ {100, 10}
+ });
+ }
+
+ @Test
+ public void testPipelineDatanodesIntersection() {
+ NodeManager nodeManager= new MockNodeManager(true, nodeCount);
+ conf.setInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, nodeHeaviness);
+ conf.setBoolean(OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, false);
+ PipelineStateManager stateManager = new PipelineStateManager(conf);
+ PipelineProvider provider = new MockRatisPipelineProvider(nodeManager,
+ stateManager, conf);
+
+ int healthyNodeCount = nodeManager
+ .getNodeCount(HddsProtos.NodeState.HEALTHY);
+ int intersectionCount = 0;
+ int createdPipelineCount = 0;
+ while (!end && createdPipelineCount <= healthyNodeCount * nodeHeaviness) {
+ try {
+ Pipeline pipeline = provider.create(HddsProtos.ReplicationFactor.THREE);
+ stateManager.addPipeline(pipeline);
+ nodeManager.addPipeline(pipeline);
+ Pipeline overlapPipeline = RatisPipelineUtils
+ .checkPipelineContainSameDatanodes(stateManager, pipeline);
+ if (overlapPipeline != null){
+ intersectionCount++;
+ LOG.info("This pipeline: " + pipeline.getId().toString() +
+ " overlaps with previous pipeline: " + overlapPipeline.getId() +
+ ". They share same set of datanodes as: " +
+ pipeline.getNodesInOrder().get(0).getUuid() + "/" +
+ pipeline.getNodesInOrder().get(1).getUuid() + "/" +
+ pipeline.getNodesInOrder().get(2).getUuid() + " and " +
+ overlapPipeline.getNodesInOrder().get(0).getUuid() + "/" +
+ overlapPipeline.getNodesInOrder().get(1).getUuid() + "/" +
+ overlapPipeline.getNodesInOrder().get(2).getUuid() +
+ " is the same.");
+ }
+ createdPipelineCount++;
+ } catch(SCMException e) {
+ end = true;
+ } catch (IOException e) {
+ end = true;
+ // Should not throw regular IOException.
+ Assert.fail();
+ }
+ }
+
+ end = false;
+
+ LOG.info("Among total " +
+ stateManager.getPipelines(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE).size() + " created pipelines" +
+ " with " + healthyNodeCount + " healthy datanodes and " +
+ nodeHeaviness + " as node heaviness, " +
+ intersectionCount + " pipelines has same set of datanodes.");
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index 66991e4..46fd8c8 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -34,9 +35,14 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
import static org.apache.commons.collections.CollectionUtils.intersection;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
/**
@@ -50,12 +56,13 @@ public class TestRatisPipelineProvider {
private NodeManager nodeManager;
private PipelineProvider provider;
private PipelineStateManager stateManager;
+ private OzoneConfiguration conf;
@Before
public void init() throws Exception {
nodeManager = new MockNodeManager(true, 10);
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 1);
+ conf.setInt(ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT, 2);
stateManager = new PipelineStateManager();
provider = new MockRatisPipelineProvider(nodeManager,
stateManager, conf);
@@ -75,8 +82,12 @@ public class TestRatisPipelineProvider {
// New pipeline should not overlap with the previous created pipeline
assertTrue(
intersection(pipeline.getNodes(), pipeline1.getNodes())
- .isEmpty());
+ .size() < factor.getNumber());
+ if (pipeline.getFactor() == HddsProtos.ReplicationFactor.THREE) {
+ assertNotEquals(pipeline.getNodeIdsHash(), pipeline1.getNodeIdsHash());
+ }
stateManager.addPipeline(pipeline1);
+ nodeManager.addPipeline(pipeline1);
}
@Test
@@ -92,10 +103,9 @@ public class TestRatisPipelineProvider {
assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE,
Pipeline.PipelineState.ALLOCATED);
stateManager.addPipeline(pipeline1);
- // New pipeline should overlap with the previous created pipeline,
- // and one datanode should overlap between the two types.
- assertEquals(1,
- intersection(pipeline.getNodes(), pipeline1.getNodes()).size());
+ // With enough pipeline quote on datanodes, they should not share
+ // the same set of datanodes.
+ assertNotEquals(pipeline.getNodeIdsHash(), pipeline1.getNodeIdsHash());
}
@Test
@@ -131,6 +141,49 @@ public class TestRatisPipelineProvider {
}
@Test
+ public void testComputeNodeIdsHash() {
+ int total = HddsProtos.ReplicationFactor.THREE.getNumber();
+ List<DatanodeDetails> nodes1 = new ArrayList<>();
+ for (int i = 0; i < total; i++) {
+ nodes1.add(MockDatanodeDetails.createDatanodeDetails(
+ UUID.fromString("00000-11000-00000-00000-0000" + (i + 1))));
+ }
+
+ Assert.assertEquals(total, nodes1.size());
+ Assert.assertNotEquals(0,
+ RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes1));
+
+ List<DatanodeDetails> nodes2 = new ArrayList<>();
+ for (int i = 0; i < total; i++) {
+ nodes2.add(MockDatanodeDetails.createDatanodeDetails(
+ UUID.fromString("00000-11000-00000-00000-0000" + (total - i))));
+ }
+ Assert.assertEquals(total, nodes2.size());
+ Assert.assertNotEquals(0,
+ RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes2));
+
+ Assert.assertEquals(
+ RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes1),
+ RatisPipelineUtils.encodeNodeIdsOfFactorThreePipeline(nodes2));
+ }
+
+ @Test
+ public void testCreateFactorTHREEPipelineWithSameDatanodes() {
+ List<DatanodeDetails> healthyNodes = nodeManager
+ .getNodes(HddsProtos.NodeState.HEALTHY).stream()
+ .limit(3).collect(Collectors.toList());
+
+ Pipeline pipeline1 = provider.create(
+ HddsProtos.ReplicationFactor.THREE, healthyNodes);
+ Pipeline pipeline2 = provider.create(
+ HddsProtos.ReplicationFactor.THREE, healthyNodes);
+
+ Assert.assertTrue(pipeline1.getNodes().parallelStream()
+ .allMatch(pipeline2.getNodes()::contains));
+ Assert.assertEquals(pipeline1.getNodeIdsHash(), pipeline2.getNodeIdsHash());
+ }
+
+ @Test
public void testCreatePipelinesDnExclude() throws IOException {
List<DatanodeDetails> healthyNodes =
nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
@@ -141,7 +194,11 @@ public class TestRatisPipelineProvider {
// Use up first 3 DNs for an open pipeline.
List<DatanodeDetails> dns = healthyNodes.subList(0, 3);
- addPipeline(dns, factor, Pipeline.PipelineState.OPEN, REPLICATION_TYPE);
+ for (int i = 0; i < conf.getInt(OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
+ OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT); i++) {
+ // Saturate pipeline counts on all the 1st 3 DNs.
+ addPipeline(dns, factor, Pipeline.PipelineState.OPEN, REPLICATION_TYPE);
+ }
Set<DatanodeDetails> membersOfOpenPipelines = new HashSet<>(dns);
// Use up next 3 DNs for a closed pipeline.
@@ -160,7 +217,7 @@ public class TestRatisPipelineProvider {
List<DatanodeDetails> nodes = pipeline.getNodes();
assertTrue(
- "nodes of new pipeline cannot be from open pipelines",
+ "nodes of new pipeline cannot be all from open pipelines",
nodes.stream().noneMatch(membersOfOpenPipelines::contains));
assertTrue(
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index 491e289..e6bf7a0 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
+import static org.apache.commons.collections.CollectionUtils.intersection;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
@@ -30,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
@@ -89,8 +91,10 @@ public class TestSCMPipelineManager {
pipelineManager.getStateManager(), conf);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
+ int pipelineNum = 5;
+
Set<Pipeline> pipelines = new HashSet<>();
- for (int i = 0; i < 5; i++) {
+ for (int i = 0; i < pipelineNum; i++) {
Pipeline pipeline = pipelineManager
.createPipeline(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
@@ -112,6 +116,15 @@ public class TestSCMPipelineManager {
List<Pipeline> pipelineList =
pipelineManager.getPipelines(HddsProtos.ReplicationType.RATIS);
Assert.assertEquals(pipelines, new HashSet<>(pipelineList));
+ // All NodeIdsHash from original pipeline list
+ List<Integer> originalPipelineHash = pipelineList.stream()
+ .map(Pipeline::getNodeIdsHash).collect(Collectors.toList());
+ // All NodeIdsHash from reloaded pipeline list
+ List<Integer> reloadedPipelineHash = pipelines.stream()
+ .map(Pipeline::getNodeIdsHash).collect(Collectors.toList());
+ // Original NodeIdsHash list should contain same items from reloaded one.
+ Assert.assertEquals(pipelineNum,
+ intersection(originalPipelineHash, reloadedPipelineHash).size());
// clean up
for (Pipeline pipeline : pipelines) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 7e8ec52..0698443 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -168,11 +168,11 @@ public class ReplicationNodeManagerMock implements NodeManager {
/**
* Get the count of pipelines a datanodes is associated with.
- * @param dnId DatanodeDetails
+ * @param dn DatanodeDetails
* @return The number of pipelines
*/
@Override
- public int getPipelinesCount(DatanodeDetails dnId) {
+ public int getPipelinesCount(DatanodeDetails dn) {
throw new UnsupportedOperationException("Not yet implemented");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 18/18: HDDS-2923 Add fall-back protection for rack
awareness in pipeline creation. (#516)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit d5666c5e3dde1c3a4e8ceb888ffea2607a1ab254
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Mon Feb 10 09:13:57 2020 +0800
HDDS-2923 Add fall-back protection for rack awareness in pipeline creation. (#516)
---
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 81 ++++++++++++++++------
.../scm/pipeline/TestPipelinePlacementPolicy.java | 56 +++++++++++++++
2 files changed, 114 insertions(+), 23 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index 9d78063..0f30449 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -208,6 +208,29 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
}
}
+ // Fall back logic for node pick up.
+ DatanodeDetails fallBackPickNodes(
+ List<DatanodeDetails> nodeSet, List<DatanodeDetails> excludedNodes)
+ throws SCMException{
+ DatanodeDetails node;
+ if (excludedNodes == null || excludedNodes.isEmpty()) {
+ node = chooseNode(nodeSet);
+ } else {
+ List<DatanodeDetails> inputNodes = nodeSet.stream()
+ .filter(p -> !excludedNodes.contains(p)).collect(Collectors.toList());
+ node = chooseNode(inputNodes);
+ }
+
+ if (node == null) {
+ String msg = String.format("Unable to find fall back node in" +
+ " pipeline allocation. nodeSet size: {}", nodeSet.size());
+ LOG.warn(msg);
+ throw new SCMException(msg,
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return node;
+ }
+
/**
* Get result set based on the pipeline placement algorithm which considers
* network topology and rack awareness.
@@ -220,50 +243,59 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
public List<DatanodeDetails> getResultSet(
int nodesRequired, List<DatanodeDetails> healthyNodes)
throws SCMException {
+ if (nodesRequired != HddsProtos.ReplicationFactor.THREE.getNumber()) {
+ throw new SCMException("Nodes required number is not supported: " +
+ nodesRequired, SCMException.ResultCodes.INVALID_CAPACITY);
+ }
+
+ // Assume rack awareness is not enabled.
+ boolean rackAwareness = false;
List <DatanodeDetails> results = new ArrayList<>(nodesRequired);
// Since nodes are widely distributed, the results should be selected
// base on distance in topology, rack awareness and load balancing.
List<DatanodeDetails> exclude = new ArrayList<>();
// First choose an anchor nodes randomly
DatanodeDetails anchor = chooseNode(healthyNodes);
- if (anchor == null) {
- LOG.warn("Unable to find healthy node for anchor(first) node." +
- " Required nodes: {}, Found nodes: {}",
- nodesRequired, results.size());
- throw new SCMException("Unable to find required number of nodes.",
+ if (anchor != null) {
+ results.add(anchor);
+ exclude.add(anchor);
+ } else {
+ LOG.warn("Unable to find healthy node for anchor(first) node.");
+ throw new SCMException("Unable to find anchor node.",
SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
if (LOG.isDebugEnabled()) {
LOG.debug("First node chosen: {}", anchor);
}
- results.add(anchor);
- exclude.add(anchor);
// Choose the second node on different racks from anchor.
- DatanodeDetails nodeOnDifferentRack = chooseNodeBasedOnRackAwareness(
+ DatanodeDetails nextNode = chooseNodeBasedOnRackAwareness(
healthyNodes, exclude,
nodeManager.getClusterNetworkTopologyMap(), anchor);
- if (nodeOnDifferentRack == null) {
- LOG.warn("Pipeline Placement: Unable to find 2nd node on different " +
- "racks that meets the criteria. Required nodes: {}, Found nodes:" +
- " {}", nodesRequired, results.size());
- throw new SCMException("Unable to find required number of nodes.",
- SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Second node chosen: {}", nodeOnDifferentRack);
+ if (nextNode != null) {
+ // Rack awareness is detected.
+ rackAwareness = true;
+ results.add(nextNode);
+ exclude.add(nextNode);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Second node chosen: {}", nextNode);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Pipeline Placement: Unable to find 2nd node on different " +
+ "rack based on rack awareness.");
+ }
}
- results.add(nodeOnDifferentRack);
- exclude.add(nodeOnDifferentRack);
-
// Then choose nodes close to anchor based on network topology
int nodesToFind = nodesRequired - results.size();
for (int x = 0; x < nodesToFind; x++) {
- // invoke the choose function defined in the derived classes.
- DatanodeDetails pick = chooseNodeFromNetworkTopology(
- nodeManager.getClusterNetworkTopologyMap(), anchor, exclude);
+ // Pick remaining nodes based on the existence of rack awareness.
+ DatanodeDetails pick = rackAwareness
+ ? chooseNodeFromNetworkTopology(
+ nodeManager.getClusterNetworkTopologyMap(), anchor, exclude)
+ : fallBackPickNodes(healthyNodes, exclude);
if (pick != null) {
results.add(pick);
exclude.add(pick);
@@ -293,6 +325,9 @@ public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
@Override
public DatanodeDetails chooseNode(
List<DatanodeDetails> healthyNodes) {
+ if (healthyNodes == null || healthyNodes.isEmpty()) {
+ return null;
+ }
int firstNodeNdx = getRand().nextInt(healthyNodes.size());
int secondNodeNdx = getRand().nextInt(healthyNodes.size());
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index b9aa9af..daad808 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -83,10 +83,66 @@ public class TestPipelinePlacementPolicy {
DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnRackAwareness(
healthyNodes, new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
topologyWithDifRacks, anchor);
+ Assert.assertNotNull(nextNode);
Assert.assertFalse(anchor.getNetworkLocation().equals(
nextNode.getNetworkLocation()));
}
+ @Test
+ public void testFallBackPickNodes() {
+ List<DatanodeDetails> healthyNodes = overWriteLocationInNodes(
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY));
+ DatanodeDetails node;
+ try {
+ node = placementPolicy.fallBackPickNodes(healthyNodes, null);
+ Assert.assertNotNull(node);
+ } catch (SCMException e) {
+ Assert.fail("Should not reach here.");
+ }
+
+ // when input nodeSet are all excluded.
+ List<DatanodeDetails> exclude = healthyNodes;
+ try {
+ node = placementPolicy.fallBackPickNodes(healthyNodes, exclude);
+ Assert.assertNull(node);
+ } catch (SCMException e) {
+ Assert.assertEquals(SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE,
+ e.getResult());
+ } catch (Exception ex) {
+ Assert.fail("Should not reach here.");
+ }
+ }
+
+ @Test
+ public void testRackAwarenessNotEnabledWithFallBack() throws SCMException{
+ List<DatanodeDetails> healthyNodes =
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
+ DatanodeDetails anchor = placementPolicy.chooseNode(healthyNodes);
+ DatanodeDetails randomNode = placementPolicy.chooseNode(healthyNodes);
+ // rack awareness is not enabled.
+ Assert.assertTrue(anchor.getNetworkLocation().equals(
+ randomNode.getNetworkLocation()));
+
+ NetworkTopology topology = new NetworkTopologyImpl(new Configuration());
+ DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnRackAwareness(
+ healthyNodes, new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ topology, anchor);
+ // RackAwareness should not be able to choose any node.
+ Assert.assertNull(nextNode);
+
+ // PlacementPolicy should still be able to pick a set of 3 nodes.
+ int numOfNodes = HddsProtos.ReplicationFactor.THREE.getNumber();
+ List<DatanodeDetails> results = placementPolicy
+ .getResultSet(numOfNodes, healthyNodes);
+
+ Assert.assertEquals(numOfNodes, results.size());
+ // All nodes are on same rack.
+ Assert.assertEquals(results.get(0).getNetworkLocation(),
+ results.get(1).getNetworkLocation());
+ Assert.assertEquals(results.get(0).getNetworkLocation(),
+ results.get(2).getNetworkLocation());
+ }
+
private final static Node[] NODES = new NodeImpl[] {
new NodeImpl("h1", "/r1", NetConstants.NODE_COST_DEFAULT),
new NodeImpl("h2", "/r1", NetConstants.NODE_COST_DEFAULT),
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 07/18: HDDS-2650 Fix createPipeline CLI. (#340)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit e48ec929abe7e5bb5dc65ea3ba9892845249d2d2
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Thu Dec 12 14:11:00 2019 +0800
HDDS-2650 Fix createPipeline CLI. (#340)
---
.../hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
index edeb786..58a1778 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.cli.pipeline;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.cli.SCMCLI;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import picocli.CommandLine;
@@ -30,13 +29,13 @@ import java.util.concurrent.Callable;
* Handler of createPipeline command.
*/
@CommandLine.Command(
- name = "createPipeline",
+ name = "create",
description = "create pipeline",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
public class CreatePipelineSubcommand implements Callable<Void> {
@CommandLine.ParentCommand
- private SCMCLI parent;
+ private PipelineCommands parent;
@CommandLine.Option(
names = {"-t", "--replicationType"},
@@ -60,7 +59,7 @@ public class CreatePipelineSubcommand implements Callable<Void> {
throw new IllegalArgumentException(type.name()
+ " is not supported yet.");
}
- try (ScmClient scmClient = parent.createScmClient()) {
+ try (ScmClient scmClient = parent.getParent().createScmClient()) {
scmClient.createReplicationPipeline(
type,
factor,
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org