You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2021/08/20 19:44:47 UTC
[ozone] branch HDDS-4440-s3-performance updated (cf8eb33 -> 3b4e2f5)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a change to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from cf8eb33 HDDS-5210. Create GRPC service definition for existing OM protocol (#2375)
add 7792c40 HDDS-5428. Fix potential BigDecimal precision in StorageUnit.java (#2405)
add 7ca3e13 HDDS-5394. Fix skipped volume check due to disk.check.min.gap. (#2373)
add f3ec595 HDDS-5427. Either re-interrupt this method or rethrow the "InterruptedException" that can be caught here (#2404)
add de23d45 HDDS-5422. Avoid eager string formatting in preconditions (#2396)
add f44440b HDDS-5433. Rename hadooprpc proto compilation in pom to refer to proto2. (#2407)
add d8a5d9e HDDS-5426. Remove unused method parameter in XceiverClientGrpc.java (#2409)
add d76aee4 HDDS-5384 OM refreshPipeline should not invoke the expensive OmKeyLocationInfoGroup.getLocationList() (#2367)
add 154c0c2 HDDS-5386. Add a NSSummaryTask to write NSSummary info into RDB (#2389)
add 59e31c4 HDDS-3432. Enable TestBlockDeletion test cases. (#2393)
add c41d08f HDDS-5296. Replication Manager should not sent replicate command to DN which is not IN_SERVICE (#2377)
add 1414bc3 HDDS-5434. Fix potential InterruptedException in Ozone (#2413)
add 76384ca HDDS-5442. Datanode HTTP server fails to start in ozonesecure due to wrong keytab name (#2419)
add aa4f969 HDDS-5431. compile both proto2 and proto3 versions of Client to OM interaction proto files. (#2410)
add 2b3a4ee HDDS-5403 Support list node based on NodeOperationalState and NodeState options in datanode list CLI (#2397)
add e99a0eb HDDS-5337. Apply container space check to Ratis factor one pipelines (#2354)
add 1474e46 HDDS-5291. Handle SIGTERM to ensure clean shutdown of OM/DN/SCM (#2301)
add dccfbae HDDS-5392. Avoid catching Error while creating Ozone client (#2408)
add d404339 HDDS-5456. Inject a Clock into Replication Manager to allow timeouts to be tested (#2425)
add b83039e HDDS-5436. Add acceptance test for Hadoop 3.3 (#2428)
add 31b2e6a HDDS-5005. Multipart Upload fails due to partName mismatch. (#2411)
add 4ccc128 HDDS-4928. Support container move in Replication Manager (#2349)
add 9081612 HDDS-3740. PipelineStateManagerV2Impl#removePipeline will remove pipeline from db in case of failure (#2415)
add b3b7108 HDDS-5451. Incorrect PENDINGDELETEBLOCKCOUNT caused by resent command. (#2421)
add 24f5641 HDDS-5464. fail move if node is not in IN_SERVICE state (#2437)
add af8752b HDDS-5310. Enhance freon streaming generator to support multiple threads (#2306)
add 5e6f6fe HDDS-5457. OZONE-SITE.XML_ozone.om.enable.filesystem.paths must be a string, number or null (#2426)
add b4fa8d9 HDDS-5454. Modify Statement to PrepareStatement in DatabaseHelper (#2423)
add 7b8bf98 HDDS-5467. Handle potential InterruptedException in Ozone (#2440)
add b588694 HDDS-5459. Replication Manager should process containers synchronously for tests (#2429)
add 57d42b1 HDDS-5459. Replication Manager should process containers synchronously for tests (#2446)
add c550a84 HDDS-5468. Avoid long sleep in TestPeriodicVolumeChecker (#2445)
add 9129424 HDDS-5469. Bump Apache Ratis version to 2.1 (#2443)
add 37f1b90 HDDS-5479. s3g bucket list failed when there is non-english key name. (#2450)
add 2759a9f HDDS-5418. Let Recon send reregisterCommand to Datanodes if DatanodeDetails changed (#2392)
add 839fc94 HDDS-4330. Bootstrap new OM node (#1494)
add 12958d3 HDDS-5492. Disable failing kubernetes test (#2459)
add 39beb7e HDDS-5443 Create and then recreate a bucket with a randomized name (#2436)
add 3fbfc0e HDDS-5484. Intermittent failure in TestReplicationManager#testMovePrerequisites (#2454)
add d3fe588 Multi-raft style placement with permutations for offline data generator (#2434)
add ceb79ac HDDS-5489. Install OS-specific flekszible (#2462)
add 0236213 HDDS-5500. New k3s version breaks kubernetes test (#2464)
add 0439b16 HDDS-5450. Avoid refresh pipeline for S3 headObject (#2431)
add bd7219a HDDS-5414. Data buffers incorrectly filtered for Ozone Insight (#2387)
add 5336bb9 HDDS-5494. Reduce retry in Kubernetes test (#2461)
add 24ee206 HDDS-5184. Use separate DB profile for Datanodes. (#2214)
add 08e948b HDDS-5465. Delete redundant code when set、add and remove bucket acl (#2439)
add 59d6e95 HDDS-5466. Refactor BlockOutputStream. (#2442)
add cd8287b HDDS-5378 Add APIs to retrieve Namespace Summary from Recon (#2417)
add 074d8f4 HDDS-5382. Increase default container report interval to 60 mins (#2363)
add 5da8b22 HDDS-5328. Remove delete container command from admin CLI (#2456)
add 1c6e0fb HDDS-5279. OFS mkdir -p does not work when Volume is not pre-created (#2412)
add 210eb3b HDDS-5514. Skip check for UNHEALTHY containers for datanode finalize. (#2469)
add b447ffc HDDS-5401. Add more metrics to ReplicationManager to help monitor replication progress. (#2382)
add 622275f HDDS-5513. Race condition upon dn restart at prefinalization. (#2471)
add 99fa944 HDDS-5453. Add metrics in SCM block deleting service. (#2438)
add d557817 HDDS-5524. Use try-with-resources to properly close resources in DirectoryServerSource (#2481)
add 8739fd2 HDDS-5512. Intermittent failure in TestSCMInstallSnapshotWithHA (#2473)
add d0f0f80 HDDS-5501. Support to upload/read keys from encrypted buckets through S3G (#2467)
add 01c6165 HDDS-5511. ContainerStateMap should handle ecReplication config map (#2470)
add 7df1925 HDDS-5362. [FSO] Support bucket layouts in OM (#2357)
add 0b5e965 HDDS-5302. CleanUp V1 Code of DeletedBlockLogImpl (#2465)
add e2d1727 HDDS-5472. Old versions of location in OmKeyLocationInfoGroup causes OOM of OM (#2448)
add fd7e850 HDDS-5460: ReplicationConfig#getDefault is hardcoded with RatisReplicationConfig (#2430)
add ec031d47 HDDS-5515. SKip update firstUpgradeActionLayoutVersion when on-disk layout version is up-to-date. (#2472)
add cead31e HDDS-5530. Support links to encrypted buckets. (#2487)
add b3a2a3e HDDS-5441. Disallow same set of DNs to be part of multiple pipelines. (#2416)
add 30287c7 HDDS-5539. Fix actual value in assertion in TestRDBStore (#2494)
add 00a5afe HDDS-5354. Allow to restrict available ReplicationConfig (#2345)
add 4503249 HDDS-5532. Missing integration test cleanup (#2496)
add 181d44a HDDS-5527. Move tests back to root partition (#2486)
add 61396ba HDDS-4668. Intermittent failure in TestOMRatisSnapshots (#2474)
add 94b8c4b HDDS-5516. Duplicate metrics registered while running checkScmHA upon scm startup. (#2475)
add e23fb3c HDDS-5473. Bump jetty version to 9.4.43.v20210629 (#2449)
add bc1dfb8 HDDS-5538. Uncaught NPE in GrpcReplicationService. (#2503)
add 063e7fd HDDS-5390. reconPipelineReportHandler should not retry when pipeline not found (#2371)
add 2020d30 HDDS-5503. On finalize upgrade actions not running. (#2476)
add d51bd4b HDDS-5021. dev-support Dockerfile is badly outdated (#2480)
add f9ec7e9 HDDS-5360. DN failed to process all delete block commands in one heartbeat interval (#2420)
add 2edb375 HDDS-5546.OM Service ID change causes OM startup failure. (#2512)
add f922b0d HDDS-5555. remove pipeline manager v1 code (#2511)
add fa7dc30 HDDS-5549. Add 1.1 to supported versions in security policy (#2519)
add 7eed08f HDDS-5531. For Link Buckets avoid showing metadata. (#2502)
add 1c96a5b HDDS-5558. vUnit invocation unit() may produce NPE (#2513)
add 4625061 HDDS-5449 Recon namespace summary 'du' information should return replicated size of a key (#2489)
add dc9bb4e HDDS-5283. getStorageSize cast to int can cause issue (#2303)
add a956b49 HDDS-4929. Select target datanodes and containers to move for Container Balancer (#2441)
add ee929dc HDDS-5000. Run CI checks selectively (#2479)
add 43554e9 HDDS-5608. Fix wrong command in ugrade doc (#2524)
add 44be763 HDDS-5358. Incorrect cache entry invalidation causes intermittent failure in testGetS3SecretAndRevokeS3Secret (#2518)
add 1045ffc HDDS-5498. Containers are always quasi closed during finalization. (#2515)
add cef8bf5 HDDS-5508. Add documentation regarding access encrypted buckets from S3G (#2504)
add 66ee3b4 HDDS-5618. feature/_index.zh.md retranslation (#2535)
add 05c7a98 HDDS-5253. Support container move HA (#2488)
add c9c715c HDDS-5603. Stop ContainerBalancer when SCM is stopped. (#2523)
add 2b70185 HDDS-5533. Update Documentation for Trash (#2505)
add 375f3c4 HDDS-5483. Validate block file length during put block (#2463)
add 8b28e46 HDDS-5458. Extend s3 compatibility script, and fix the related errors (#2482)
add 8088cf8 HDDS-5601. Publish upgrade documentation on Ozone website. (#2527)
add bbf9acf HDDS-5054. Merge SCM HA configs to ScmConfigKeys. (#2506)
add d963991 HDDS-5624. Disable test TestPipelineClose.testPipelineCloseWithLogFailure (#2546)
add 75532de HDDS-5319. Ignore flaky tests in TestSCMUpdateServiceGrpcServer
add c654ed8 HDDS-5606. Intermittent failure in TestBlockDeletion#testContainerStatisticsAfterDelete (#2528)
add 800d9a9 HDDS-5175. ozone fs trash service does not handle bucket -rm -R (#2458)
add bf9fc53 HDDS-5622. Some improvements in SCMCommonPlacementPolicy (#2543)
add f4ef0e2 HDDS-5621. Support returning nodes of a level in NetworkTopology (#2541)
add b175d0b HDDS-5437. Move dead DN out of topology (#2435)
add 3accc98 HDDS-5616. Document how to run ozone sample cluster with FSO-enabled (#2539)
add a595d06 Revert "HDDS-5483. Validate block file length during put block (#2463)"
add 04816a6 HDDS-5620. Fix and extend hadoop.http.idle_timeout.ms to 60k (#2540)
add 18aad55 HDDS-5377. Add a Namespace Summary tab to Recon Web UI (#2530)
add db574d1 HDDS-5632. Intermittent failure in TestOzoneManagerBootstrap#testBootstrapTwoNewOMs (#2550)
new 3b4e2f5 Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 130 ++-
SECURITY.md | 1 +
dev-support/ci/lib/_all_libs.sh | 34 +
dev-support/ci/lib/_initialization.sh | 140 +++
dev-support/ci/lib/_script_init.sh | 42 +
dev-support/ci/lib/_start_end.sh | 112 +++
dev-support/ci/lib/_traps.sh | 38 +
dev-support/ci/lib/_verbosity.sh | 57 ++
dev-support/ci/selective_ci_checks.bats | 210 ++++
dev-support/ci/selective_ci_checks.sh | 522 ++++++++++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 72 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 19 +
.../hadoop/hdds/client/ReplicationConfig.java | 10 +-
.../hdds/client/ReplicationConfigValidator.java | 63 ++
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 80 ++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 ------
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +-
.../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 +
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 +
.../hadoop/hdds/scm/net/NetworkTopology.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 17 +
.../protocol/StorageContainerLocationProtocol.java | 11 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 5 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 10 +-
.../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++
.../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++
.../org/apache/hadoop/ozone/conf/package-info.java | 15 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++
.../org/apache/hadoop/ozone/util/package-info.java | 15 +-
.../main/resources/hdds-version-info.properties | 3 +-
.../common/src/main/resources/ozone-default.xml | 160 ++-
.../hadoop/hdds/client/TestReplicationConfig.java | 24 +
.../client/TestReplicationConfigValidator.java | 73 ++
.../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +-
.../ozone/container/ContainerTestHelper.java | 75 ++
.../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +-
.../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 +
.../hadoop/hdds/freon/FakeRatisFollower.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 13 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/report/ReportManager.java | 3 +-
.../common/statemachine/DatanodeConfiguration.java | 52 +-
.../common/statemachine/DatanodeStateMachine.java | 15 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 276 ++++--
.../common/transport/server/XceiverServerGrpc.java | 3 +-
.../server/ratis/ContainerStateMachine.java | 8 +-
.../transport/server/ratis/XceiverServerRatis.java | 20 +-
.../common/utils/db/DatanodeDBProfile.java | 145 +++
.../container/common/utils/db/package-info.java | 17 +-
.../common/volume/ImmutableVolumeSet.java | 16 +
.../container/common/volume/MutableVolumeSet.java | 25 +-
.../common/volume/StorageVolumeChecker.java | 54 +-
.../ozone/container/common/volume/VolumeSet.java | 3 +
.../container/metadata/AbstractDatanodeStore.java | 47 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 11 +-
.../replication/GrpcReplicationClient.java | 3 +-
.../OnDemandContainerReplicationSource.java | 10 +-
.../container/replication/ReplicationServer.java | 1 +
.../replication/ReplicationSupervisor.java | 33 +-
.../container/stream/DirectoryServerSource.java | 12 +-
.../ozone/container/stream/StreamingClient.java | 1 +
.../ozone/container/stream/StreamingServer.java | 1 +
.../upgrade/DataNodeUpgradeFinalizer.java | 4 +-
.../common/helpers/TestContainerUtils.java | 60 ++
.../common/volume/TestPeriodicVolumeChecker.java | 131 +++
.../container/keyvalue/TestKeyValueContainer.java | 77 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +-
.../replication/TestReplicationSupervisor.java | 4 +-
.../docs/content/feature/Nonrolling-Upgrade.md | 107 ++
hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/_index.zh.md | 14 +-
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 --
hadoop-hdds/docs/content/interface/Ofs.md | 28 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++
.../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +-
.../container/common/helpers/MoveDataNodePair.java | 71 ++
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 +
...inerLocationProtocolClientSideTranslatorPB.java | 30 +-
.../scm/update/client/CRLClientUpdateHandler.java | 5 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 3 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 2 +
.../hadoop/hdds/server/http/HttpServer2.java | 18 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +
.../hadoop/hdds/server/http/TestHttpServer2.java | 54 ++
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
hadoop-hdds/interface-admin/pom.xml | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 4 +-
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdds/interface-client/pom.xml | 38 +-
.../interface-client/src/main/proto/hdds.proto | 5 +
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 122 ++-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 27 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 281 +++---
.../hdds/scm/block/DeletedBlockLogImplV2.java | 432 ---------
.../hdds/scm/block/SCMBlockDeletingService.java | 18 +-
.../scm/block/ScmBlockDeletingServiceMetrics.java | 186 ++++
.../scm/command/CommandStatusReportHandler.java | 6 +-
.../hdds/scm/container/ContainerReplicaCount.java | 1 +
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../hdds/scm/container/ReplicationManager.java | 953 +++++++++++++++---
.../scm/container/balancer/ContainerBalancer.java | 637 +++++++++---
.../balancer/ContainerBalancerConfiguration.java | 206 +++-
.../balancer/ContainerBalancerMetrics.java | 13 +-
.../ContainerBalancerSelectionCriteria.java | 169 ++++
.../container/balancer/ContainerMoveSelection.java | 55 ++
.../scm/container/balancer/FindTargetGreedy.java | 135 +++
.../scm/container/balancer/FindTargetStrategy.java | 71 ++
.../algorithms/SCMContainerPlacementCapacity.java | 10 +-
.../algorithms/SCMContainerPlacementRackAware.java | 86 +-
.../algorithms/SCMContainerPlacementRandom.java | 10 +-
.../replication/ReplicationManagerMetrics.java | 161 ++++
.../scm/container/states/ContainerQueryKey.java | 33 +-
.../scm/container/states/ContainerStateMap.java | 41 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 +
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 10 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 131 ++-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +-
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 +
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 9 +
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 5 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 27 +
...ManagerV2Impl.java => PipelineManagerImpl.java} | 22 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 57 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 2 +
.../scm/pipeline/PipelineStateManagerV2Impl.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 801 ---------------
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 40 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 56 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 1 +
.../hdds/scm/server/StorageContainerManager.java | 32 +-
.../scm/server/StorageContainerManagerStarter.java | 12 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 7 +-
.../ScmHAUnfinalizedStateValidationAction.java | 13 +-
.../update/server/SCMUpdateServiceGrpcServer.java | 3 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 67 +-
.../container/TestCloseContainerEventHandler.java | 6 +-
.../hdds/scm/container/TestReplicationManager.java | 843 +++++++++++++---
.../scm/container/TestSCMContainerManager.java | 6 +-
.../container/balancer/TestContainerBalancer.java | 466 ++++++++-
.../algorithms/TestContainerPlacementFactory.java | 39 +-
.../TestSCMContainerPlacementCapacity.java | 2 +-
.../TestSCMContainerPlacementRackAware.java | 109 ++-
.../TestSCMContainerPlacementRandom.java | 70 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 -
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 14 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 44 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 ----------------
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 +
.../safemode/TestHealthyPipelineSafeModeRule.java | 14 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 3 +
.../ozone/container/common/TestEndPoint.java | 3 +-
.../placement/TestContainerPlacement.java | 5 +-
.../main/java/org/apache/ozone/test/TestClock.java | 76 ++
.../hdds/scm/cli/ContainerBalancerCommands.java | 19 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 16 +-
.../hdds/scm/cli/ContainerOperationClient.java | 26 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +-
.../hdds/scm/cli/container/ContainerCommands.java | 1 -
.../hdds/scm/cli/container/DeleteSubcommand.java | 54 --
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 61 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 4 +
.../ozone/client/io/BlockOutputStreamEntry.java | 3 +-
.../ozone/client/protocol/ClientProtocol.java | 16 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 54 +-
hadoop-ozone/common/pom.xml | 4 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../ozone/om/ha/OMFailoverProxyProvider.java | 55 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 62 ++
.../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 56 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 1 -
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 6 +-
.../ozone/om/protocol/OMInterServiceProtocol.java} | 28 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +-
.../OMInterServiceProtocolClientSideImpl.java | 121 +++
.../om/protocolPB/OMInterServiceProtocolPB.java | 52 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 1 +
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 5 +-
.../main/resources/ozone-version-info.properties | 3 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 22 +-
hadoop-ozone/dev-support/checks/_lib.sh | 9 +-
hadoop-ozone/dev-support/docker/Dockerfile | 38 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +-
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../{ozonesecure-mr => ozone-mr/hadoop33}/.env | 7 +-
.../{hadoop32 => hadoop33}/docker-compose.yaml | 0
.../ozone-mr/{hadoop32 => hadoop33}/docker-config | 0
.../ozone-mr/{hadoop32 => hadoop33}/test.sh | 0
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 +
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 +
.../src/main/compose/ozonesecure-ha/docker-config | 4 +
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../src/main/compose/ozonesecure/docker-config | 6 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../src/main/smoketest/admincli/container.robot | 4 -
.../src/main/smoketest/admincli/datanode.robot | 18 +
.../dist/src/main/smoketest/basic/links.robot | 10 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +-
.../dist/src/main/smoketest/s3/boto3.robot | 5 +-
.../dist/src/main/smoketest/s3/boto_client.py | 10 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 6 +-
.../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +-
.../dist/src/main/smoketest/s3/buckethead.robot | 3 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 10 +-
.../src/main/smoketest/s3/s3_compatbility_check.sh | 39 +
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 3 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 55 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 30 +-
.../ozone/contract/ITestOzoneContractDistCp.java | 8 +
.../rooted/ITestRootedOzoneContractDistCp.java | 8 +
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +-
.../TestContainerStateManagerIntegration.java | 40 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 ++++
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 5 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 235 ++++-
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 5 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../ozone/TestContainerBalancerOperations.java | 11 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 40 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../hadoop/ozone/TestStorageContainerManager.java | 12 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 13 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 13 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 17 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 130 ++-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestWatchForCommit.java | 3 +-
.../ozone/container/TestContainerReplication.java | 82 +-
.../apache/hadoop/ozone/container/TestHelper.java | 2 +-
.../commandhandler/TestBlockDeletion.java | 369 ++++---
.../server/TestSecureContainerServer.java | 6 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +-
.../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 ----
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 76 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 21 +-
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 +++++
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 166 ++++
.../hadoop/ozone/scm/TestCloseContainer.java | 8 +
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 19 +-
.../scm/node/TestDecommissionAndMaintenance.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 9 +-
.../dev-support/findbugsExcludeFile.xml | 6 +
hadoop-ozone/interface-client/pom.xml | 66 +-
.../src/main/proto/OmClientProtocol.proto | 10 +
.../src/main/proto/OmInterServiceProtocol.proto | 64 ++
hadoop-ozone/interface-storage/pom.xml | 2 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 7 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 45 +-
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 +
.../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 416 ++++++--
.../hadoop/ozone/om/OzoneManagerStarter.java | 47 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 13 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 383 +++++---
.../ozone/om/ratis/OzoneManagerStateMachine.java | 30 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 88 ++
.../om/request/bucket/OMBucketCreateRequest.java | 102 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 7 -
.../bucket/acl/OMBucketRemoveAclRequest.java | 7 -
.../request/bucket/acl/OMBucketSetAclRequest.java | 7 -
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +-
.../S3MultipartUploadCommitPartRequest.java | 11 +-
.../request/s3/security/S3RevokeSecretRequest.java | 2 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +-
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +-
.../OMInterServiceProtocolServerSideImpl.java | 90 ++
...OzoneManagerProtocolServerSideTranslatorPB.java | 76 +-
.../protocolPB/OzoneManagerRequestHandler.java | 8 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 6 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +-
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 4 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 40 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 9 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 11 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 +
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 1019 ++++++++++++++++++++
.../hadoop/ozone/recon/api/NodeEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/types/DUResponse.java | 177 ++++
.../hadoop/ozone/recon/api/types/EntityType.java | 16 +-
.../api/types/FileSizeDistributionResponse.java | 61 ++
.../hadoop/ozone/recon/api/types/NSSummary.java | 57 +-
.../recon/api/types/NamespaceSummaryResponse.java | 106 ++
.../ozone/recon/api/types/QuotaUsageResponse.java | 69 ++
.../ozone/recon/api/types/ResponseStatus.java | 13 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 56 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 106 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../recon/scm/ReconPipelineReportHandler.java | 22 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 4 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 7 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 24 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 302 ++++++
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 +
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 2 +-
.../webapps/recon/ozone-recon-web/api/db.json | 324 ++++++-
.../webapps/recon/ozone-recon-web/api/routes.json | 22 +-
.../src/components/navBar/navBar.tsx | 5 +
.../src/components/rightDrawer/rightDrawer.tsx | 72 ++
.../src/constants/breadcrumbs.constants.tsx | 3 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 5 +
.../src/views/datanodes/datanodes.tsx | 8 +-
.../diskUsage/diskUsage.less} | 41 +-
.../src/views/diskUsage/diskUsage.tsx | 439 +++++++++
.../src/views/overview/overview.tsx | 3 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 109 +++
.../ozone/recon/api/TestNSSummaryEndpoint.java | 648 +++++++++++++
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../ozone/recon/scm/TestReconNodeManager.java | 38 +
.../impl/TestReconNamespaceSummaryManagerImpl.java | 22 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 17 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 519 ++++++++++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 40 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 4 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 7 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 20 +
.../hadoop/ozone/s3/util/TestContinueToken.java | 20 +
.../ozone/audit/parser/common/DatabaseHelper.java | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../freon/FollowerAppendLogEntryGenerator.java | 4 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 70 +-
.../containergenerator/GeneratorDatanode.java | 60 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 6 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 20 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 52 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +-
.../containergenerator/TestGeneratorDatanode.java | 71 ++
pom.xml | 19 +-
405 files changed, 17021 insertions(+), 5667 deletions(-)
create mode 100755 dev-support/ci/lib/_all_libs.sh
create mode 100644 dev-support/ci/lib/_initialization.sh
create mode 100755 dev-support/ci/lib/_script_init.sh
create mode 100644 dev-support/ci/lib/_start_end.sh
create mode 100644 dev-support/ci/lib/_traps.sh
create mode 100644 dev-support/ci/lib/_verbosity.sh
create mode 100644 dev-support/ci/selective_ci_checks.bats
create mode 100755 dev-support/ci/selective_ci_checks.sh
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/RatisBlockOutputStream.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/MonotonicClock.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/OzoneServiceConfig.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java (74%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/ShutdownHookManager.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/package-info.java (74%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfigValidator.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/db/DatanodeDBProfile.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/db/package-info.java (74%)
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestPeriodicVolumeChecker.java
create mode 100644 hadoop-hdds/docs/content/feature/Nonrolling-Upgrade.md
delete mode 100644 hadoop-hdds/docs/content/feature/how-to-do-a-nonrolling-upgrade.md
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpServer2.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImplV2.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/ScmBlockDeletingServiceMetrics.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveSelection.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerMetrics.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/MoveDataNodePairCodec.java
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/{PipelineManagerV2Impl.java => PipelineManagerImpl.java} (96%)
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
create mode 100644 hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/TestClock.java
delete mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/DeleteSubcommand.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
rename hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha => common/src/main/java/org/apache/hadoop/ozone/om/helpers}/OMNodeDetails.java (54%)
copy hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java => common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMInterServiceProtocol.java} (61%)
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMInterServiceProtocolClientSideImpl.java
copy hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto => hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMInterServiceProtocolPB.java (52%)
copy hadoop-ozone/dist/src/main/compose/{ozonesecure-mr => ozone-mr/hadoop33}/.env (84%)
copy hadoop-ozone/dist/src/main/compose/ozone-mr/{hadoop32 => hadoop33}/docker-compose.yaml (100%)
copy hadoop-ozone/dist/src/main/compose/ozone-mr/{hadoop32 => hadoop33}/docker-config (100%)
copy hadoop-ozone/dist/src/main/compose/ozone-mr/{hadoop32 => hadoop33}/test.sh (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestMultiRaftSetup.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithLayout.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerBootstrap.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java
create mode 100644 hadoop-ozone/interface-client/src/main/proto/OmInterServiceProtocol.proto
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMInterServiceProtocolServerSideImpl.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DUResponse.java
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/EntityType.java (76%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/FileSizeDistributionResponse.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/QuotaUsageResponse.java
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ResponseStatus.java (76%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/rightDrawer/rightDrawer.tsx
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{constants/breadcrumbs.constants.tsx => views/diskUsage/diskUsage.less} (63%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/containergenerator/TestGeneratorDatanode.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'origin/master' into
HDDS-4440-s3-performance
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 3b4e2f58b35f1e2dbf553ea183063867f1fd4771
Merge: cf8eb33 db574d1
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Fri Aug 20 07:36:19 2021 +0200
Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
.github/workflows/post-commit.yml | 130 ++-
SECURITY.md | 1 +
dev-support/ci/lib/_all_libs.sh | 34 +
dev-support/ci/lib/_initialization.sh | 140 +++
dev-support/ci/lib/_script_init.sh | 42 +
dev-support/ci/lib/_start_end.sh | 112 +++
dev-support/ci/lib/_traps.sh | 38 +
dev-support/ci/lib/_verbosity.sh | 57 ++
dev-support/ci/selective_ci_checks.bats | 210 ++++
dev-support/ci/selective_ci_checks.sh | 522 ++++++++++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 72 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 19 +
.../hadoop/hdds/client/ReplicationConfig.java | 10 +-
.../hdds/client/ReplicationConfigValidator.java | 63 ++
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 80 ++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 ------
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +-
.../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 +
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 +
.../hadoop/hdds/scm/net/NetworkTopology.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 17 +
.../protocol/StorageContainerLocationProtocol.java | 11 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 5 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 10 +-
.../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++
.../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++
.../org/apache/hadoop/ozone/conf/package-info.java | 15 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++
.../org/apache/hadoop/ozone/util/package-info.java | 15 +-
.../main/resources/hdds-version-info.properties | 3 +-
.../common/src/main/resources/ozone-default.xml | 160 ++-
.../hadoop/hdds/client/TestReplicationConfig.java | 24 +
.../client/TestReplicationConfigValidator.java | 73 ++
.../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +-
.../ozone/container/ContainerTestHelper.java | 75 ++
.../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +-
.../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 +
.../hadoop/hdds/freon/FakeRatisFollower.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 13 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/report/ReportManager.java | 3 +-
.../common/statemachine/DatanodeConfiguration.java | 52 +-
.../common/statemachine/DatanodeStateMachine.java | 15 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 276 ++++--
.../common/transport/server/XceiverServerGrpc.java | 3 +-
.../server/ratis/ContainerStateMachine.java | 8 +-
.../transport/server/ratis/XceiverServerRatis.java | 20 +-
.../common/utils/db/DatanodeDBProfile.java | 145 +++
.../container/common/utils/db/package-info.java | 17 +-
.../common/volume/ImmutableVolumeSet.java | 16 +
.../container/common/volume/MutableVolumeSet.java | 25 +-
.../common/volume/StorageVolumeChecker.java | 54 +-
.../ozone/container/common/volume/VolumeSet.java | 3 +
.../container/metadata/AbstractDatanodeStore.java | 47 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 11 +-
.../replication/GrpcReplicationClient.java | 3 +-
.../OnDemandContainerReplicationSource.java | 10 +-
.../container/replication/ReplicationServer.java | 1 +
.../replication/ReplicationSupervisor.java | 33 +-
.../container/stream/DirectoryServerSource.java | 12 +-
.../ozone/container/stream/StreamingClient.java | 1 +
.../ozone/container/stream/StreamingServer.java | 1 +
.../upgrade/DataNodeUpgradeFinalizer.java | 4 +-
.../common/helpers/TestContainerUtils.java | 60 ++
.../common/volume/TestPeriodicVolumeChecker.java | 131 +++
.../container/keyvalue/TestKeyValueContainer.java | 77 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +-
.../replication/TestReplicationSupervisor.java | 4 +-
.../docs/content/feature/Nonrolling-Upgrade.md | 107 ++
hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/_index.zh.md | 14 +-
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 --
hadoop-hdds/docs/content/interface/Ofs.md | 28 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++
.../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +-
.../container/common/helpers/MoveDataNodePair.java | 71 ++
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 +
...inerLocationProtocolClientSideTranslatorPB.java | 30 +-
.../scm/update/client/CRLClientUpdateHandler.java | 5 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 3 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 2 +
.../hadoop/hdds/server/http/HttpServer2.java | 18 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +
.../hadoop/hdds/server/http/TestHttpServer2.java | 54 ++
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
hadoop-hdds/interface-admin/pom.xml | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 4 +-
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdds/interface-client/pom.xml | 38 +-
.../interface-client/src/main/proto/hdds.proto | 5 +
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 122 ++-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 27 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 281 +++---
.../hdds/scm/block/DeletedBlockLogImplV2.java | 432 ---------
.../hdds/scm/block/SCMBlockDeletingService.java | 18 +-
.../scm/block/ScmBlockDeletingServiceMetrics.java | 186 ++++
.../scm/command/CommandStatusReportHandler.java | 6 +-
.../hdds/scm/container/ContainerReplicaCount.java | 1 +
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../hdds/scm/container/ReplicationManager.java | 953 +++++++++++++++---
.../scm/container/balancer/ContainerBalancer.java | 637 +++++++++---
.../balancer/ContainerBalancerConfiguration.java | 206 +++-
.../balancer/ContainerBalancerMetrics.java | 13 +-
.../ContainerBalancerSelectionCriteria.java | 169 ++++
.../container/balancer/ContainerMoveSelection.java | 55 ++
.../scm/container/balancer/FindTargetGreedy.java | 135 +++
.../scm/container/balancer/FindTargetStrategy.java | 71 ++
.../algorithms/SCMContainerPlacementCapacity.java | 10 +-
.../algorithms/SCMContainerPlacementRackAware.java | 86 +-
.../algorithms/SCMContainerPlacementRandom.java | 10 +-
.../replication/ReplicationManagerMetrics.java | 161 ++++
.../scm/container/states/ContainerQueryKey.java | 33 +-
.../scm/container/states/ContainerStateMap.java | 41 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 +
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 10 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 131 ++-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +-
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 +
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 9 +
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 5 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 27 +
...ManagerV2Impl.java => PipelineManagerImpl.java} | 22 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 57 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 2 +
.../scm/pipeline/PipelineStateManagerV2Impl.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 801 ---------------
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 40 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 56 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 1 +
.../hdds/scm/server/StorageContainerManager.java | 32 +-
.../scm/server/StorageContainerManagerStarter.java | 12 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 7 +-
.../ScmHAUnfinalizedStateValidationAction.java | 13 +-
.../update/server/SCMUpdateServiceGrpcServer.java | 3 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 67 +-
.../container/TestCloseContainerEventHandler.java | 6 +-
.../hdds/scm/container/TestReplicationManager.java | 843 +++++++++++++---
.../scm/container/TestSCMContainerManager.java | 6 +-
.../container/balancer/TestContainerBalancer.java | 466 ++++++++-
.../algorithms/TestContainerPlacementFactory.java | 39 +-
.../TestSCMContainerPlacementCapacity.java | 2 +-
.../TestSCMContainerPlacementRackAware.java | 109 ++-
.../TestSCMContainerPlacementRandom.java | 70 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 -
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 14 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 44 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 ----------------
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 +
.../safemode/TestHealthyPipelineSafeModeRule.java | 14 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 3 +
.../ozone/container/common/TestEndPoint.java | 3 +-
.../placement/TestContainerPlacement.java | 5 +-
.../main/java/org/apache/ozone/test/TestClock.java | 76 ++
.../hdds/scm/cli/ContainerBalancerCommands.java | 19 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 16 +-
.../hdds/scm/cli/ContainerOperationClient.java | 26 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +-
.../hdds/scm/cli/container/ContainerCommands.java | 1 -
.../hdds/scm/cli/container/DeleteSubcommand.java | 54 --
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 61 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 4 +
.../ozone/client/io/BlockOutputStreamEntry.java | 3 +-
.../ozone/client/protocol/ClientProtocol.java | 16 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 54 +-
hadoop-ozone/common/pom.xml | 4 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../ozone/om/ha/OMFailoverProxyProvider.java | 55 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 62 ++
.../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 56 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 1 -
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 16 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 6 +-
.../ozone/om/protocol/OMInterServiceProtocol.java} | 28 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +-
.../OMInterServiceProtocolClientSideImpl.java | 121 +++
.../om/protocolPB/OMInterServiceProtocolPB.java | 52 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 1 +
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 5 +-
.../main/resources/ozone-version-info.properties | 3 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 22 +-
hadoop-ozone/dev-support/checks/_lib.sh | 9 +-
hadoop-ozone/dev-support/docker/Dockerfile | 38 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +-
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../{ozonesecure-mr => ozone-mr/hadoop33}/.env | 7 +-
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 +++
.../run.sh => ozone-mr/hadoop33/docker-config} | 11 +-
.../{ozone/run.sh => ozone-mr/hadoop33/test.sh} | 26 +-
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 +
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 +
.../src/main/compose/ozonesecure-ha/docker-config | 4 +
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../src/main/compose/ozonesecure/docker-config | 6 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../src/main/smoketest/admincli/container.robot | 4 -
.../src/main/smoketest/admincli/datanode.robot | 18 +
.../dist/src/main/smoketest/basic/links.robot | 10 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +-
.../dist/src/main/smoketest/s3/boto3.robot | 5 +-
.../dist/src/main/smoketest/s3/boto_client.py | 10 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 6 +-
.../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +-
.../dist/src/main/smoketest/s3/buckethead.robot | 3 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 10 +-
.../src/main/smoketest/s3/s3_compatbility_check.sh | 39 +
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 3 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 55 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 30 +-
.../ozone/contract/ITestOzoneContractDistCp.java | 8 +
.../rooted/ITestRootedOzoneContractDistCp.java | 8 +
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +-
.../TestContainerStateManagerIntegration.java | 40 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 ++++
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 5 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 235 ++++-
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 5 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../ozone/TestContainerBalancerOperations.java | 11 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 40 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../hadoop/ozone/TestStorageContainerManager.java | 12 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 13 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 13 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 17 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 130 ++-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestWatchForCommit.java | 3 +-
.../ozone/container/TestContainerReplication.java | 82 +-
.../apache/hadoop/ozone/container/TestHelper.java | 2 +-
.../commandhandler/TestBlockDeletion.java | 369 ++++---
.../server/TestSecureContainerServer.java | 6 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +-
.../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 ----
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 76 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 21 +-
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 +++++
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 166 ++++
.../hadoop/ozone/scm/TestCloseContainer.java | 8 +
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 19 +-
.../scm/node/TestDecommissionAndMaintenance.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 9 +-
.../dev-support/findbugsExcludeFile.xml | 6 +
hadoop-ozone/interface-client/pom.xml | 66 +-
.../src/main/proto/OmClientProtocol.proto | 10 +
.../src/main/proto/OmInterServiceProtocol.proto | 64 ++
hadoop-ozone/interface-storage/pom.xml | 2 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 7 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 45 +-
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 +
.../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 416 ++++++--
.../hadoop/ozone/om/OzoneManagerStarter.java | 47 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 13 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 383 +++++---
.../ozone/om/ratis/OzoneManagerStateMachine.java | 30 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 88 ++
.../om/request/bucket/OMBucketCreateRequest.java | 102 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 7 -
.../bucket/acl/OMBucketRemoveAclRequest.java | 7 -
.../request/bucket/acl/OMBucketSetAclRequest.java | 7 -
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +-
.../S3MultipartUploadCommitPartRequest.java | 11 +-
.../request/s3/security/S3RevokeSecretRequest.java | 2 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +-
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +-
.../OMInterServiceProtocolServerSideImpl.java | 90 ++
...OzoneManagerProtocolServerSideTranslatorPB.java | 76 +-
.../protocolPB/OzoneManagerRequestHandler.java | 8 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 6 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +-
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 4 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 40 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 9 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 11 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 +
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 1019 ++++++++++++++++++++
.../hadoop/ozone/recon/api/NodeEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/types/DUResponse.java | 177 ++++
.../hadoop/ozone/recon/api/types/EntityType.java | 16 +-
.../api/types/FileSizeDistributionResponse.java | 61 ++
.../hadoop/ozone/recon/api/types/NSSummary.java | 57 +-
.../recon/api/types/NamespaceSummaryResponse.java | 106 ++
.../ozone/recon/api/types/QuotaUsageResponse.java | 69 ++
.../ozone/recon/api/types/ResponseStatus.java | 13 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 56 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 106 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../recon/scm/ReconPipelineReportHandler.java | 22 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 4 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 7 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 24 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 302 ++++++
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 +
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 2 +-
.../webapps/recon/ozone-recon-web/api/db.json | 324 ++++++-
.../webapps/recon/ozone-recon-web/api/routes.json | 22 +-
.../src/components/navBar/navBar.tsx | 5 +
.../src/components/rightDrawer/rightDrawer.tsx | 72 ++
.../src/constants/breadcrumbs.constants.tsx | 3 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 5 +
.../src/views/datanodes/datanodes.tsx | 8 +-
.../diskUsage/diskUsage.less} | 41 +-
.../src/views/diskUsage/diskUsage.tsx | 439 +++++++++
.../src/views/overview/overview.tsx | 3 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 109 +++
.../ozone/recon/api/TestNSSummaryEndpoint.java | 648 +++++++++++++
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../ozone/recon/scm/TestReconNodeManager.java | 38 +
.../impl/TestReconNamespaceSummaryManagerImpl.java | 22 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 17 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 519 ++++++++++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 40 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 4 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 7 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 20 +
.../hadoop/ozone/s3/util/TestContinueToken.java | 20 +
.../ozone/audit/parser/common/DatabaseHelper.java | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../freon/FollowerAppendLogEntryGenerator.java | 4 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 70 +-
.../containergenerator/GeneratorDatanode.java | 60 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 6 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 20 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 52 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +-
.../containergenerator/TestGeneratorDatanode.java | 71 ++
pom.xml | 19 +-
405 files changed, 17159 insertions(+), 5676 deletions(-)
diff --cc hadoop-ozone/interface-client/pom.xml
index 1695e4e,743dcb0..e6325ca
--- a/hadoop-ozone/interface-client/pom.xml
+++ b/hadoop-ozone/interface-client/pom.xml
@@@ -73,26 -55,74 +73,82 @@@ https://maven.apache.org/xsd/maven-4.0.
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
+ <goal>compile-custom</goal>
+ <goal>test-compile-custom</goal>
</goals>
<configuration>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
<protocArtifact>
- com.google.protobuf:protoc:${hadooprpc.protobuf.version}:exe:${os.detected.classifier}
+ com.google.protobuf:protoc:${proto2.hadooprpc.protobuf.version}:exe:${os.detected.classifier}
</protocArtifact>
+ <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
- <includes>
- <include>OmClientProtocol.proto</include>
- <include>Security.proto</include>
- </includes>
+ <outputDirectory>target/generated-sources/protobuf/java</outputDirectory>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>
+ io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier}
+ </pluginArtifact>
</configuration>
</execution>
+ <execution>
+ <id>compile-protoc3</id>
+ <goals>
+ <goal>compile</goal>
+ <goal>test-compile</goal>
+ </goals>
+ <configuration>
+ <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+ <protocArtifact>
+ com.google.protobuf:protoc:${proto3.hadooprpc.protobuf.version}:exe:${os.detected.classifier}
+ </protocArtifact>
+ <outputDirectory>target/generated-sources/protobuf/java/proto3</outputDirectory>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <configuration>
+ <tasks>
+ <replace token="com.google.protobuf"
+ value="org.apache.hadoop.thirdparty.protobuf"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <replace token="com.google.protobuf"
+ value="org.apache.hadoop.thirdparty.protobuf"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto">
+ </replace>
+ <replace token="org.apache.hadoop.ozone.protocol.proto"
+ value="org.apache.hadoop.ozone.protocol.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <replace token="org.apache.hadoop.ozone.security.proto"
+ value="org.apache.hadoop.ozone.security.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto">
+ </replace>
+ <replace token="org.apache.hadoop.hdds.protocol.proto"
+ value="org.apache.hadoop.hdds.protocol.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <replace token="org.apache.hadoop.ozone.security.proto"
+ value="org.apache.hadoop.ozone.security.proto3"
+ dir="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto">
+ </replace>
+ <move file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto"
+ tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/protocol/proto3"/>
+ <move file="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto"
+ tofile="target/generated-sources/protobuf/java/proto3/org/apache/hadoop/ozone/security/proto3"/>
+ <move file="target/generated-sources/protobuf/java/proto3"
+ tofile="target/generated-sources/protobuf/java/"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
<plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org