You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2021/07/30 12:45:10 UTC
[ozone] branch HDDS-3816-ec updated (cd781b9 -> 0c5c859)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from cd781b9 HDDS-5174. EC: Allow EC blocks to be requested from OM (#2414)
add 3e21d75 HDDS-3829. Introduce Layout Feature interface in Ozone. (#1322)
add 3fe83ce HDDS-4173. Implement HDDS Version management using the LayoutVersionManager interface. (#1392)
add 9b55f69 HDDS-4141. Implement Finalize command in Ozone Manager client. (#1400)
add 95b8712 HDDS-4174. Add current HDDS layout version to Datanode heartbeat/registration (#1421)
add 6fa5fa1 HDDS-4143. Implement a factory for OM Requests that returns an instance based on layout version. (#1405)
add b96834f HDDS-4252. Add the current layout versions to DN - SCM proto payload. (#1432)
add 08d8d36 HDDS-4227. Implement a 'Prepare For Upgrade' step in OM that applies all committed Ratis transactions. (#1430)
add 8fe8a1f HDDS-4253. Add LayoutVersion request/response for DN registration. (#1457)
add 515c619 HDDS-4319. Compile error with Java 11 (#1483)
add bfce8b9 HDDS-4172. Implement Finalize command in Ozone Manager server. (#1456)
add 0d014c0 HDDS-4314. OM Layout Version Manager init throws silent CNF error in integration tests. (#1492)
add b8dcf6b Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
add d4212a7 Fix copilation issue after mergee
add 5186dec HDDS-4401. Fix compilation issue in HDDS-3698-upgrade branch. (#1529)
add 8e29229 HDDS-4296. SCM changes to process Layout Info in heartbeat request/response (#1486)
add 18bef92 HDDS-4142. Expose upgrade related state through JMX. (#1534)
add 28ecd8d Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
add 7a26250 HDDS-4342. Add DataNode state and transitions for a node going through upgrade. (#1508)
add 4384789 Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
add bf7624b Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
add 6b9ed19 Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
add ad9d019 HDDS-4536. Add more unit tests for OM layout version manager. (#1646)
add 9c4f2b4 HDDS-4463. Verify that OM/SCM start fails when Software Layout Version < Metadata Layout Version. (#1645)
add c3e3b35 HDDS-4179. Implement post-finalize SCM logic. (#1611)
add a7797aa HDDS-4480. Implement OM Prepare Request/Response. (#1613)
add 2489968 HDDS-4540. Add a new OM admin operation to submit the OMPrepareRequest. (#1664)
add 6519142 HDDS-4178. SCM Finalize client command implementation (#1651)
add bdd00a7 HDDS-4564. Prepare client should check every OM individually for the prepared check based on Txn ID. (#1692)
add 17a126d HDDS-4569. Add pre append gate and marker file to OM prepare state. (#1705)
add 72c215c HDDS-4612. Create OMCancelPrepareRequest and Response to cancel the prepared state of an OM. (#1765)
add d36b87d HDDS-4175. Implement Datanode Finalization. (#1720)
add 81c1dee HDDS-4219. Revisit 'static' nature of OM Layout Version Manager. (#1773)
add 1de63f4 HDDS-4645. Starting OM with the --upgrade flag should delete the prepare marker file. (#1798)
add a81b36e HDDS-4614. Add Integration test for HDDS upgrade (happy path cases). (#1726)
add ee8d276 HDDS-4611. Add an admin command to cancel "preparation" of an OM quorum. (#1830)
add 1e187f7 Merge remote-tracking branch 'upstream/master' into upgrade-branch-merge-candidate
add 7ae013e Fix spotbugs issues.
add ba4aeca trigger new CI check
add dd14670 trigger new CI check
add 5720ebf HDDS-4182. Onboard HDDS-3869 into Layout version management. (#1913)
add 277cc0b HDDS-4610. Fix issues in 'prepare' operation with one OM down. (#1836)
add 9a078c4 Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade
add 8a9be0d trigger new CI check
add 447024b HDDS-4817. Fresh deploy of Ozone must use the highest layout version by default. (#1933)
add 485ec25 HDDS-4902 Add Layout version information to Recon datanode info API. (#1993)
add df57ebc HDDS-4946. Do not wait one heartbeat to move newly registered datanodes that match SCM's MLV from HEALTHY_READONLY to HEALTHY (#2015)
add 7eefee7 Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade-merge-candidate
add f6a302a HDDS-4908. Layout version should be available in DB for an un-finalized OM to be finalized through a Ratis snapshot. (#2017)
add f6ecb1d Merge remote-tracking branch 'ssh-upstream/master' into HDDS-3698-nonrolling-upgrade
add d55a714 HDDS-5023. Datanodes should always use MLV 0 when no VERSION file is present. (#2080)
add ed00db7 HDDS-4990. Introduce First upgrade startup action and Pre-finalized state validation in Layout Feature. (#2040)
add 2f96d94 Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade
add 1730b0d Allow layout version to be specified through config on --init (for MiniOzone cluster testing).
add de3d735 Fix SCM admin command issues after merge.
add 666f33b Ignore failing upgrade integration test.
add 8f3b5ae Fix rat failure.
add dfc4d99 trigger new CI check
add f16092e Intialize DN layout version before security init.
add 805d93d Checkstyle fix.
add 32d0834 Merge remote-tracking branch 'upstream/master' into HDDS-5046-2
add 9e2e43f Merge remote-tracking branch 'upstream/master' into HDDS-5046-2
add 9363434 Fix ozone-ha acceptance test.
add 3ea88e1 HDDS-5014. Add non-rolling upgrade design docs.
add a55f657 Merge remote-tracking branch 'upstream/master' into ssh-upstream-upgrade-branch
add baeb46d Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
add c4b7e7b HDDS-4828. SCM should go into "safe mode" until there is at least 1 pipeline to work with after finalization. (#2101)
add 5c6c858 HDDS-4181. Add acceptance tests for upgrade, finalization and downgrade. (#2056)
add 6e027a1 HDDS-5014. Upgrade usage primer documentation. (#2133)
add e8e7ced HDDS-5014. Move upgrade user flow to 'feature' folder.
add 7266f32 HDDS-4914. Failure injection and validating HDDS upgrade. (#1998)
add 58a87fb HDDS-5086. Add pre-finalize validation action for SCM HA. (#2143)
add d2da5a4 HDDS-4992. SCM should not use pipelines with HEALTHY_READONLY datanodes (#2142)
add 3221362 Update upgrade-dev-primer.md
add 2823320 HDDS-5108. Attempt to remove state from *UpgradeFinalizer classes. (#2160)
add 40223ec HDDS-4818. Add command to get status of Finalize in OM & SCM (#2083)
add 5927bd2 HDDS-5118. Recover from failure during upgrade action. (#2179)
add 9168e07 HDDS-4959. NoSuchMethodException when wrapping RpcException on downgrade (#2178)
add bef4aed HDDS-5137. Update licenses for non-rolling upgrade new dependencies. (#2201)
add cdb1471 HDDS-5170. Race condition in NodestateManager#addNode allows datanodes with lower MLV to be used in pipelines. (#2204)
add 253cc55 Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade-master-merge2
add 1686e56 Fix compilation errors afte merge Update javassist in recon pom Fix changes introduced in merge that failed TestSCMNodeManager upgrade tests Fix checkstyle Fix intermittent test failure TestSCMNodeManager#testSetNodeOpStateAndCommandFired after merge Skip scm init default layout version in TestOzoneConfigurationFields
add ea239ee HDDS-5226. Do not fail SCM HA pre-finalize validation if SCM HA was already being used. (#2257)
add 5f046fa HDDS-5138. Upgrade related RPC calls should be allowed only for admins. (#2217)
add 92b5106 HDDS-5244. Allow multiple OM request versions to be supported at same layout version. (#2265)
add 1d18f47 Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade-master-merge
add a1b50ad HDDS-5321. Merge master branch at 12e2918 into upgrade branch (#2316)
add 2c0adac HDDS-5109. Track OM prepare intermittent integration test failure. (#2288)
add 1290b87 Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
add 4c8b62f HDDS-5333. Remove getRequestType method from OM request classes.
add b335431 HDDS-5336. Fix datanode capacity related race condition. (#2327)
add 92eb6a2 Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
add 4b4e475 Fix checkstyle after merge.
add 4506883 Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
add 0766d2c HDDS-5381. SCM terminated with exit status 1: null. (#2362)
add bf42bea HDDS-5369. Cleanup unused configuration related to SCM HA (#2359)
add e251bab HDDS-5289. Update container's deleteTransactionId on creation of the transaction in SCM. (#2361)
add 3232fb0 HDDS-5387. ProfileServlet to move the default output location to an ozone specific directory (#2368)
add 450c375 HDDS-5341. Container report processing is single threaded (#2338)
add be7264d HDDS-5380. Get more accurate space info for DedicatedDiskSpaceUsage. (#2365)
add b7ecf8b HDDS-5383. Eliminate expensive string creation in debug log messages (#2372)
add 474457c HDDS-5389. Include ozoneserviceid in fs.defaultFS when configuring o3fs (#2370)
add 1e5c708 HDDS-5395. Avoid unnecessary numKeyOps.incr() call in OMMetrics (#2374)
add 88383d1 HDDS-5396. Fix negligence issue conditional expressions in MockCRLStore.java (#2380)
add 1d8f972 HDDS-5398. Avoid object creation in ReplicationManger debug log statements (#2379)
add 039afb3 Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
add 4e8aec5 HDDS-5406. add proto version to all the proto files. (#2385)
add 9948632 HDDS-5405. Refactor pom files for HadoopRpc and Grpc/Ratis compilation properties. (#2386)
add b9a86c3 HDDS-5332. Add a new column family and a service provider in Recon DB for Namespace Summaries (#2366)
add 01e8811 Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade
add aaeae47 HDDS-5415. Fix TestSCMNodeManager after merge of master at 1d8f972 into upgrade branch (#2388)
add 19ed794 HDDS-3292. Support Hadoop 3.3 (#2356)
add 240af4d Merge branch 'master' into HDDS-3698-nonrolling-upgrade
add 4859acc HDDS-5417. Fix json parse stackoverflow for cmd ozone debug container list. (#2391)
add eb753e1 HDDS-5416. Intermittent test failure in TestSCMPipelineManager#testPipelineReload (#2394)
add a2739e2 HDDS-5424. Pipeline list command's output should be in local timezone. (#2400)
add 77c83c0 HDDS-5400. Remove inflight actions in Replication Manager if node switches to not IN_SERVICE (#2381)
add dba2d54 HDDS-5421. SCM throws NPE during JMX call. (#2398)
add 5af5a33 Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
add 7792c40 HDDS-5428. Fix potential BigDecimal precision in StorageUnit.java (#2405)
add 7ca3e13 HDDS-5394. Fix skipped volume check due to disk.check.min.gap. (#2373)
add f3ec595 HDDS-5427. Either re-interrupt this method or rethrow the "InterruptedException" that can be caught here (#2404)
add de23d45 HDDS-5422. Avoid eager string formatting in preconditions (#2396)
add f44440b HDDS-5433. Rename hadooprpc proto compilation in pom to refer to proto2. (#2407)
add d8a5d9e HDDS-5426. Remove unused method parameter in XceiverClientGrpc.java (#2409)
add d76aee4 HDDS-5384 OM refreshPipeline should not invoke the expensive OmKeyLocationInfoGroup.getLocationList() (#2367)
add 154c0c2 HDDS-5386. Add a NSSummaryTask to write NSSummary info into RDB (#2389)
add 59e31c4 HDDS-3432. Enable TestBlockDeletion test cases. (#2393)
add c41d08f HDDS-5296. Replication Manager should not sent replicate command to DN which is not IN_SERVICE (#2377)
add 1414bc3 HDDS-5434. Fix potential InterruptedException in Ozone (#2413)
add 76384ca HDDS-5442. Datanode HTTP server fails to start in ozonesecure due to wrong keytab name (#2419)
add aa4f969 HDDS-5431. compile both proto2 and proto3 versions of Client to OM interaction proto files. (#2410)
add 2b3a4ee HDDS-5403 Support list node based on NodeOperationalState and NodeState options in datanode list CLI (#2397)
add e99a0eb HDDS-5337. Apply container space check to Ratis factor one pipelines (#2354)
add 1474e46 HDDS-5291. Handle SIGTERM to ensure clean shutdown of OM/DN/SCM (#2301)
add dccfbae HDDS-5392. Avoid catching Error while creating Ozone client (#2408)
add d404339 HDDS-5456. Inject a Clock into Replication Manager to allow timeouts to be tested (#2425)
add b83039e HDDS-5436. Add acceptance test for Hadoop 3.3 (#2428)
add 31b2e6a HDDS-5005. Multipart Upload fails due to partName mismatch. (#2411)
add 4ccc128 HDDS-4928. Support container move in Replication Manager (#2349)
add 9081612 HDDS-3740. PipelineStateManagerV2Impl#removePipeline will remove pipeline from db in case of failure (#2415)
add b3b7108 HDDS-5451. Incorrect PENDINGDELETEBLOCKCOUNT caused by resent command. (#2421)
add 24f5641 HDDS-5464. fail move if node is not in IN_SERVICE state (#2437)
add af8752b HDDS-5310. Enhance freon streaming generator to support multiple threads (#2306)
add 5e6f6fe HDDS-5457. OZONE-SITE.XML_ozone.om.enable.filesystem.paths must be a string, number or null (#2426)
add b4fa8d9 HDDS-5454. Modify Statement to PrepareStatement in DatabaseHelper (#2423)
add 7b8bf98 HDDS-5467. Handle potential InterruptedException in Ozone (#2440)
add b588694 HDDS-5459. Replication Manager should process containers synchronously for tests (#2429)
add 57d42b1 HDDS-5459. Replication Manager should process containers synchronously for tests (#2446)
add c550a84 HDDS-5468. Avoid long sleep in TestPeriodicVolumeChecker (#2445)
add 9129424 HDDS-5469. Bump Apache Ratis version to 2.1 (#2443)
add 37f1b90 HDDS-5479. s3g bucket list failed when there is non-english key name. (#2450)
add 2759a9f HDDS-5418. Let Recon send reregisterCommand to Datanodes if DatanodeDetails changed (#2392)
add 839fc94 HDDS-4330. Bootstrap new OM node (#1494)
add 12958d3 HDDS-5492. Disable failing kubernetes test (#2459)
add 39beb7e HDDS-5443 Create and then recreate a bucket with a randomized name (#2436)
add 3fbfc0e HDDS-5484. Intermittent failure in TestReplicationManager#testMovePrerequisites (#2454)
add d3fe588 Multi-raft style placement with permutations for offline data generator (#2434)
add ceb79ac HDDS-5489. Install OS-specific flekszible (#2462)
add 0236213 HDDS-5500. New k3s version breaks kubernetes test (#2464)
add 0439b16 HDDS-5450. Avoid refresh pipeline for S3 headObject (#2431)
add bd7219a HDDS-5414. Data buffers incorrectly filtered for Ozone Insight (#2387)
add 5336bb9 HDDS-5494. Reduce retry in Kubernetes test (#2461)
add 24ee206 HDDS-5184. Use separate DB profile for Datanodes. (#2214)
add 08e948b HDDS-5465. Delete redundant code when set、add and remove bucket acl (#2439)
add 59d6e95 HDDS-5466. Refactor BlockOutputStream. (#2442)
new a0f6881 Merge branch 'master' into ec-merge-master
new 0c5c859 Fix issues caused by merging master into branch
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 78 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
hadoop-hdds/common/pom.xml | 17 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 9 +
.../hadoop/hdds/fs/AbstractSpaceUsageSource.java | 7 +-
.../hadoop/hdds/fs/DedicatedDiskSpaceUsage.java | 8 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 17 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 79 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../hadoop/hdds/scm/container/ContainerInfo.java | 5 +
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 4 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 8 +
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 76 ++
.../hdds/upgrade/HDDSLayoutVersionManager.java | 97 ++
.../hadoop/hdds/upgrade/HDDSUpgradeAction.java | 15 +-
.../apache/hadoop/hdds/upgrade}/package-info.java | 5 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 4 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 16 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 1 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 +-
.../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++
.../org/apache/hadoop/ozone/common/Storage.java | 41 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 33 +-
.../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++
.../apache/hadoop/ozone/conf}/package-info.java | 12 +-
.../upgrade/AbstractLayoutVersionManager.java | 162 +++
.../ozone/upgrade/BasicUpgradeFinalizer.java | 367 +++++++
.../DefaultUpgradeFinalizationExecutor.java | 66 ++
.../apache/hadoop/ozone/upgrade/LayoutFeature.java | 79 ++
.../upgrade/LayoutVersionInstanceFactory.java | 253 +++++
.../hadoop/ozone/upgrade/LayoutVersionManager.java | 76 ++
.../ozone/upgrade/LayoutVersionManagerMXBean.java | 18 +-
.../hadoop/ozone/upgrade/UpgradeActionHdds.java | 39 +-
.../hadoop/ozone/upgrade/UpgradeException.java | 159 +--
.../hadoop/ozone/upgrade/UpgradeFinalizer.java | 195 ++++
.../hadoop/ozone/upgrade/VersionFactoryKey.java | 70 ++
.../apache/hadoop/ozone/upgrade}/package-info.java | 12 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++
.../apache/hadoop/ozone/util}/package-info.java | 12 +-
.../main/resources/hdds-version-info.properties | 4 +-
.../common/src/main/resources/ozone-default.xml | 139 +--
.../hdds/upgrade/TestHDDSLayoutVersionManager.java | 102 ++
.../hadoop/hdds/upgrade/test/MockComponent.java | 67 ++
.../ozone/container/ContainerTestHelper.java | 75 ++
.../InjectedUpgradeFinalizationExecutor.java | 128 +++
.../upgrade/TestAbstractLayoutVersionManager.java | 203 ++++
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 126 +++
.../TestDefaultUpgradeFinalizationExecutor.java | 99 ++
.../upgrade/TestLayoutVersionInstanceFactory.java | 198 ++++
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 210 ++++
.../hadoop/ozone/upgrade/TestUpgradeUtils.java | 54 +
.../hadoop/hdds/conf/ConfigFileGenerator.java | 17 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +-
.../hadoop/hdds/freon/FakeRatisFollower.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 21 +-
.../container/common/DatanodeLayoutStorage.java | 98 ++
...utVersion.java => HDDSVolumeLayoutVersion.java} | 16 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../ozone/container/common/impl/ContainerData.java | 2 +
.../container/common/report/ReportManager.java | 3 +-
.../common/statemachine/DatanodeConfiguration.java | 4 +-
.../common/statemachine/DatanodeStateMachine.java | 53 +-
.../common/statemachine/StateContext.java | 3 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 2 +-
.../FinalizeNewLayoutVersionCommandHandler.java | 121 +++
.../states/endpoint/HeartbeatEndpointTask.java | 65 +-
.../states/endpoint/RegisterEndpointTask.java | 60 +-
.../common/transport/server/XceiverServerGrpc.java | 3 +-
.../server/ratis/ContainerStateMachine.java | 4 +-
.../transport/server/ratis/XceiverServerRatis.java | 2 +-
.../container/common/utils/HddsVolumeUtil.java | 6 +-
.../common/utils/db/DatanodeDBProfile.java | 145 +++
.../container/common/utils/db}/package-info.java | 10 +-
.../ozone/container/common/volume/HddsVolume.java | 7 +-
.../common/volume/ImmutableVolumeSet.java | 16 +
.../container/common/volume/MutableVolumeSet.java | 25 +-
.../common/volume/StorageVolumeChecker.java | 54 +-
.../ozone/container/common/volume/VolumeSet.java | 3 +
.../container/keyvalue/KeyValueContainer.java | 4 +-
.../container/metadata/AbstractDatanodeStore.java | 47 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 11 +-
.../replication/GrpcReplicationClient.java | 3 +-
.../container/replication/ReplicationServer.java | 1 +
.../replication/ReplicationSupervisor.java | 33 +-
.../ozone/container/stream/StreamingClient.java | 1 +
.../ozone/container/stream/StreamingServer.java | 1 +
.../upgrade/DataNodeUpgradeFinalizer.java | 95 ++
.../upgrade/DatanodeMetadataFeatures.java | 53 +
.../upgrade/DatanodeSchemaV2FinalizeAction.java | 48 +
.../ScmHAFirstUpgradeLayoutChangeAction.java | 45 +
.../UpgradeUtils.java} | 45 +-
.../ozone/container/upgrade}/package-info.java | 13 +-
.../protocol/StorageContainerDatanodeProtocol.java | 6 +-
.../protocol/StorageContainerNodeProtocol.java | 12 +-
.../commands/FinalizeNewLayoutVersionCommand.java | 80 ++
...inerDatanodeProtocolClientSideTranslatorPB.java | 11 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 16 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 11 +-
.../common/TestDatanodeLayOutVersion.java | 10 +-
.../common/helpers/TestContainerUtils.java | 60 ++
.../common/helpers/TestDatanodeVersionFile.java | 4 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 12 +-
.../common/volume/TestPeriodicVolumeChecker.java | 131 +++
.../container/keyvalue/TestKeyValueContainer.java | 77 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +-
.../replication/TestReplicationSupervisor.java | 4 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 96 ++
.../docs/content/design/nonrolling-upgrade.md | 32 +
hadoop-hdds/docs/content/design/omprepare.md | 60 ++
.../docs/content/design/upgrade-dev-primer.md | 111 ++
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 +
hadoop-hdds/docs/content/interface/O3fs.md | 8 +
hadoop-hdds/docs/content/interface/O3fs.zh.md | 8 +
.../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +-
...inerLocationProtocolClientSideTranslatorPB.java | 54 +-
.../SCMBlockLocationFailoverProxyProvider.java | 8 +-
.../SCMSecurityProtocolFailoverProxyProvider.java | 8 +-
.../scm/update/client/CRLClientUpdateHandler.java | 7 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 3 +-
.../authority/PKIProfiles/DefaultProfile.java | 2 +-
.../hadoop/hdds/server/events/EventExecutor.java | 5 +
.../hadoop/hdds/server/events/EventQueue.java | 44 +-
...dExecutor.java => FixedThreadPoolExecutor.java} | 42 +-
.../hdds/server/events/SingleThreadExecutor.java | 9 +
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../hadoop/hdds/server/http/ProfileServlet.java | 2 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +
.../apache/hadoop/hdds/utils/db/ShortCodec.java} | 50 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 62 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 9 +
hadoop-hdds/interface-admin/pom.xml | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 23 +-
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdds/interface-client/pom.xml | 40 +-
.../interface-client/src/main/proto/hdds.proto | 24 +-
hadoop-hdds/interface-server/pom.xml | 4 +-
.../src/main/proto/SCMRatisProtocol.proto | 2 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +
.../src/main/proto/ScmServerProtocol.proto | 2 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 1 +
hadoop-hdds/server-scm/pom.xml | 24 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 73 +-
.../block/DatanodeDeletedBlockTransactions.java | 6 -
.../hdds/scm/block/DeletedBlockLogImplV2.java | 1 +
.../scm/block/DeletedBlockLogStateManagerImpl.java | 26 +-
.../hdds/scm/block/PendingDeleteStatusList.java | 85 --
.../hdds/scm/block/SCMBlockDeletingService.java | 26 +-
.../hdds/scm/container/ContainerReportHandler.java | 38 -
.../scm/container/ContainerStateManagerImpl.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 562 +++++++++--
.../scm/container/balancer/ContainerBalancer.java | 4 +-
.../algorithms/SCMContainerPlacementCapacity.java | 10 +-
.../algorithms/SCMContainerPlacementRackAware.java | 102 +-
.../algorithms/SCMContainerPlacementRandom.java | 10 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 23 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 +
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 3 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 5 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 8 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 +
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 43 +-
...andler.java => HealthyReadOnlyNodeHandler.java} | 41 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 39 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 216 +++-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 14 +-
...va => ReadOnlyHealthyToHealthyNodeHandler.java} | 15 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 170 +++-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 10 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 15 +
.../hdds/scm/pipeline/ECPipelineProvider.java | 11 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 12 +-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 25 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 49 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +-
.../scm/pipeline/PipelineStateManagerV2Impl.java | 6 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 29 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 63 ++
.../hdds/scm/server/SCMClientProtocolServer.java | 38 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 31 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 24 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 1 +
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 9 +-
.../hdds/scm/server/StorageContainerManager.java | 190 ++--
.../scm/server/StorageContainerManagerStarter.java | 12 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 163 +++
.../ScmHAUnfinalizedStateValidationAction.java | 72 ++
.../ScmOnFinalizeActionForDatanodeSchemaV2.java | 46 +
.../hdds/scm/server/upgrade}/package-info.java | 5 +-
.../update/server/SCMUpdateServiceGrpcServer.java | 3 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 19 +
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 96 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 26 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 40 +-
.../scm/container/TestContainerReportHandler.java | 4 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../hdds/scm/container/TestReplicationManager.java | 455 +++++++--
.../algorithms/TestContainerPlacementFactory.java | 41 +-
.../TestSCMContainerPlacementCapacity.java | 6 +-
.../TestSCMContainerPlacementRackAware.java | 112 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../hdds/scm/node/TestContainerPlacement.java | 23 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 2 +-
.../hdds/scm/node/TestNodeReportHandler.java | 16 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 57 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 532 +++++++++-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 17 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 26 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 10 +
.../hdds/scm/pipeline/TestECPipelineProvider.java | 15 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 44 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 157 ++-
.../hdds/scm/update/server/MockCRLStore.java | 6 +-
.../TestSCMHAUnfinalizedStateValidationAction.java | 128 +++
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 75 ++
.../hadoop/hdds/scm/upgrade/package-info.java | 21 +
.../ozone/container/common/TestEndPoint.java | 20 +-
.../placement/TestContainerPlacement.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 23 +-
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 63 +-
.../main/java/org/apache/ozone/test/TestClock.java | 76 ++
hadoop-hdds/tools/pom.xml | 12 +-
.../hdds/scm/cli/ContainerOperationClient.java | 16 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 +
.../scm/cli/datanode/TestListInfoSubcommand.java | 17 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 16 +
.../org/apache/hadoop/ozone/client/OzoneKey.java | 4 +
.../ozone/client/io/BlockOutputStreamEntry.java | 5 +-
.../ozone/client/protocol/ClientProtocol.java | 16 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 21 +
hadoop-ozone/common/pom.xml | 4 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 31 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../apache/hadoop/ozone/om/OmUpgradeConfig.java | 82 ++
.../hadoop/ozone/om/exceptions/OMException.java | 10 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 67 +-
.../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 11 +
.../hadoop/ozone/om/helpers/WithObjectID.java | 6 +-
.../ozone/om/protocol/OMInterServiceProtocol.java | 32 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 105 ++
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +-
.../OMInterServiceProtocolClientSideImpl.java | 121 +++
.../om/protocolPB/OMInterServiceProtocolPB.java | 52 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 111 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 4 +-
.../main/resources/ozone-version-info.properties | 4 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 31 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 3 +-
hadoop-ozone/csi/pom.xml | 6 +-
hadoop-ozone/dev-support/checks/_lib.sh | 9 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +-
hadoop-ozone/dist/pom.xml | 2 +-
.../1.0.0/test.sh => ozone-mr/hadoop33/.env} | 13 +-
.../{hadoop32 => hadoop33}/docker-compose.yaml | 0
.../ozone-mr/{hadoop32 => hadoop33}/docker-config | 0
.../ozone-mr/{hadoop32 => hadoop33}/test.sh | 0
.../compose/{upgrade => ozone-om-prepare}/.env | 10 +-
.../docker-compose.yaml | 75 +-
.../{upgrade => ozone-om-prepare}/docker-config | 28 +-
.../dist/src/main/compose/ozone-om-prepare/test.sh | 74 ++
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../src/main/compose/ozonesecure/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 22 +-
.../dist/src/main/compose/upgrade/README.md | 95 +-
.../src/main/compose/upgrade/{ => compose/ha}/.env | 7 +-
.../upgrade/{ => compose/ha}/docker-compose.yaml | 120 ++-
.../compose/upgrade/{ => compose/ha}/docker-config | 30 +-
.../upgrade/{0.5.0/test.sh => compose/ha/load.sh} | 14 +-
.../main/compose/upgrade/{ => compose/non-ha}/.env | 2 +-
.../{ => compose/non-ha}/docker-compose.yaml | 99 +-
.../upgrade/{ => compose/non-ha}/docker-config | 13 +-
.../{0.5.0/test.sh => compose/non-ha/load.sh} | 13 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 30 +-
.../dist/src/main/compose/upgrade/testlib.sh | 135 ++-
.../src/main/compose/upgrade/upgrade_to_current.sh | 51 -
.../manual-upgrade/0.5.0-1.1.0/callback.sh} | 26 +-
.../manual-upgrade/driver.sh} | 63 +-
.../non-rolling-upgrade/1.0.0-1.1.0/callback.sh | 79 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 101 ++
.../dist/src/main/compose/versions/0.5.0.sh | 4 -
.../dist/src/main/compose/versions/1.0.0.sh | 4 -
.../dist/src/main/compose/versions/1.1.0.sh | 4 -
.../dist/src/main/compose/versions/README.md | 3 -
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 8 +
.../license/bin/licenses/LICENSE-org.aspectj.html | 91 ++
.../LICENSE-org.reflections-reflections.txt | 14 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 69 +-
.../src/main/smoketest/admincli/datanode.robot | 18 +
.../main/smoketest/omha/om-cancel-prepare.robot | 43 +
.../dist/src/main/smoketest/omha/om-prepare.robot | 49 +
.../bucketcreate.robot => omha/om-prepared.robot} | 29 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +-
.../dist/src/main/smoketest/spnego/web.robot | 2 +-
.../bucketcreate.robot => upgrade/check-mlv.robot} | 30 +-
.../bucketcreate.robot => upgrade/finalize.robot} | 30 +-
.../bucketcreate.robot => upgrade/generate.robot} | 26 +-
.../bucketcreate.robot => upgrade/prepare.robot} | 25 +-
.../bucketcreate.robot => upgrade/validate.robot} | 23 +-
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../dist/src/shell/upgrade/{v1.sh => 1.0.0.sh} | 2 +-
.../upgrade/{v1 => 1.0.0}/01-migrate-scm-db.sh | 0
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +-
hadoop-ozone/integration-test/pom.xml | 3 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 1067 ++++++++++++++++++++
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 25 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 52 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 263 ++++-
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 7 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 +
.../hadoop/ozone/TestStorageContainerManager.java | 37 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 13 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 13 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 17 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestWatchForCommit.java | 3 +-
.../ozone/container/TestContainerReplication.java | 81 +-
.../apache/hadoop/ozone/container/TestHelper.java | 6 +-
.../commandhandler/TestBlockDeletion.java | 313 +++---
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 254 +++++
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 1 -
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 80 ++
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 ++++
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 +
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 510 ++++++++++
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 1 +
.../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 1 -
.../dev-support/findbugsExcludeFile.xml | 6 +
hadoop-ozone/interface-client/pom.xml | 62 +-
.../src/main/proto/OmClientProtocol.proto | 85 +-
.../src/main/proto/OmInterServiceProtocol.proto | 64 ++
.../interface-client/src/main/proto/Security.proto | 1 +
hadoop-ozone/interface-storage/pom.xml | 2 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-ozone/ozone-manager/pom.xml | 43 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 47 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 1 -
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 +
.../apache/hadoop/ozone/om/OMStarterInterface.java | 4 +
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 7 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 30 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 621 +++++++++---
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 311 ++++++
.../hadoop/ozone/om/OzoneManagerStarter.java | 79 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 21 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 15 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 405 +++++---
.../ozone/om/ratis/OzoneManagerStateMachine.java | 87 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 97 ++
.../hadoop/ozone/om/request/OMClientRequest.java | 7 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 1 +
.../request/bucket/OMBucketSetPropertyRequest.java | 1 +
.../om/request/bucket/acl/OMBucketAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 12 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 7 -
.../om/request/file/OMDirectoryCreateRequest.java | 1 +
.../ozone/om/request/file/OMFileCreateRequest.java | 7 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 1 +
.../ozone/om/request/key/OMKeyDeleteRequest.java | 1 +
.../ozone/om/request/key/OMKeyPurgeRequest.java | 1 +
.../ozone/om/request/key/OMKeyRenameRequest.java | 1 +
.../om/request/key/acl/OMKeyAddAclRequest.java | 1 +
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 1 +
.../om/request/key/acl/OMKeySetAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixAddAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixSetAclRequest.java | 1 +
.../S3InitiateMultipartUploadRequest.java | 1 +
.../S3MultipartUploadCommitPartRequest.java | 11 +-
.../S3MultipartUploadCompleteRequest.java | 1 +
.../om/request/s3/security/S3GetSecretRequest.java | 1 +
.../security/OMCancelDelegationTokenRequest.java | 2 +-
.../security/OMGetDelegationTokenRequest.java | 2 +-
.../security/OMRenewDelegationTokenRequest.java | 1 +
.../om/request/upgrade/OMCancelPrepareRequest.java | 97 ++
.../request/upgrade/OMFinalizeUpgradeRequest.java | 116 +++
.../ozone/om/request/upgrade/OMPrepareRequest.java | 257 +++++
.../security => request/upgrade}/package-info.java | 15 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 2 +
.../om/request/volume/OMVolumeSetOwnerRequest.java | 1 +
.../om/request/volume/OMVolumeSetQuotaRequest.java | 1 +
.../hadoop/ozone/om/response/CleanupTableInfo.java | 10 +-
.../om/response/key/OMOpenKeysDeleteRequest.java | 1 +
.../om/response/s3/security/package-info.java | 2 +-
.../response/upgrade/OMCancelPrepareResponse.java | 46 +
.../upgrade/OMFinalizeUpgradeResponse.java | 58 ++
.../om/response/upgrade/OMPrepareResponse.java | 60 ++
.../{s3/security => upgrade}/package-info.java | 15 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +-
.../BelongsToLayoutVersion.java} | 23 +-
.../DisallowedUntilLayoutVersion.java} | 31 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 80 ++
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 114 +++
.../ozone/om/upgrade/OMLayoutVersionManager.java | 210 ++++
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 50 +
.../hadoop/ozone/om/upgrade/OmUpgradeAction.java | 19 +-
.../hadoop/ozone/om/upgrade/UpgradeActionOm.java | 37 +
.../s3/security => upgrade}/package-info.java | 7 +-
.../OMInterServiceProtocolServerSideImpl.java | 90 ++
...OzoneManagerProtocolServerSideTranslatorPB.java | 87 +-
.../protocolPB/OzoneManagerRequestHandler.java | 70 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 9 +
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 34 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 108 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 5 +
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
.../upgrade/TestOMCancelPrepareRequest.java | 110 ++
.../request/volume/TestOMVolumeDeleteRequest.java | 2 +-
.../om/request/volume/TestOMVolumeRequest.java | 9 +-
.../ozone/om/response/TestCleanupTableInfo.java | 26 +-
.../hadoop/ozone/om/upgrade/MockOmRequest.java | 23 +-
.../ozone/om/upgrade/OMLayoutFeatureUtil.java | 66 ++
.../om/upgrade/TestOMLayoutFeatureAspect.java | 99 ++
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 290 ++++++
.../ozone/om/upgrade/TestOMVersionManager.java | 195 ++++
.../TestOmVersionManagerRequestFactory.java | 111 ++
.../om/upgrade/TestOzoneManagerPrepareState.java | 283 ++++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 5 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +-
hadoop-ozone/recon/pom.xml | 5 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 7 +
.../hadoop/ozone/recon/ReconControllerModule.java | 20 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 29 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 +
.../ozone/recon/api/ClusterStateEndpoint.java | 3 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 13 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 15 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 13 +
.../hadoop/ozone/recon/api/types/NSSummary.java | 86 ++
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 101 ++
.../ozone/recon/scm/ReconContainerManager.java | 8 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 139 ++-
.../scm/ReconStorageContainerManagerFacade.java | 16 +-
...der.java => ReconContainerMetadataManager.java} | 9 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 22 +-
...java => ReconContainerMetadataManagerImpl.java} | 54 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 13 +-
...ntainerDBProvider.java => ReconDBProvider.java} | 86 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 72 ++
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 35 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 25 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 303 ++++++
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 +
.../src/views/datanodes/datanodes.tsx | 8 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 77 ++
.../hadoop/ozone/recon/ReconTestInjector.java | 20 +-
.../ozone/recon/api/TestContainerEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 13 +-
.../ozone/recon/api/TestOpenContainerCount.java | 7 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../scm/AbstractReconContainerManagerTest.java | 17 +-
...TestReconIncrementalContainerReportHandler.java | 14 +-
.../ozone/recon/scm/TestReconNodeManager.java | 57 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 23 +-
... => TestReconContainerMetadataManagerImpl.java} | 144 +--
...nerDBProvider.java => TestReconDBProvider.java} | 13 +-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 104 ++
.../recon/tasks/TestContainerKeyMapperTask.java | 75 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 480 +++++++++
hadoop-ozone/s3gateway/pom.xml | 5 +
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 11 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 4 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 20 +
.../hadoop/ozone/s3/util/TestContinueToken.java | 20 +
.../ozone/admin/om/CancelPrepareSubCommand.java | 53 +
.../admin/om/FinalizationStatusSubCommand.java | 64 ++
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 171 ++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 44 +-
.../hadoop/ozone/admin/om/PrepareSubCommand.java | 173 ++++
.../admin/scm/FinalizationScmStatusSubcommand.java | 51 +
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 156 +++
.../admin/scm/FinalizeUpgradeCommandUtil.java | 92 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 4 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../freon/FollowerAppendLogEntryGenerator.java | 4 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 70 +-
.../containergenerator/GeneratorDatanode.java | 60 +-
.../containergenerator/TestGeneratorDatanode.java | 71 ++
pom.xml | 22 +-
519 files changed, 21686 insertions(+), 3722 deletions(-)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/RatisBlockOutputStream.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java (73%)
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade}/package-info.java (87%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/MonotonicClock.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/OzoneServiceConfig.java
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf}/package-info.java (80%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionInstanceFactory.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManager.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManagerMXBean.java (73%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/CleanupTableInfo.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java (63%)
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java (52%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalizer.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/VersionFactoryKey.java
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade}/package-info.java (80%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/ShutdownHookManager.java
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util}/package-info.java (80%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/test/MockComponent.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/InjectedUpgradeFinalizationExecutor.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestBasicUpgradeFinalizer.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestDefaultUpgradeFinalizationExecutor.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestLayoutVersionInstanceFactory.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeUtils.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeLayoutStorage.java
rename hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/{DataNodeLayoutVersion.java => HDDSVolumeLayoutVersion.java} (78%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeNewLayoutVersionCommandHandler.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/db/DatanodeDBProfile.java
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/db}/package-info.java (81%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeMetadataFeatures.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFirstUpgradeLayoutChangeAction.java
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/{common/volume/ImmutableVolumeSet.java => upgrade/UpgradeUtils.java} (52%)
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade}/package-info.java (80%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeNewLayoutVersionCommand.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestPeriodicVolumeChecker.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDataNodeStartupSlvLessThanMlv.java
create mode 100644 hadoop-hdds/docs/content/design/nonrolling-upgrade.md
create mode 100644 hadoop-hdds/docs/content/design/omprepare.md
create mode 100644 hadoop-hdds/docs/content/design/upgrade-dev-primer.md
create mode 100644 hadoop-hdds/docs/content/feature/how-to-do-a-nonrolling-upgrade.md
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/{SingleThreadExecutor.java => FixedThreadPoolExecutor.java} (67%)
copy hadoop-hdds/{config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java => framework/src/main/java/org/apache/hadoop/hdds/utils/db/ShortCodec.java} (60%)
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/PendingDeleteStatusList.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/{NonHealthyToHealthyNodeHandler.java => HealthyReadOnlyNodeHandler.java} (54%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/{NonHealthyToHealthyNodeHandler.java => ReadOnlyHealthyToHealthyNodeHandler.java} (78%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmHAUnfinalizedStateValidationAction.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade}/package-info.java (87%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestSCMHAUnfinalizedStateValidationAction.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/package-info.java
create mode 100644 hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/TestClock.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmUpgradeConfig.java
rename hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha => common/src/main/java/org/apache/hadoop/ozone/om/helpers}/OMNodeDetails.java (54%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/PendingDeleteHandler.java => hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMInterServiceProtocol.java (53%)
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMInterServiceProtocolClientSideImpl.java
copy hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto => hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMInterServiceProtocolPB.java (52%)
rename hadoop-ozone/dist/src/main/compose/{upgrade/1.0.0/test.sh => ozone-mr/hadoop33/.env} (77%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/compose/ozone-mr/{hadoop32 => hadoop33}/docker-compose.yaml (100%)
copy hadoop-ozone/dist/src/main/compose/ozone-mr/{hadoop32 => hadoop33}/docker-config (100%)
copy hadoop-ozone/dist/src/main/compose/ozone-mr/{hadoop32 => hadoop33}/test.sh (100%)
copy hadoop-ozone/dist/src/main/compose/{upgrade => ozone-om-prepare}/.env (73%)
copy hadoop-ozone/dist/src/main/compose/{upgrade => ozone-om-prepare}/docker-compose.yaml (65%)
copy hadoop-ozone/dist/src/main/compose/{upgrade => ozone-om-prepare}/docker-config (60%)
create mode 100755 hadoop-ozone/dist/src/main/compose/ozone-om-prepare/test.sh
copy hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/ha}/.env (79%)
copy hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/ha}/docker-compose.yaml (71%)
copy hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/ha}/docker-config (73%)
copy hadoop-ozone/dist/src/main/compose/upgrade/{0.5.0/test.sh => compose/ha/load.sh} (73%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/non-ha}/.env (97%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/non-ha}/docker-compose.yaml (78%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/non-ha}/docker-config (89%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{0.5.0/test.sh => compose/non-ha/load.sh} (75%)
delete mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh
copy hadoop-ozone/dist/src/main/compose/{versions/1.0.0.sh => upgrade/upgrades/manual-upgrade/0.5.0-1.1.0/callback.sh} (64%)
mode change 100644 => 100755
rename hadoop-ozone/dist/src/main/compose/upgrade/{upgrade_to_release.sh => upgrades/manual-upgrade/driver.sh} (50%)
create mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.0.0-1.1.0/callback.sh
create mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.aspectj.html
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.reflections-reflections.txt
create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/om-cancel-prepare.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/om-prepare.robot
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => omha/om-prepared.robot} (52%)
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => upgrade/check-mlv.robot} (58%)
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => upgrade/finalize.robot} (54%)
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => upgrade/generate.robot} (60%)
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => upgrade/prepare.robot} (57%)
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => upgrade/validate.robot} (62%)
rename hadoop-ozone/dist/src/shell/upgrade/{v1.sh => 1.0.0.sh} (85%)
rename hadoop-ozone/dist/src/shell/upgrade/{v1 => 1.0.0}/01-migrate-scm-db.sh (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupSlvLessThanMlv.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerBootstrap.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
create mode 100644 hadoop-ozone/interface-client/src/main/proto/OmInterServiceProtocol.proto
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerPrepareState.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMCancelPrepareRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMPrepareRequest.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/s3/security => request/upgrade}/package-info.java (73%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCancelPrepareResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMPrepareResponse.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{s3/security => upgrade}/package-info.java (73%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/CleanupTableInfo.java => upgrade/BelongsToLayoutVersion.java} (66%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/CleanupTableInfo.java => upgrade/DisallowedUntilLayoutVersion.java} (61%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizer.java
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OmUpgradeAction.java (67%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/UpgradeActionOm.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/s3/security => upgrade}/package-info.java (86%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMInterServiceProtocolServerSideImpl.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMCancelPrepareRequest.java
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java => hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java (65%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizer.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOmVersionManagerRequestFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOzoneManagerPrepareState.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java
rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/{ContainerDBServiceProvider.java => ReconContainerMetadataManager.java} (97%)
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java (59%)
rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/{ContainerDBServiceProviderImpl.java => ReconContainerMetadataManagerImpl.java} (90%)
rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/{ReconContainerDBProvider.java => ReconDBProvider.java} (61%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java
rename hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/{TestContainerDBServiceProviderImpl.java => TestReconContainerMetadataManagerImpl.java} (65%)
rename hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/{TestReconContainerDBProvider.java => TestReconDBProvider.java} (83%)
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/CancelPrepareSubCommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizeUpgradeSubCommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/PrepareSubCommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/FinalizationScmStatusSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/FinalizeScmUpgradeSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/FinalizeUpgradeCommandUtil.java
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/containergenerator/TestGeneratorDatanode.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 02/02: Fix issues caused by merging master into branch
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 0c5c859b99e367f70210bbb9793eb8f26c616c7a
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Fri Jul 30 11:06:35 2021 +0100
Fix issues caused by merging master into branch
---
.../apache/hadoop/hdds/scm/storage/BlockOutputStream.java | 8 --------
.../hadoop/hdds/scm/pipeline/ECPipelineProvider.java | 11 +++++++++--
.../hadoop/hdds/scm/pipeline/TestECPipelineProvider.java | 15 +++++++++++----
.../hadoop/ozone/client/io/BlockOutputStreamEntry.java | 10 ++--------
.../src/main/proto/OmClientProtocol.proto | 2 +-
5 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
index c251c46..ebcd962 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
@@ -213,7 +213,6 @@ public class BlockOutputStream extends OutputStream {
return ioException.get();
}
-<<<<<<< HEAD
XceiverClientSpi getXceiverClientSpi(){
return this.xceiverClient;
}
@@ -230,13 +229,6 @@ public class BlockOutputStream extends OutputStream {
return this.responseExecutor;
}
- @VisibleForTesting
- public Map<Long, List<ChunkBuffer>> getCommitIndex2flushedDataMap() {
- return commitWatcher.getCommitIndex2flushedDataMap();
- }
-
-=======
->>>>>>> master
@Override
public void write(int b) throws IOException {
checkOpen();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
index d8fd343..f674b6f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.hdds.scm.pipeline;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import java.io.IOException;
@@ -40,6 +42,7 @@ public class ECPipelineProvider extends PipelineProvider<ECReplicationConfig> {
private final ConfigurationSource conf;
private final PlacementPolicy placementPolicy;
+ private final long containerSizeBytes;
public ECPipelineProvider(NodeManager nodeManager,
StateManager stateManager,
@@ -48,6 +51,9 @@ public class ECPipelineProvider extends PipelineProvider<ECReplicationConfig> {
super(nodeManager, stateManager);
this.conf = conf;
this.placementPolicy = placementPolicy;
+ this.containerSizeBytes = (long) this.conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
}
@Override
@@ -61,8 +67,9 @@ public class ECPipelineProvider extends PipelineProvider<ECReplicationConfig> {
protected Pipeline create(ECReplicationConfig replicationConfig,
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes)
throws IOException {
- List<DatanodeDetails> dns = placementPolicy.chooseDatanodes(excludedNodes,
- favoredNodes, replicationConfig.getRequiredNodes(), 0);
+ List<DatanodeDetails> dns = placementPolicy
+ .chooseDatanodes(excludedNodes, favoredNodes,
+ replicationConfig.getRequiredNodes(), 0, this.containerSizeBytes);
return create(replicationConfig, dns);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
index 326bc1c..eb88a60 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.hdds.scm.pipeline;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.junit.Assert;
import org.junit.Before;
@@ -47,16 +49,20 @@ public class TestECPipelineProvider {
private NodeManager nodeManager = Mockito.mock(NodeManager.class);
private StateManager stateManager = Mockito.mock(StateManager.class);
private PlacementPolicy placementPolicy = Mockito.mock(PlacementPolicy.class);
-
+ private long containerSizeBytes;
@Before
public void setup() throws IOException {
conf = new OzoneConfiguration();
provider = new ECPipelineProvider(
nodeManager, stateManager, conf, placementPolicy);
-
+ this.containerSizeBytes = (long) this.conf.getStorageSize(
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
+ StorageUnit.BYTES);
// Placement policy will always return EC number of random nodes.
Mockito.when(placementPolicy.chooseDatanodes(Mockito.anyList(),
- Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong()))
+ Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong(),
+ Mockito.anyLong()))
.thenAnswer(invocation -> {
List<DatanodeDetails> dns = new ArrayList<>();
for (int i=0; i<(int)invocation.getArguments()[2]; i++) {
@@ -64,6 +70,7 @@ public class TestECPipelineProvider {
}
return dns;
});
+
}
@@ -99,7 +106,7 @@ public class TestECPipelineProvider {
pipeline.getNodes().size());
verify(placementPolicy).chooseDatanodes(excludedNodes, favoredNodes,
- ecConf.getRequiredNodes(), 0);
+ ecConf.getRequiredNodes(), 0, containerSizeBytes);
}
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
index 3fa339a..2d151ed 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
@@ -96,19 +96,13 @@ public class BlockOutputStreamEntry extends OutputStream {
*/
private void checkStream() throws IOException {
if (this.outputStream == null) {
-<<<<<<< HEAD
this.outputStream = createOutputStream();
-=======
- this.outputStream =
- new RatisBlockOutputStream(blockID, xceiverClientManager,
- pipeline, bufferPool, config, token);
->>>>>>> master
}
}
BlockOutputStream createOutputStream() throws IOException {
- return new BlockOutputStream(blockID, xceiverClientManager,
- pipeline, bufferPool, config, token);
+ return new RatisBlockOutputStream(blockID, xceiverClientManager, pipeline,
+ bufferPool, config, token);
}
@Override
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 3706a62..a330e78 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -772,7 +772,7 @@ message KeyArgs {
// When it is a head operation which is to check whether key exist
optional bool headOp = 18;
- optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 18;
+ optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 19;
}
message KeyLocation {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/02: Merge branch 'master' into ec-merge-master
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a0f6881a281b12752d87b9444e1b99d4af30478a
Merge: cd781b9 59d6e95
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Wed Jul 28 21:41:07 2021 +0100
Merge branch 'master' into ec-merge-master
.github/workflows/post-commit.yml | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 76 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
hadoop-hdds/common/pom.xml | 17 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 9 +
.../hadoop/hdds/fs/AbstractSpaceUsageSource.java | 7 +-
.../hadoop/hdds/fs/DedicatedDiskSpaceUsage.java | 8 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 17 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 79 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../hadoop/hdds/scm/container/ContainerInfo.java | 5 +
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 4 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 8 +
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 76 ++
.../hdds/upgrade/HDDSLayoutVersionManager.java | 97 ++
.../hadoop/hdds/upgrade/HDDSUpgradeAction.java | 15 +-
.../apache/hadoop/hdds/upgrade}/package-info.java | 5 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 4 +-
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 16 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 1 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 +-
.../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++
.../org/apache/hadoop/ozone/common/Storage.java | 41 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 33 +-
.../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++
.../apache/hadoop/ozone/conf}/package-info.java | 12 +-
.../upgrade/AbstractLayoutVersionManager.java | 162 +++
.../ozone/upgrade/BasicUpgradeFinalizer.java | 367 +++++++
.../DefaultUpgradeFinalizationExecutor.java | 66 ++
.../apache/hadoop/ozone/upgrade/LayoutFeature.java | 79 ++
.../upgrade/LayoutVersionInstanceFactory.java | 253 +++++
.../hadoop/ozone/upgrade/LayoutVersionManager.java | 76 ++
.../ozone/upgrade/LayoutVersionManagerMXBean.java | 18 +-
.../hadoop/ozone/upgrade/UpgradeActionHdds.java | 39 +-
.../hadoop/ozone/upgrade/UpgradeException.java | 159 +--
.../hadoop/ozone/upgrade/UpgradeFinalizer.java | 195 ++++
.../hadoop/ozone/upgrade/VersionFactoryKey.java | 70 ++
.../apache/hadoop/ozone/upgrade}/package-info.java | 12 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++
.../apache/hadoop/ozone/util}/package-info.java | 12 +-
.../main/resources/hdds-version-info.properties | 4 +-
.../common/src/main/resources/ozone-default.xml | 139 +--
.../hdds/upgrade/TestHDDSLayoutVersionManager.java | 102 ++
.../hadoop/hdds/upgrade/test/MockComponent.java | 67 ++
.../ozone/container/ContainerTestHelper.java | 75 ++
.../InjectedUpgradeFinalizationExecutor.java | 128 +++
.../upgrade/TestAbstractLayoutVersionManager.java | 203 ++++
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 126 +++
.../TestDefaultUpgradeFinalizationExecutor.java | 99 ++
.../upgrade/TestLayoutVersionInstanceFactory.java | 198 ++++
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 210 ++++
.../hadoop/ozone/upgrade/TestUpgradeUtils.java | 54 +
.../hadoop/hdds/conf/ConfigFileGenerator.java | 17 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +-
.../hadoop/hdds/freon/FakeRatisFollower.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 21 +-
.../container/common/DatanodeLayoutStorage.java | 98 ++
...utVersion.java => HDDSVolumeLayoutVersion.java} | 16 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../ozone/container/common/impl/ContainerData.java | 2 +
.../container/common/report/ReportManager.java | 3 +-
.../common/statemachine/DatanodeConfiguration.java | 4 +-
.../common/statemachine/DatanodeStateMachine.java | 53 +-
.../common/statemachine/StateContext.java | 3 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 2 +-
.../FinalizeNewLayoutVersionCommandHandler.java | 121 +++
.../states/endpoint/HeartbeatEndpointTask.java | 65 +-
.../states/endpoint/RegisterEndpointTask.java | 60 +-
.../common/transport/server/XceiverServerGrpc.java | 3 +-
.../server/ratis/ContainerStateMachine.java | 4 +-
.../transport/server/ratis/XceiverServerRatis.java | 2 +-
.../container/common/utils/HddsVolumeUtil.java | 6 +-
.../common/utils/db/DatanodeDBProfile.java | 145 +++
.../container/common/utils/db}/package-info.java | 10 +-
.../ozone/container/common/volume/HddsVolume.java | 7 +-
.../common/volume/ImmutableVolumeSet.java | 16 +
.../container/common/volume/MutableVolumeSet.java | 25 +-
.../common/volume/StorageVolumeChecker.java | 54 +-
.../ozone/container/common/volume/VolumeSet.java | 3 +
.../container/keyvalue/KeyValueContainer.java | 4 +-
.../container/metadata/AbstractDatanodeStore.java | 47 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 11 +-
.../replication/GrpcReplicationClient.java | 3 +-
.../container/replication/ReplicationServer.java | 1 +
.../replication/ReplicationSupervisor.java | 33 +-
.../ozone/container/stream/StreamingClient.java | 1 +
.../ozone/container/stream/StreamingServer.java | 1 +
.../upgrade/DataNodeUpgradeFinalizer.java | 95 ++
.../upgrade/DatanodeMetadataFeatures.java | 53 +
.../upgrade/DatanodeSchemaV2FinalizeAction.java | 48 +
.../ScmHAFirstUpgradeLayoutChangeAction.java | 45 +
.../UpgradeUtils.java} | 45 +-
.../ozone/container/upgrade}/package-info.java | 13 +-
.../protocol/StorageContainerDatanodeProtocol.java | 6 +-
.../protocol/StorageContainerNodeProtocol.java | 12 +-
.../commands/FinalizeNewLayoutVersionCommand.java | 80 ++
...inerDatanodeProtocolClientSideTranslatorPB.java | 11 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 16 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 11 +-
.../common/TestDatanodeLayOutVersion.java | 10 +-
.../common/helpers/TestContainerUtils.java | 60 ++
.../common/helpers/TestDatanodeVersionFile.java | 4 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 12 +-
.../common/volume/TestPeriodicVolumeChecker.java | 131 +++
.../container/keyvalue/TestKeyValueContainer.java | 77 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +-
.../replication/TestReplicationSupervisor.java | 4 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 96 ++
.../docs/content/design/nonrolling-upgrade.md | 32 +
hadoop-hdds/docs/content/design/omprepare.md | 60 ++
.../docs/content/design/upgrade-dev-primer.md | 111 ++
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 +
hadoop-hdds/docs/content/interface/O3fs.md | 8 +
hadoop-hdds/docs/content/interface/O3fs.zh.md | 8 +
.../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +-
...inerLocationProtocolClientSideTranslatorPB.java | 54 +-
.../SCMBlockLocationFailoverProxyProvider.java | 8 +-
.../SCMSecurityProtocolFailoverProxyProvider.java | 8 +-
.../scm/update/client/CRLClientUpdateHandler.java | 7 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 3 +-
.../authority/PKIProfiles/DefaultProfile.java | 2 +-
.../hadoop/hdds/server/events/EventExecutor.java | 5 +
.../hadoop/hdds/server/events/EventQueue.java | 44 +-
...dExecutor.java => FixedThreadPoolExecutor.java} | 42 +-
.../hdds/server/events/SingleThreadExecutor.java | 9 +
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../hadoop/hdds/server/http/ProfileServlet.java | 2 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +
.../apache/hadoop/hdds/utils/db/ShortCodec.java} | 50 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 62 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 9 +
hadoop-hdds/interface-admin/pom.xml | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 23 +-
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdds/interface-client/pom.xml | 40 +-
.../interface-client/src/main/proto/hdds.proto | 24 +-
hadoop-hdds/interface-server/pom.xml | 4 +-
.../src/main/proto/SCMRatisProtocol.proto | 2 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +
.../src/main/proto/ScmServerProtocol.proto | 2 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 1 +
hadoop-hdds/server-scm/pom.xml | 24 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 73 +-
.../block/DatanodeDeletedBlockTransactions.java | 6 -
.../hdds/scm/block/DeletedBlockLogImplV2.java | 1 +
.../scm/block/DeletedBlockLogStateManagerImpl.java | 26 +-
.../hdds/scm/block/PendingDeleteStatusList.java | 85 --
.../hdds/scm/block/SCMBlockDeletingService.java | 26 +-
.../hdds/scm/container/ContainerReportHandler.java | 38 -
.../scm/container/ContainerStateManagerImpl.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 562 +++++++++--
.../scm/container/balancer/ContainerBalancer.java | 4 +-
.../algorithms/SCMContainerPlacementCapacity.java | 10 +-
.../algorithms/SCMContainerPlacementRackAware.java | 102 +-
.../algorithms/SCMContainerPlacementRandom.java | 10 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 23 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 +
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 3 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 5 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 8 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 +
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 43 +-
...andler.java => HealthyReadOnlyNodeHandler.java} | 41 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 39 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 216 +++-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 14 +-
...va => ReadOnlyHealthyToHealthyNodeHandler.java} | 15 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 170 +++-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 10 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 15 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 12 +-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 25 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 49 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +-
.../scm/pipeline/PipelineStateManagerV2Impl.java | 6 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 29 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 63 ++
.../hdds/scm/server/SCMClientProtocolServer.java | 38 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 31 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 24 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 1 +
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 9 +-
.../hdds/scm/server/StorageContainerManager.java | 190 ++--
.../scm/server/StorageContainerManagerStarter.java | 12 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 163 +++
.../ScmHAUnfinalizedStateValidationAction.java | 72 ++
.../ScmOnFinalizeActionForDatanodeSchemaV2.java | 46 +
.../hdds/scm/server/upgrade}/package-info.java | 5 +-
.../update/server/SCMUpdateServiceGrpcServer.java | 3 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 19 +
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 96 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 26 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 40 +-
.../scm/container/TestContainerReportHandler.java | 4 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../hdds/scm/container/TestReplicationManager.java | 455 +++++++--
.../algorithms/TestContainerPlacementFactory.java | 41 +-
.../TestSCMContainerPlacementCapacity.java | 6 +-
.../TestSCMContainerPlacementRackAware.java | 112 +-
.../TestSCMContainerPlacementRandom.java | 6 +-
.../hdds/scm/node/TestContainerPlacement.java | 23 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 2 +-
.../hdds/scm/node/TestNodeReportHandler.java | 16 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 57 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 532 +++++++++-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 17 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 26 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 10 +
.../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 44 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 157 ++-
.../hdds/scm/update/server/MockCRLStore.java | 6 +-
.../TestSCMHAUnfinalizedStateValidationAction.java | 128 +++
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 75 ++
.../hadoop/hdds/scm/upgrade/package-info.java | 21 +
.../ozone/container/common/TestEndPoint.java | 20 +-
.../placement/TestContainerPlacement.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 23 +-
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 63 +-
.../main/java/org/apache/ozone/test/TestClock.java | 76 ++
hadoop-hdds/tools/pom.xml | 12 +-
.../hdds/scm/cli/ContainerOperationClient.java | 16 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 +
.../scm/cli/datanode/TestListInfoSubcommand.java | 17 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 16 +
.../org/apache/hadoop/ozone/client/OzoneKey.java | 4 +
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +
.../ozone/client/protocol/ClientProtocol.java | 16 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 21 +
hadoop-ozone/common/pom.xml | 4 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 31 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../apache/hadoop/ozone/om/OmUpgradeConfig.java | 82 ++
.../hadoop/ozone/om/exceptions/OMException.java | 10 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 67 +-
.../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 11 +
.../hadoop/ozone/om/helpers/WithObjectID.java | 6 +-
.../ozone/om/protocol/OMInterServiceProtocol.java | 32 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 105 ++
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +-
.../OMInterServiceProtocolClientSideImpl.java | 121 +++
.../om/protocolPB/OMInterServiceProtocolPB.java | 52 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 111 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 4 +-
.../main/resources/ozone-version-info.properties | 4 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 31 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 3 +-
hadoop-ozone/csi/pom.xml | 6 +-
hadoop-ozone/dev-support/checks/_lib.sh | 9 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +-
hadoop-ozone/dist/pom.xml | 2 +-
.../1.0.0/test.sh => ozone-mr/hadoop33/.env} | 13 +-
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 ++
.../test.sh => ozone-mr/hadoop33/docker-config} | 12 +-
.../compose/{upgrade => ozone-mr/hadoop33}/test.sh | 31 +-
.../compose/{upgrade => ozone-om-prepare}/.env | 10 +-
.../docker-compose.yaml | 75 +-
.../{upgrade => ozone-om-prepare}/docker-config | 28 +-
.../dist/src/main/compose/ozone-om-prepare/test.sh | 74 ++
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../src/main/compose/ozonesecure/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 22 +-
.../dist/src/main/compose/upgrade/README.md | 95 +-
.../src/main/compose/upgrade/{ => compose/ha}/.env | 7 +-
.../upgrade/{ => compose/ha}/docker-compose.yaml | 120 ++-
.../compose/upgrade/{ => compose/ha}/docker-config | 30 +-
.../upgrade/{0.5.0/test.sh => compose/ha/load.sh} | 14 +-
.../main/compose/upgrade/{ => compose/non-ha}/.env | 2 +-
.../{ => compose/non-ha}/docker-compose.yaml | 99 +-
.../upgrade/{ => compose/non-ha}/docker-config | 13 +-
.../{0.5.0/test.sh => compose/non-ha/load.sh} | 13 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 30 +-
.../dist/src/main/compose/upgrade/testlib.sh | 135 ++-
.../src/main/compose/upgrade/upgrade_to_current.sh | 51 -
.../manual-upgrade/0.5.0-1.1.0/callback.sh} | 26 +-
.../manual-upgrade/driver.sh} | 63 +-
.../non-rolling-upgrade/1.0.0-1.1.0/callback.sh | 79 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 101 ++
.../dist/src/main/compose/versions/0.5.0.sh | 4 -
.../dist/src/main/compose/versions/1.0.0.sh | 4 -
.../dist/src/main/compose/versions/1.1.0.sh | 4 -
.../dist/src/main/compose/versions/README.md | 3 -
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 8 +
.../license/bin/licenses/LICENSE-org.aspectj.html | 91 ++
.../LICENSE-org.reflections-reflections.txt | 14 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 69 +-
.../src/main/smoketest/admincli/datanode.robot | 18 +
.../main/smoketest/omha/om-cancel-prepare.robot | 43 +
.../dist/src/main/smoketest/omha/om-prepare.robot | 49 +
.../bucketcreate.robot => omha/om-prepared.robot} | 29 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +-
.../dist/src/main/smoketest/spnego/web.robot | 2 +-
.../bucketcreate.robot => upgrade/check-mlv.robot} | 30 +-
.../bucketcreate.robot => upgrade/finalize.robot} | 30 +-
.../bucketcreate.robot => upgrade/generate.robot} | 26 +-
.../bucketcreate.robot => upgrade/prepare.robot} | 25 +-
.../bucketcreate.robot => upgrade/validate.robot} | 23 +-
.../dist/src/shell/ozone/ozone-functions.sh | 2 +-
.../dist/src/shell/upgrade/{v1.sh => 1.0.0.sh} | 2 +-
.../upgrade/{v1 => 1.0.0}/01-migrate-scm-db.sh | 0
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +-
hadoop-ozone/integration-test/pom.xml | 3 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 1067 ++++++++++++++++++++
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 25 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 52 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 263 ++++-
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 7 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 +
.../hadoop/ozone/TestStorageContainerManager.java | 37 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 13 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 13 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 17 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestWatchForCommit.java | 3 +-
.../ozone/container/TestContainerReplication.java | 81 +-
.../apache/hadoop/ozone/container/TestHelper.java | 6 +-
.../commandhandler/TestBlockDeletion.java | 313 +++---
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 254 +++++
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 1 -
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 80 ++
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 ++++
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 +
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 510 ++++++++++
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 1 +
.../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 1 -
.../dev-support/findbugsExcludeFile.xml | 6 +
hadoop-ozone/interface-client/pom.xml | 62 +-
.../src/main/proto/OmClientProtocol.proto | 83 ++
.../src/main/proto/OmInterServiceProtocol.proto | 64 ++
.../interface-client/src/main/proto/Security.proto | 1 +
hadoop-ozone/interface-storage/pom.xml | 2 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-ozone/ozone-manager/pom.xml | 43 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 47 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 1 -
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 +
.../apache/hadoop/ozone/om/OMStarterInterface.java | 4 +
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 7 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 30 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 621 +++++++++---
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 311 ++++++
.../hadoop/ozone/om/OzoneManagerStarter.java | 79 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 21 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 15 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 405 +++++---
.../ozone/om/ratis/OzoneManagerStateMachine.java | 87 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 97 ++
.../hadoop/ozone/om/request/OMClientRequest.java | 7 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 1 +
.../request/bucket/OMBucketSetPropertyRequest.java | 1 +
.../om/request/bucket/acl/OMBucketAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 12 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 8 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 7 -
.../om/request/file/OMDirectoryCreateRequest.java | 1 +
.../ozone/om/request/file/OMFileCreateRequest.java | 7 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 1 +
.../ozone/om/request/key/OMKeyDeleteRequest.java | 1 +
.../ozone/om/request/key/OMKeyPurgeRequest.java | 1 +
.../ozone/om/request/key/OMKeyRenameRequest.java | 1 +
.../om/request/key/acl/OMKeyAddAclRequest.java | 1 +
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 1 +
.../om/request/key/acl/OMKeySetAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixAddAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixSetAclRequest.java | 1 +
.../S3InitiateMultipartUploadRequest.java | 1 +
.../S3MultipartUploadCommitPartRequest.java | 11 +-
.../S3MultipartUploadCompleteRequest.java | 1 +
.../om/request/s3/security/S3GetSecretRequest.java | 1 +
.../security/OMCancelDelegationTokenRequest.java | 2 +-
.../security/OMGetDelegationTokenRequest.java | 2 +-
.../security/OMRenewDelegationTokenRequest.java | 1 +
.../om/request/upgrade/OMCancelPrepareRequest.java | 97 ++
.../request/upgrade/OMFinalizeUpgradeRequest.java | 116 +++
.../ozone/om/request/upgrade/OMPrepareRequest.java | 257 +++++
.../security => request/upgrade}/package-info.java | 15 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 2 +
.../om/request/volume/OMVolumeSetOwnerRequest.java | 1 +
.../om/request/volume/OMVolumeSetQuotaRequest.java | 1 +
.../hadoop/ozone/om/response/CleanupTableInfo.java | 10 +-
.../om/response/key/OMOpenKeysDeleteRequest.java | 1 +
.../om/response/s3/security/package-info.java | 2 +-
.../response/upgrade/OMCancelPrepareResponse.java | 46 +
.../upgrade/OMFinalizeUpgradeResponse.java | 58 ++
.../om/response/upgrade/OMPrepareResponse.java | 60 ++
.../{s3/security => upgrade}/package-info.java | 15 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +-
.../BelongsToLayoutVersion.java} | 23 +-
.../DisallowedUntilLayoutVersion.java} | 31 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 80 ++
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 114 +++
.../ozone/om/upgrade/OMLayoutVersionManager.java | 210 ++++
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 50 +
.../hadoop/ozone/om/upgrade/OmUpgradeAction.java | 19 +-
.../hadoop/ozone/om/upgrade/UpgradeActionOm.java | 37 +
.../s3/security => upgrade}/package-info.java | 7 +-
.../OMInterServiceProtocolServerSideImpl.java | 90 ++
...OzoneManagerProtocolServerSideTranslatorPB.java | 87 +-
.../protocolPB/OzoneManagerRequestHandler.java | 70 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 9 +
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 34 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 108 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 5 +
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
.../upgrade/TestOMCancelPrepareRequest.java | 110 ++
.../request/volume/TestOMVolumeDeleteRequest.java | 2 +-
.../om/request/volume/TestOMVolumeRequest.java | 9 +-
.../ozone/om/response/TestCleanupTableInfo.java | 26 +-
.../hadoop/ozone/om/upgrade/MockOmRequest.java | 23 +-
.../ozone/om/upgrade/OMLayoutFeatureUtil.java | 66 ++
.../om/upgrade/TestOMLayoutFeatureAspect.java | 99 ++
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 290 ++++++
.../ozone/om/upgrade/TestOMVersionManager.java | 195 ++++
.../TestOmVersionManagerRequestFactory.java | 111 ++
.../om/upgrade/TestOzoneManagerPrepareState.java | 283 ++++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 5 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +-
hadoop-ozone/recon/pom.xml | 5 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 7 +
.../hadoop/ozone/recon/ReconControllerModule.java | 20 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 29 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 +
.../ozone/recon/api/ClusterStateEndpoint.java | 3 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 13 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 15 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 13 +
.../hadoop/ozone/recon/api/types/NSSummary.java | 86 ++
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 101 ++
.../ozone/recon/scm/ReconContainerManager.java | 8 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 139 ++-
.../scm/ReconStorageContainerManagerFacade.java | 16 +-
...der.java => ReconContainerMetadataManager.java} | 9 +-
.../recon/spi/ReconNamespaceSummaryManager.java | 22 +-
...java => ReconContainerMetadataManagerImpl.java} | 54 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 13 +-
...ntainerDBProvider.java => ReconDBProvider.java} | 86 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 72 ++
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 35 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 25 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 303 ++++++
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 +
.../src/views/datanodes/datanodes.tsx | 8 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 77 ++
.../hadoop/ozone/recon/ReconTestInjector.java | 20 +-
.../ozone/recon/api/TestContainerEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 13 +-
.../ozone/recon/api/TestOpenContainerCount.java | 7 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../scm/AbstractReconContainerManagerTest.java | 17 +-
...TestReconIncrementalContainerReportHandler.java | 14 +-
.../ozone/recon/scm/TestReconNodeManager.java | 57 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 23 +-
... => TestReconContainerMetadataManagerImpl.java} | 144 +--
...nerDBProvider.java => TestReconDBProvider.java} | 13 +-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 104 ++
.../recon/tasks/TestContainerKeyMapperTask.java | 75 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 480 +++++++++
hadoop-ozone/s3gateway/pom.xml | 5 +
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 11 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 4 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 20 +
.../hadoop/ozone/s3/util/TestContinueToken.java | 20 +
.../ozone/admin/om/CancelPrepareSubCommand.java | 53 +
.../admin/om/FinalizationStatusSubCommand.java | 64 ++
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 171 ++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 44 +-
.../hadoop/ozone/admin/om/PrepareSubCommand.java | 173 ++++
.../admin/scm/FinalizationScmStatusSubcommand.java | 51 +
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 156 +++
.../admin/scm/FinalizeUpgradeCommandUtil.java | 92 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 4 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../freon/FollowerAppendLogEntryGenerator.java | 4 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 70 +-
.../containergenerator/GeneratorDatanode.java | 60 +-
.../containergenerator/TestGeneratorDatanode.java | 71 ++
pom.xml | 22 +-
517 files changed, 21808 insertions(+), 3725 deletions(-)
diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
index 98bd157,0cfbb5a..c251c46
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
@@@ -154,10 -148,8 +149,10 @@@ public class BlockOutputStream extends
this.bufferPool = bufferPool;
this.token = token;
+ replicationIndex = pipeline.getReplicaIndex(pipeline.getClosestNode());
+
//number of buffers used before doing a flush
- refreshCurrentBuffer(bufferPool);
+ refreshCurrentBuffer();
flushPeriod = (int) (config.getStreamBufferFlushSize() / config
.getStreamBufferSize());
@@@ -219,27 -210,6 +213,30 @@@
return ioException.get();
}
++<<<<<<< HEAD
+ XceiverClientSpi getXceiverClientSpi(){
+ return this.xceiverClient;
+ }
+
+ BlockData.Builder getContainerBlockData(){
+ return this.containerBlockData;
+ }
+
+ Token<? extends TokenIdentifier> getToken(){
+ return this.token;
+ }
+
+ ExecutorService getResponseExecutor(){
+ return this.responseExecutor;
+ }
+
+ @VisibleForTesting
+ public Map<Long, List<ChunkBuffer>> getCommitIndex2flushedDataMap() {
+ return commitWatcher.getCommitIndex2flushedDataMap();
+ }
+
++=======
++>>>>>>> master
@Override
public void write(int b) throws IOException {
checkOpen();
@@@ -580,16 -557,10 +584,10 @@@
}
}
- private void waitOnFlushFutures()
- throws InterruptedException, ExecutionException {
- CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(
- commitWatcher.getFutureMap().values().toArray(
- new CompletableFuture[commitWatcher.getFutureMap().size()]));
- // wait for all the transactions to complete
- combinedFuture.get();
+ void waitOnFlushFutures() throws InterruptedException, ExecutionException {
}
- private void validateResponse(
+ void validateResponse(
ContainerProtos.ContainerCommandResponseProto responseProto)
throws IOException {
try {
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 638180c,7b375f4..fbf034e
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@@ -20,9 -20,9 +20,10 @@@ package org.apache.hadoop.hdds.scm.pipe
import java.io.IOException;
import java.time.Instant;
+ import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java
index adc75f4,a0f7e0f..ccddb11
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java
@@@ -160,10 -156,17 +165,18 @@@ public class PipelineManagerV2Impl impl
throw new IOException("Pipeline creation is not allowed as safe mode " +
"prechecks have not yet passed");
}
+
+ if (freezePipelineCreation.get()) {
+ String message = "Cannot create new pipelines while pipeline creation " +
+ "is frozen.";
+ LOG.info(message);
+ throw new IOException(message);
+ }
+
lock.lock();
try {
- Pipeline pipeline = pipelineFactory.create(replicationConfig);
+ Pipeline pipeline = pipelineFactory.create(replicationConfig,
+ excludedNodes, favoredNodes);
stateManager.addPipeline(pipeline.getProtobufMessage(
ClientVersions.CURRENT_VERSION));
recordMetricsForPipeline(pipeline);
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index 8f4ed85,9cc3954..85ab1cc
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@@ -144,11 -146,13 +155,13 @@@ public class RatisPipelineProvide
replicationConfig.getReplicationFactor();
switch (factor) {
case ONE:
- dns = pickNodesNeverUsed(replicationConfig);
+ dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes,
+ containerSizeBytes);
break;
case THREE:
- dns = placementPolicy.chooseDatanodes(null,
- null, factor.getNumber(), minRatisVolumeSizeBytes,
+ dns = placementPolicy.chooseDatanodes(excludedNodes,
- favoredNodes, factor.getNumber(), 0);
++ favoredNodes, factor.getNumber(), minRatisVolumeSizeBytes,
+ containerSizeBytes);
break;
default:
throw new IllegalStateException("Unknown factor: " + factor.name());
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 3ecd965,31b8115..4ef5fcd
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@@ -279,10 -277,15 +286,16 @@@ public class SCMPipelineManager impleme
throw new IOException("Pipeline creation is not allowed as safe mode " +
"prechecks have not yet passed");
}
+ if (freezePipelineCreation.get()) {
+ LOG.debug("Pipeline creation is frozen while an upgrade is in " +
+ "progress");
+ throw new IOException("Pipeline creation is frozen while an upgrade " +
+ "is in progress");
+ }
lock.writeLock().lock();
try {
- Pipeline pipeline = pipelineFactory.create(replicationConfig);
+ Pipeline pipeline = pipelineFactory.create(replicationConfig,
+ excludedNodes, favoredNodes);
if (pipelineStore != null) {
pipelineStore.put(pipeline.getId(), pipeline);
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
index 8a22947,147f773..4daf8c2
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
@@@ -41,15 -41,7 +41,15 @@@ public class SimplePipelineProvide
@Override
public Pipeline create(StandaloneReplicationConfig replicationConfig)
throws IOException {
+ return create(replicationConfig, Collections.emptyList(),
+ Collections.emptyList());
+ }
+
+ @Override
+ public Pipeline create(StandaloneReplicationConfig replicationConfig,
+ List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes)
+ throws IOException {
- List<DatanodeDetails> dns = pickNodesNeverUsed(replicationConfig);
+ List<DatanodeDetails> dns = pickNodesNotUsed(replicationConfig);
if (dns.size() < replicationConfig.getRequiredNodes()) {
String e = String
.format("Cannot create pipeline of factor %d using %d nodes.",
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
index b2cb636,376752b..2f41720
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
@@@ -38,7 -38,7 +38,8 @@@ import org.apache.hadoop.hdds.scm.net.N
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.junit.After;
+ import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index ef86c2f,7eb8977..b5fd0a7
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@@ -41,8 -41,9 +42,10 @@@ import java.util.Set
import java.util.stream.Collectors;
import static org.apache.commons.collections.CollectionUtils.intersection;
+ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
+ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
@@@ -222,30 -226,39 +228,64 @@@ public class TestRatisPipelineProvider
}
@Test
+ // Test excluded nodes work correctly. Note that for Ratis, the
+ // PipelinePlacementPolicy, which Ratis is hardcoded to use, does not consider
+ // favored nodes.
+ public void testCreateFactorTHREEPipelineWithExcludedDatanodes()
+ throws Exception {
+ init(1);
+ int healthyCount = nodeManager.getNodes(NodeStatus.inServiceHealthy())
+ .size();
+ // Add all but 3 nodes to the exclude list and ensure that the 3 picked
+ // nodes are not in the excluded list.
+ List<DatanodeDetails> excludedNodes = nodeManager
+ .getNodes(NodeStatus.inServiceHealthy()).stream()
+ .limit(healthyCount - 3).collect(Collectors.toList());
+
+ Pipeline pipeline1 = provider.create(
+ new RatisReplicationConfig(ReplicationFactor.THREE), excludedNodes,
+ Collections.EMPTY_LIST);
+
+ for (DatanodeDetails dn : pipeline1.getNodes()) {
+ assertFalse(excludedNodes.contains(dn));
+ }
+ }
+
+
++ @Test
+ public void testCreatePipelinesWhenNotEnoughSpace() throws Exception {
+ String expectedErrorSubstring = "Unable to find enough" +
+ " nodes that meet the space requirement";
+
+ // Use large enough container or metadata sizes that no node will have
+ // enough space to hold one.
+ OzoneConfiguration largeContainerConf = new OzoneConfiguration();
+ largeContainerConf.set(OZONE_SCM_CONTAINER_SIZE, "100TB");
+ init(1, largeContainerConf);
+ for (ReplicationFactor factor: ReplicationFactor.values()) {
+ try {
+ provider.create(new RatisReplicationConfig(factor));
+ Assert.fail("Expected SCMException for large container size with " +
+ "replication factor " + factor.toString());
+ } catch(SCMException ex) {
+ Assert.assertTrue(ex.getMessage().contains(expectedErrorSubstring));
+ }
+ }
+
+ OzoneConfiguration largeMetadataConf = new OzoneConfiguration();
+ largeMetadataConf.set(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, "100TB");
+ init(1, largeMetadataConf);
+ for (ReplicationFactor factor: ReplicationFactor.values()) {
+ try {
+ provider.create(new RatisReplicationConfig(factor));
+ Assert.fail("Expected SCMException for large metadata size with " +
+ "replication factor " + factor.toString());
+ } catch(SCMException ex) {
+ Assert.assertTrue(ex.getMessage().contains(expectedErrorSubstring));
+ }
+ }
+ }
+
private void addPipeline(
List<DatanodeDetails> dns,
Pipeline.PipelineState open, ReplicationConfig replicationConfig)
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
index 21c88cf,548587c..3fa339a
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
@@@ -95,7 -96,9 +96,13 @@@ public class BlockOutputStreamEntry ext
*/
private void checkStream() throws IOException {
if (this.outputStream == null) {
++<<<<<<< HEAD
+ this.outputStream = createOutputStream();
++=======
+ this.outputStream =
+ new RatisBlockOutputStream(blockID, xceiverClientManager,
+ pipeline, bufferPool, config, token);
++>>>>>>> master
}
}
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 28aa93d,7c8b785..7743ab7
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@@ -25,7 -25,8 +25,9 @@@ import java.util.Map
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+ import org.apache.hadoop.hdds.protocol.proto.HddsProtos
+ .UpgradeFinalizationStatus;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.tracing.TracingUtil;
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 07c6767,41604a3..3706a62
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -746,7 -769,9 +769,10 @@@ message KeyArgs
// This will be set when user performs delete directory recursively.
optional bool recursive = 17;
+
+ // When it is a head operation which is to check whether key exist
+ optional bool headOp = 18;
+ optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 18;
}
message KeyLocation {
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index 4dfedd2,6bcf706..93b7787
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@@ -24,7 -24,8 +24,9 @@@ import java.util.Random
import java.util.UUID;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+ import org.apache.hadoop.ozone.OzoneConfigKeys;
+ import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.KeyManagerImpl;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org