You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2022/10/13 06:36:05 UTC
[ozone] branch HDDS-6517-Snapshot updated (6eb19a7710 -> 4be3864675)
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a change to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 6eb19a7710 HDDS-6855. [SNAPSHOTS] Path Based Access for Ozone Snapshots. (#3729)
add 7d50d29d51 HDDS-7066. Include number of tests requested in Freon output (#3638)
add 8240341efd HDDS-6997. Add support for "echo" lookup to measure raw performance in OM (#3614)
add 1afaa652c1 HDDS-7047. Add Recon to ozone-topology sample compose environment (#3641)
add b696473437 HDDS-7109. SCM should close LayoutVersionManager in stopping stage (#3666)
add fded237013 HDDS-7053. Add client-side pipelines distribution metrics (#3626)
add dd699d952d HDDS-7043. Disallow ellipsis in PR title (#3651)
add 9583e47331 HDDS-6581. Introduce KEY_PATH_LOCK in OMKeyCreateRequest class (#3560)
add 64032cc223 HDDS-7022. EC: Open EC container are not closed when SCM container was already closed. (#3668)
add 658e139b45 HDDS-7116. Avoid leaking RocksObject from DBProfile (#3673)
add 13ccc5f8d3 HDDS-5504. Refactor namespace summaries (du, dist & counts) for upcoming legacy FS buckets support (#3309)
add 5adbef2135 HDDS-6127. file checksum to support both CRC32 and CRC32C. (#3570)
add 5ba6728f7c HDDS-6726. Close RocksObject in Recon and tools (#3678)
add f9bd49e043 HDDS-7102. Exclude unnecessary RangerClient dependencies (#3669)
add 2554228edc HDDS-7033. Include committed space in log for disk out of space (#3636)
add ec3d55f6b5 HDDS-7127. Fix Transparent Data Encryption documentation (#3685)
add 5f84fd1c9f HDDS-7074. DN EndpointStateMachineMBean to add getType() (#3650)
add 636d23615c HDDS-7120. Prometheus displays only one volume_io_stats per datanode (#3682)
add ab608c5d46 HDDS-7045. Election info is out of date in Recon (#3677)
add 82b6441c6f HDDS-7076. Log container file path when container cannot be written. (#3684)
add be2fe71d69 HDDS-6458. Add ReplicationConfig options for bucketArgs in MiniOzoneChaosCluster (#3686)
add 5d8573a970 HDDS-7134. NPE when Ranger client throws RangerServiceException without Status. (#3689)
add 3efb039091 HDDS-7113. Support overriding response header values (#3680)
add 749b526503 HDDS-7147. DirectoryDeletingService interval unit mismatch (#3695)
add 01afc1ff33 HDDS-7136. Memory leak due to ChunkInputStream.close() not releasing buffer (#3692)
add 20863d3148 HDDS-7139. INVALID_TOKEN is handled as internal error in S3G (#3698)
add e9689764d3 HDDS-7108. Include supported EC data-parity combinations in error message (#3665)
add 4fd698f05e Multiple RPC clients per test run. (#3696)
add 38c767fa5f HDDS-7135. ofs file input stream should support StreamCapabilities interface (#3694)
add b11c5f12f8 HDDS-5779. Create s3g bucket creation performance test (#3629)
add b2b19de090 HDDS-7154. Fixed code issues in org.apache.hadoop.hdds.client reported by sonar (#3700)
add 9fc0f9801d HDDS-7150. Freon fail-at-end option caused not shutdown (#3697)
add a1ba9eaa55 HDDS-7108. Make ReplicationConfig validation error message type-specific (#3702)
add ac45a349ef HDDS-7159. Upgrade ubuntu version from 18.04 to 20.04 (#3707)
add 5ea8e6ab3f HDDS-7157. Log the error msg explicitly in BlockDeletingService (#3704)
add df9ed54ba1 HDDS-7159. Use Java 8 for integration tests (#3708)
add 8bb3d6e7af HDDS-7151. Avoid using GeneratedMessage in non-generated code (#3699)
add 132bec2363 HDDS-7153. User defined override of ozone shell hardcoded values for log files (#3706)
add 59a795b2cc HDDS-7164. Skip installing awscli if already present (#3712)
add a0b996651d HDDS-6456. Update RocksDB Version to 7.x (#3690)
add a828dba870 HDDS-7104. Remove WARNINGs while running ozone fs/CLI commands. (#3671)
add 5c18a5624f HDDS-7162. Add Double in ConfigType (#3710)
add 3cf882e93c HDDS-6243. sourceToTargetMap in ContainerBalancer doesn't support multiple entries with same source. (#3485)
add 20c45aee39 HDDS-7161. Make Checksum.int2ByteString() zero-copy. (#3709)
add 9333f1e5bf HDDS-7166. Memory leak in Recon when replacing DB for new checkpoint (#3713)
add e1f4548d35 HDDS-7173. Fix TestReconOmMetadataManagerImpl#testUpdateOmDB (#3719)
add 1caf752269 HDDS-7172. Intermittent failure in Close pipeline smoketest (#3717)
add 93ae1cfcd5 HDDS-7132. GetFileStatus returns NULL for some paths. (#3687)
add f8ca8c5aff HDDS-6967. EC: Handle maintenance replicas in ECUnderReplicationHandler (#3661)
add 4f3e767b69 HDDS-6493. Improve TestOmKeyInfoCodec error handling (#3718)
add 2f757f6287 HDDS-7165. Remove AbstractContractUnbufferTest (#3714)
add 4585bbe89d HDDS-2954. Support admin groups (#3693)
add 48230b6fe0 HDDS-7158. ldb cli command supports to scan container V3. (#3705)
add 3dcd68ca3d HDDS-6621. Include annotation processor module in regular project hierarchy (#3326)
add 09e89fd67e HDDS-6312. Use KeyPrefixContainer table to accelerate the process of DELETE/UPDATE events (#3082)
add 537bac8887 HDDS-7176. Sort the container replica according to the replica index (#3720)
add b7c13dba3c HDDS-7163. Ozone debug container CLI supports container export. (#3715)
add 3e0cb27632 HDDS-6975. EC: Define the value of Maintenance Redundancy for EC containers (#3723)
add 919e7755ab HDDS-7186. Add support for Rocksdb user info LOG configure. (#3731)
add b093984f16 HDDS-7175. Timed out tests missing from bundle (#3721)
add abda1f2786 HDDS-7125. Inaccurate numBlocksDeletion when getTransactions (#3681)
add 492ae919a2 HDDS-7171. Add SupportedSourceVersion in annotation processors (#3724)
add a8efd9e6a7 HDDS-7200. Fixed inaccurate get request status code (#3733)
add 3d3e9eaf00 HDDS-7197. Schema V3 RocksDB instance statitics register as metrics collision (#3734)
add ec9327de59 HDDS-7201. testContainerIsReplicatedWhenAllNodesGotoMaintenance is failing frequently (#3737)
add 2ce64319a4 HDDS-7143. Recon: Fix left to right scrolling on Pipelines page (#3735)
add ab4288ccb8 HDDS-7191. Separate prop for s3 admin (#3732)
add 25dc267115 HDDS-7086. DataNode UI to show failed volumes too (#3740)
add e84aa4c4ea HDDS-2476. Share more code between metadata and data scanners. (#3727)
add 8f32c7e05a HDDS-7174. Migrate SCM integration tests to JUnit 5 (#3722)
add 4030265b0c HDDS-7204. EC: Schedule UnderReplicatedProcessor and OverReplicatedProcessor threads in RM instead of StorageContainerManager (#3738)
add 15fcb733c7 HDDS-7209. Bump Jersey2 to 2.34 (#3745)
add 64ec833e2b HDDS-7192. EC: ReplicationManager - create handlers to perform various container checks (#3743)
add 9ad75b2300 HDDS-7114. CHAINED and STAND_ALONE replication-type should be removed from help message (#3672)
add 286dffa8fa HDDS-7193. S3gateway Grpc Ranger Authorizer NPE Unhandled Exception (#3749)
add 3130b03846 HDDS-7144. Recon: Make only hostname fixed in Datanodes page. (#3736)
add 1dd5315704 HDDS-7225. Add Ratis tests for HealthCheck handlers of Replication Manager (#3757)
add 1cf5678224 HDDS-7207. Fixed typo "layload" in om-echo command description as payload (#3756)
add 1107fe711c HDDS-7221. EC: ReplicationManager - Encapsulate the under and over rep queues into a queue object (#3758)
add 9d3bd02215 HDDS-7185. Encode asterisk for parsing signature (#3726)
add d9b92e7a33 HDDS-7203. LookupKey Latency breakdown (#3742)
add 1a6b4d2d6f HDDS-7239. EC: Add a Handler for CLOSING containers in Replication Manager (#3769)
add 6c1a5ee07e HDDS-7240. List all volume operation should go through ACL check as well in order to trigger audit logging (#3770)
add 04efea814a HDDS-7251. Replace Log4j 1.x with Reload4j (#3773)
add f6f89a021c HDDS-7075. Implement FileSystem listStatusIterator to support file list paging (#3711)
add c7038e08a8 HDDS-7039. EC: Handle the placement policy check in ECUnderReplicationHandler (#3645)
add a8df5c530b HDDS-7145. Recon: Show last OM DB sync time on Overview page (#3765)
add 69695eabb7 HDDS-6492. Add metric for failed container moves (#3751)
add d22e42a2ac HDDS-7160. Recon: Make Container and Bucket stats clickable in Overviews page (#3775)
add cb886035ac HDDS-7257. Bump snakeyaml to 1.32 (#3776)
add 55efab1b3e HDDS-6863. Add Group-Id & Ratis-Roles Information for OM UI (#3520)
add 718f7309a8 HDDS-7222. Recon Inactive pipelines not shown. (#3753)
add efb3a80f73 HDDS-7232. Introduce container-level lock while handling container report events by SCM and Recon (#3766)
add c96a342448 HDDS-7263. Add a handler for Quasi Closed containers to RM (#3785)
add 1ff97328b7 HDDS-7206. Change the placement policy interface to allow existing nodes to be specified for Rack Scatter Policy. (#3744)
add cdd0fb4ce1 HDDS-7229. Introduce container location cache in ScmClient (#3771)
add 8dbbb4eb5f HDDS-7268. EC: Fix tests for HealthCheck handlers of RM that use Replica Indexes for Ratis Containers (#3793)
add d029910e75 HDDS-7283. Set coreSize equal to maxSize for threadPoolExecutor (#3792)
add 649d713817 HDDS-7235. EOFException occurs when executing TPC-DS using o3fs (#3764)
add 3b6ae4f761 HDDS-7285. Fix concurrency issue in DeleteBlocksCommandHandler (#3796)
add 23a0f743bc HDDS-7219. Remove not applicable info from ozone bucket list for links (#3777)
add 1585f660c1 HDDS-7250. Add HttpServer metrics (#3772)
add 0aabe8abb6 HDDS-7282. Add EstimatedKeyCount metrics for SCM DB (#3791)
add 4e364001ac HDDS-7211. Remove dead code from KeyManagerImpl (#3747)
new 4be3864675 Merge branch 'master' into HDDS-6517-Snapshot
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/close-pending.yaml | 2 +-
.github/workflows/comments.yaml | 2 +-
.github/workflows/post-commit.yml | 26 +-
dev-support/annotations/pom.xml | 114 --
.../services/javax.annotation.processing.Processor | 17 -
dev-support/ci/pr_title_check.bats | 8 +
dev-support/ci/pr_title_check.sh | 2 +
hadoop-hdds/annotations/pom.xml | 34 +
.../annotations/ReplicateAnnotationProcessor.java | 3 +
.../RequestFeatureValidatorProcessor.java | 0
.../org/apache/ozone/annotations/package-info.java | 0
hadoop-hdds/client/pom.xml | 2 +-
.../hadoop/hdds/scm/ContainerClientMetrics.java | 138 +++
.../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 10 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 3 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 6 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 7 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 2 +-
.../client/io/ECBlockReconstructedInputStream.java | 2 +-
.../hdds/scm/TestContainerClientMetrics.java | 116 ++
.../storage/TestBlockOutputStreamCorrectness.java | 4 +-
.../hdds/scm/storage/TestChunkInputStream.java | 9 +-
.../ozone/client/io/TestECBlockInputStream.java | 3 +-
hadoop-hdds/common/pom.xml | 12 +-
.../hdds/client/DefaultReplicationConfig.java | 2 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 2 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 9 -
.../org/apache/hadoop/hdds/client/QuotaList.java | 19 +-
.../hadoop/hdds/client/RatisReplicationConfig.java | 2 +-
.../hadoop/hdds/client/ReplicationConfig.java | 4 +-
.../hdds/client/ReplicationConfigValidator.java | 32 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 11 +
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 5 +-
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 18 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 3 +-
.../protocol/StorageContainerLocationProtocol.java | 2 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 +
.../org/apache/hadoop/ozone/common/Checksum.java | 3 +-
.../services/javax.annotation.processing.Processor | 2 +
.../common/src/main/resources/ozone-default.xml | 73 +-
.../hdds/client/TestECReplicationConfig.java | 29 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 36 +-
.../client/TestReplicationConfigValidator.java | 14 +-
.../hadoop/hdds/conf/SimpleConfiguration.java | 13 +
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 10 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 +
.../hadoop/hdds/conf/ConfigFileGenerator.java | 3 +
.../org/apache/hadoop/hdds/conf/ConfigType.java | 3 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 9 +
.../hadoop/hdds/conf/ConfigurationTarget.java | 4 +
.../hadoop/hdds/conf/ConfigurationExample.java | 13 +
.../hdds/conf/TestConfigurationReflectionUtil.java | 4 +
.../dev-support/findbugsExcludeFile.xml | 2 +-
hadoop-hdds/container-service/pom.xml | 2 +-
.../datanode/metadata/DatanodeCRLStoreImpl.java | 13 +-
.../container/common/report/ReportManager.java | 4 +-
.../container/common/report/ReportPublisher.java | 6 +-
.../common/report/ReportPublisherFactory.java | 6 +-
.../common/statemachine/DatanodeConfiguration.java | 107 ++
.../common/statemachine/EndpointStateMachine.java | 16 +-
.../statemachine/EndpointStateMachineMBean.java | 2 +
.../common/statemachine/StateContext.java | 56 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 32 +-
.../DeleteContainerCommandHandler.java | 11 +-
.../states/endpoint/HeartbeatEndpointTask.java | 6 +-
.../container/common/utils/HddsVolumeUtil.java | 8 +-
.../common/utils/db/DatanodeDBProfile.java | 21 +-
.../common/volume/AvailableSpaceFilter.java | 84 ++
.../volume/CapacityVolumeChoosingPolicy.java | 42 +-
.../ozone/container/common/volume/HddsVolume.java | 14 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 22 +-
.../common/volume/VolumeChoosingUtil.java | 49 +
.../container/common/volume/VolumeIOStats.java | 10 +-
.../container/common/volume/VolumeInfoMetrics.java | 5 +
.../ECReconstructionCoordinator.java | 8 +-
.../container/keyvalue/KeyValueContainer.java | 9 +-
.../keyvalue/impl/ChunkManagerFactory.java | 12 +-
.../background/BlockDeletingService.java | 27 +-
.../container/metadata/AbstractDatanodeStore.java | 40 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 2 +-
.../ozoneimpl/AbstractContainerScanner.java | 136 +++
...s.java => AbstractContainerScannerMetrics.java} | 42 +-
.../container/ozoneimpl/ContainerDataScanner.java | 135 +--
.../ozoneimpl/ContainerDataScannerMetrics.java | 69 ++
.../ozoneimpl/ContainerDataScrubberMetrics.java | 120 --
.../ozoneimpl/ContainerMetadataScanner.java | 112 +-
.../ozoneimpl/ContainerMetadataScannerMetrics.java | 44 +
...ion.java => ContainerScannerConfiguration.java} | 12 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 9 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 3 +-
.../protocol/commands/CommandForDatanode.java | 4 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 4 +-
.../webapps/hddsDatanode/dn-overview.html | 2 +
.../common/report/TestReportPublisher.java | 6 +-
.../common/statemachine/TestStateContext.java | 22 +-
.../container/common/utils/TestHddsVolumeUtil.java | 8 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 21 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 33 +-
.../TestVolumeIOStatsWithPrometheusSink.java | 92 ++
.../container/keyvalue/TestKeyValueContainer.java | 36 -
.../keyvalue/TestKeyValueContainerCheck.java | 15 +-
...java => TestContainerScannerConfiguration.java} | 28 +-
...trics.java => TestContainerScannerMetrics.java} | 20 +-
hadoop-hdds/docs/content/concept/Recon.md | 2 +-
hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 12 +-
.../docs/content/security/SecuringTDE.zh.md | 6 +-
hadoop-hdds/framework/pom.xml | 16 +-
...inerLocationProtocolClientSideTranslatorPB.java | 2 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 87 ++
.../hadoop/hdds/server/http/BaseHttpServer.java | 8 +-
.../hadoop/hdds/server/http/HttpServer2.java | 3 +
.../hdds/server/http/HttpServer2Metrics.java | 94 ++
.../hadoop/hdds/utils/DBCheckpointServlet.java | 11 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 30 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 22 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 19 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 57 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 7 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 18 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 25 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 45 +-
.../apache/hadoop/hdds/utils/db/TableConfig.java | 14 +-
.../db/managed/ManagedBlockBasedTableConfig.java | 74 ++
.../db/managed/ManagedColumnFamilyOptions.java | 37 +-
.../utils/db/managed/ManagedRocksObjectUtils.java | 2 +-
...dColumnFamilyOptions.java => ManagedSlice.java} | 28 +-
.../server/http/TestHttpServer2MetricsTest.java | 94 ++
hadoop-hdds/hadoop-dependency-client/pom.xml | 14 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 24 +
hadoop-hdds/pom.xml | 7 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 51 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 12 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 6 -
.../hdds/scm/container/ContainerReplica.java | 2 +-
.../scm/container/ContainerStateManagerImpl.java | 178 +--
.../scm/container/balancer/ContainerBalancer.java | 179 +--
.../balancer/ContainerBalancerMetrics.java | 29 +
.../algorithms/SCMContainerPlacementCapacity.java | 12 +-
.../algorithms/SCMContainerPlacementRackAware.java | 14 +-
.../SCMContainerPlacementRackScatter.java | 264 +++--
.../algorithms/SCMContainerPlacementRandom.java | 13 +-
.../replication/ContainerCheckRequest.java | 120 ++
.../replication/ContainerHealthCheck.java | 35 -
.../replication/ContainerHealthResult.java | 9 +-
.../replication/ECContainerHealthCheck.java | 86 --
.../replication/ECContainerReplicaCount.java | 48 +-
.../replication/ECOverReplicationHandler.java | 22 +-
.../replication/ECUnderReplicationHandler.java | 191 ++-
.../replication/LegacyReplicationManager.java | 4 +-
.../replication/OverReplicatedProcessor.java | 37 +-
.../container/replication/ReplicationManager.java | 252 ++--
.../container/replication/ReplicationQueue.java | 73 ++
.../replication/UnderReplicatedProcessor.java | 37 +-
.../replication/health/AbstractCheck.java | 71 ++
.../ClosedWithMismatchedReplicasHandler.java | 89 ++
.../health/ClosingContainerHandler.java | 62 +
.../health/ECReplicationCheckHandler.java | 134 +++
.../container/replication/health/HealthCheck.java | 56 +
.../replication/health/OpenContainerHandler.java | 81 ++
.../health/QuasiClosedContainerHandler.java | 131 ++
.../container/replication/health/package-info.java | 10 +-
.../scm/container/states/ContainerAttribute.java | 14 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 3 +
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 12 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 11 +-
.../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 2 +-
.../hdds/scm/ha/io/GeneratedMessageCodec.java | 12 +-
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 2 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 51 +-
.../hdds/scm/metadata/SCMMetadataStoreMetrics.java | 116 ++
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 10 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 5 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 2 +-
.../hdds/scm/server/SCMDBCheckpointServlet.java | 1 +
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 52 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 154 ++-
.../container/balancer/TestContainerBalancer.java | 189 +--
.../algorithms/TestContainerPlacementFactory.java | 4 +-
.../TestSCMContainerPlacementRackScatter.java | 76 +-
.../container/replication/ReplicationTestUtil.java | 56 +-
.../replication/TestECContainerReplicaCount.java | 84 +-
.../replication/TestECOverReplicationHandler.java | 5 +-
.../replication/TestECUnderReplicationHandler.java | 100 +-
.../replication/TestOverReplicatedProcessor.java | 7 +-
.../replication/TestReplicationManager.java | 102 +-
.../replication/TestUnderReplicatedProcessor.java | 7 +-
.../TestClosedWithMismatchedReplicasHandler.java | 217 ++++
.../health/TestClosingContainerHandler.java | 208 ++++
.../TestECReplicationCheckHandler.java} | 199 +++-
.../health/TestOpenContainerHandler.java | 168 +++
.../health/TestQuasiClosedContainerHandler.java | 238 ++++
.../container/replication/health/package-info.java | 11 +-
.../scm/metadata/TestSCMMetadataStoreImpl.java | 77 ++
hadoop-hdds/test-utils/pom.xml | 4 +-
.../org/apache/ozone/test/GenericTestUtils.java | 42 +
hadoop-hdds/tools/pom.xml | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 6 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 22 +-
hadoop-ozone/client/pom.xml | 6 +-
.../client/checksum/BaseFileChecksumHelper.java | 40 +-
.../checksum/ReplicatedFileChecksumHelper.java | 4 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 19 +-
.../client/io/BlockOutputStreamEntryPool.java | 20 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 16 +-
.../client/io/ECBlockOutputStreamEntryPool.java | 7 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 4 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 28 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 7 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 11 +-
hadoop-ozone/common/pom.xml | 12 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 39 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 11 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 14 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 20 +
hadoop-ozone/csi/pom.xml | 6 +-
.../dev-support/checks/_mvn_unit_report.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
.../compose/ozone-topology/docker-compose.yaml | 15 +
.../src/main/compose/ozone-topology/docker-config | 5 +
.../dist/src/main/compose/ozone-topology/test.sh | 2 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 10 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 12 +-
.../src/main/smoketest/admincli/pipeline.robot | 5 +-
.../dist/src/main/smoketest/ec/basic.robot | 6 +-
.../src/main/smoketest/freon/echoRPCLoad.robot | 42 +
.../dist/src/main/smoketest/recon/recon-api.robot | 14 +
.../main/smoketest/recon/recon-fso-nssummary.robot | 146 +++
.../dist/src/main/smoketest/s3/commonawslib.robot | 2 +
.../dist/src/main/smoketest/s3/freon.robot | 44 +
.../dist/src/main/smoketest/s3/objectputget.robot | 6 +
hadoop-ozone/dist/src/shell/ozone/ozone | 3 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 22 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 28 +-
hadoop-ozone/integration-test/pom.xml | 94 +-
.../fs/contract/AbstractContractUnbufferTest.java | 160 ---
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 80 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 203 ++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 247 ++++
.../hdds/scm/TestSCMDbCheckpointServlet.java | 31 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 32 +-
.../metrics/TestSCMContainerManagerMetrics.java | 81 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 42 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 40 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 22 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 33 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 35 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 21 +-
.../client/rpc/read/TestChunkInputStream.java | 20 +
.../ozone/container/TestECContainerRecovery.java | 84 +-
.../TestDataScanner.java} | 31 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 11 +-
.../om/{TestOmLDBCli.java => TestLDBCli.java} | 124 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 7 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 39 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 26 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 84 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 47 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 31 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 30 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 22 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 103 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 19 +-
.../scm/node/TestDecommissionAndMaintenance.java | 210 ++--
.../hadoop/ozone/scm/node/TestQueryNode.java | 29 +-
.../scm/pipeline/TestPipelineManagerMXBean.java | 35 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 28 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 69 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 21 +-
.../src/test/resources/testSequenceFile | Bin 0 -> 96 bytes
.../src/main/proto/OmClientProtocol.proto | 33 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 45 +-
hadoop-ozone/ozone-manager/pom.xml | 16 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 665 +----------
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 6 +-
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 +
.../hadoop/ozone/om/OMPerformanceMetrics.java | 92 ++
.../apache/hadoop/ozone/om/OmMetadataReader.java | 4 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 4 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 61 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 153 ++-
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 81 +-
.../om/hashcodegenerator/OMHashCodeGenerator.java | 2 +-
.../StringOMHashCodeGeneratorImpl.java | 2 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 38 +-
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 15 +-
.../ozone/om/lock/RegularBucketLockStrategy.java | 8 +-
.../RangerClientMultiTenantAccessController.java | 27 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 13 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../hadoop/ozone/om/request/OMEchoRPCRequest.java} | 25 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 11 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 6 +
.../request/s3/security/S3SecretRequestHelper.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 29 +
.../ozone/security/acl/OzoneNativeAuthorizer.java | 22 +-
.../webapps/ozoneManager/om-overview.html | 20 +
.../resources/webapps/ozoneManager/ozoneManager.js | 30 +
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 2 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 42 +
.../org/apache/hadoop/ozone/om/TestScmClient.java | 191 +++
.../om/request/key/TestOMKeyCreateRequest.java | 47 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 5 +
.../ozone/om/request/key/TestOMKeyRequest.java | 5 +-
.../s3/security/TestS3GetSecretRequest.java | 206 ++--
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../security/acl/TestOzoneAdministrators.java | 66 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 3 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 3 +-
hadoop-ozone/ozonefs-common/pom.xml | 5 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 138 ++-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 154 ++-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 6 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 6 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 +
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
hadoop-ozone/ozonefs/pom.xml | 5 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 +
.../hadoop/ozone/recon/ReconControllerModule.java | 4 +-
.../hadoop/ozone/recon/ReconRestServletModule.java | 2 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 929 +--------------
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 57 +-
.../ozone/recon/api/filters/ReconAdminFilter.java | 12 +-
.../recon/api/handlers/BucketEntityHandler.java | 152 +++
.../ozone/recon/api/handlers/BucketHandler.java | 207 ++++
.../recon/api/handlers/DirectoryEntityHandler.java | 150 +++
.../ozone/recon/api/handlers/EntityHandler.java | 342 ++++++
.../ozone/recon/api/handlers/FSOBucketHandler.java | 279 +++++
.../ozone/recon/api/handlers/KeyEntityHandler.java | 93 ++
.../recon/api/handlers/RootEntityHandler.java | 176 +++
.../recon/api/handlers/UnknownEntityHandler.java | 82 ++
.../recon/api/handlers/VolumeEntityHandler.java | 160 +++
.../EntityType.java => handlers/package-info.java} | 13 +-
.../ozone/recon/api/types/ContainerKeyPrefix.java | 9 +
.../hadoop/ozone/recon/api/types/EntityType.java | 85 +-
...ainerKeyPrefix.java => KeyPrefixContainer.java} | 50 +-
.../hadoop/ozone/recon/api/types/NSSummary.java | 7 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 10 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +-
.../scm/ReconStorageContainerManagerFacade.java | 11 +-
.../recon/spi/ReconContainerMetadataManager.java | 17 +
.../recon/spi/impl/KeyPrefixContainerCodec.java | 92 ++
.../impl/ReconContainerMetadataManagerImpl.java | 304 +++--
.../ozone/recon/spi/impl/ReconDBDefinition.java | 12 +-
.../ozone/recon/spi/impl/ReconDBProvider.java | 11 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 94 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 206 +---
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 217 ++++
.../webapps/recon/ozone-recon-web/api/db.json | 46 +-
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 44 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 8 +
.../src/views/datanodes/datanodes.tsx | 29 +-
.../src/views/overview/overview.tsx | 35 +-
.../src/views/pipelines/pipelines.less | 2 +-
.../src/views/pipelines/pipelines.tsx | 33 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 44 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 663 -----------
.../recon/api/TestNSSummaryEndpointWithFSO.java | 1249 ++++++++++++++++++++
.../ozone/recon/api/filters/TestAdminFilter.java | 23 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 11 +
.../TestReconContainerMetadataManagerImpl.java | 25 +
.../ozone/recon/tasks/TestNSSummaryTask.java | 541 ---------
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 604 ++++++++++
hadoop-ozone/s3gateway/pom.xml | 42 -
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 7 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 74 +-
.../ozone/s3/signature/StringToSignProducer.java | 1 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 153 ++-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 10 +
.../hadoop/ozone/debug/DBDefinitionFactory.java | 3 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 61 +-
.../ozone/debug/container/ContainerCommands.java | 12 +
.../ozone/debug/container/ExportSubcommand.java | 35 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 10 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 5 +-
.../ozone/freon/FreonReplicationOptions.java | 5 +-
.../hadoop/ozone/freon/OmRPCLoadGenerator.java | 120 ++
.../hadoop/ozone/freon/S3BucketGenerator.java | 73 ++
.../hadoop/ozone/freon/S3EntityGenerator.java | 65 +
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 47 +-
.../freon/containergenerator/GeneratorOm.java | 14 +-
.../hadoop/ozone/shell/ReplicationOptions.java | 18 +-
.../ozone/shell/ShellReplicationOptions.java | 3 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 2 +-
.../ozone/shell/bucket/ListBucketHandler.java | 17 +-
pom.xml | 44 +-
393 files changed, 14348 insertions(+), 6595 deletions(-)
delete mode 100644 dev-support/annotations/pom.xml
delete mode 100644 dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor
create mode 100644 hadoop-hdds/annotations/pom.xml
rename {dev-support => hadoop-hdds}/annotations/src/main/java/org/apache/ozone/annotations/ReplicateAnnotationProcessor.java (95%)
rename {dev-support => hadoop-hdds}/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java (100%)
rename {dev-support => hadoop-hdds}/annotations/src/main/java/org/apache/ozone/annotations/package-info.java (100%)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestContainerClientMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeChoosingUtil.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/AbstractContainerScanner.java
rename hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/{ContainerMetadataScrubberMetrics.java => AbstractContainerScannerMetrics.java} (68%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScannerMetrics.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScrubberMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScannerMetrics.java
rename hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/{ContainerScrubberConfiguration.java => ContainerScannerConfiguration.java} (92%)
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeIOStatsWithPrometheusSink.java
rename hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/{TestContainerScrubberConfiguration.java => TestContainerScannerConfiguration.java} (80%)
rename hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/{TestContainerScrubberMetrics.java => TestContainerScannerMetrics.java} (89%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneAdmins.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2Metrics.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/{ManagedColumnFamilyOptions.java => ManagedSlice.java} (62%)
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpServer2MetricsTest.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerCheckRequest.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthCheck.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationQueue.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/AbstractCheck.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithMismatchedReplicasHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ECReplicationCheckHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/HealthCheck.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/OpenContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedContainerHandler.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.less => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/package-info.java (86%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreMetrics.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithMismatchedReplicasHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java
rename hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/{TestECContainerHealthCheck.java => health/TestECReplicationCheckHandler.java} (53%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestOpenContainerHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedContainerHandler.java
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.less => hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/package-info.java (85%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSCMMetadataStoreImpl.java
create mode 100644 hadoop-ozone/dist/src/main/smoketest/freon/echoRPCLoad.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/recon/recon-fso-nssummary.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/freon.robot
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/{scrubber/TestDataScrubber.java => scanner/TestDataScanner.java} (95%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/{TestOmLDBCli.java => TestLDBCli.java} (54%)
create mode 100644 hadoop-ozone/integration-test/src/test/resources/testSequenceFile
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
copy hadoop-ozone/{tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java => ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMEchoRPCRequest.java} (55%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/{types/EntityType.java => handlers/package-info.java} (81%)
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/{ContainerKeyPrefix.java => KeyPrefixContainer.java} (65%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/KeyPrefixContainerCodec.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmRPCLoadGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge branch 'master' into HDDS-6517-Snapshot
Posted by na...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4be3864675ad98801cda5caabde29734420bc662
Merge: 6eb19a7710 4e364001ac
Author: Nandakumar <na...@apache.org>
AuthorDate: Thu Oct 13 08:56:23 2022 +0530
Merge branch 'master' into HDDS-6517-Snapshot
.github/workflows/close-pending.yaml | 2 +-
.github/workflows/comments.yaml | 2 +-
.github/workflows/post-commit.yml | 26 +-
dev-support/annotations/pom.xml | 114 --
.../services/javax.annotation.processing.Processor | 17 -
dev-support/ci/pr_title_check.bats | 8 +
dev-support/ci/pr_title_check.sh | 2 +
hadoop-hdds/annotations/pom.xml | 34 +
.../annotations/ReplicateAnnotationProcessor.java | 3 +
.../RequestFeatureValidatorProcessor.java | 0
.../org/apache/ozone/annotations/package-info.java | 0
hadoop-hdds/client/pom.xml | 2 +-
.../hadoop/hdds/scm/ContainerClientMetrics.java | 138 +++
.../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 10 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 3 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 6 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 7 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 2 +-
.../client/io/ECBlockReconstructedInputStream.java | 2 +-
.../hdds/scm/TestContainerClientMetrics.java | 116 ++
.../storage/TestBlockOutputStreamCorrectness.java | 4 +-
.../hdds/scm/storage/TestChunkInputStream.java | 9 +-
.../ozone/client/io/TestECBlockInputStream.java | 3 +-
hadoop-hdds/common/pom.xml | 12 +-
.../hdds/client/DefaultReplicationConfig.java | 2 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 2 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 9 -
.../org/apache/hadoop/hdds/client/QuotaList.java | 19 +-
.../hadoop/hdds/client/RatisReplicationConfig.java | 2 +-
.../hadoop/hdds/client/ReplicationConfig.java | 4 +-
.../hdds/client/ReplicationConfigValidator.java | 32 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 11 +
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 5 +-
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 18 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 3 +-
.../protocol/StorageContainerLocationProtocol.java | 2 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 +
.../org/apache/hadoop/ozone/common/Checksum.java | 3 +-
.../services/javax.annotation.processing.Processor | 2 +
.../common/src/main/resources/ozone-default.xml | 73 +-
.../hdds/client/TestECReplicationConfig.java | 29 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 36 +-
.../client/TestReplicationConfigValidator.java | 14 +-
.../hadoop/hdds/conf/SimpleConfiguration.java | 13 +
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 10 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 +
.../hadoop/hdds/conf/ConfigFileGenerator.java | 3 +
.../org/apache/hadoop/hdds/conf/ConfigType.java | 3 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 9 +
.../hadoop/hdds/conf/ConfigurationTarget.java | 4 +
.../hadoop/hdds/conf/ConfigurationExample.java | 13 +
.../hdds/conf/TestConfigurationReflectionUtil.java | 4 +
.../dev-support/findbugsExcludeFile.xml | 2 +-
hadoop-hdds/container-service/pom.xml | 2 +-
.../datanode/metadata/DatanodeCRLStoreImpl.java | 13 +-
.../container/common/report/ReportManager.java | 4 +-
.../container/common/report/ReportPublisher.java | 6 +-
.../common/report/ReportPublisherFactory.java | 6 +-
.../common/statemachine/DatanodeConfiguration.java | 107 ++
.../common/statemachine/EndpointStateMachine.java | 16 +-
.../statemachine/EndpointStateMachineMBean.java | 2 +
.../common/statemachine/StateContext.java | 56 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 32 +-
.../DeleteContainerCommandHandler.java | 11 +-
.../states/endpoint/HeartbeatEndpointTask.java | 6 +-
.../container/common/utils/HddsVolumeUtil.java | 8 +-
.../common/utils/db/DatanodeDBProfile.java | 21 +-
.../common/volume/AvailableSpaceFilter.java | 84 ++
.../volume/CapacityVolumeChoosingPolicy.java | 42 +-
.../ozone/container/common/volume/HddsVolume.java | 14 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 22 +-
.../common/volume/VolumeChoosingUtil.java | 49 +
.../container/common/volume/VolumeIOStats.java | 10 +-
.../container/common/volume/VolumeInfoMetrics.java | 5 +
.../ECReconstructionCoordinator.java | 8 +-
.../container/keyvalue/KeyValueContainer.java | 9 +-
.../keyvalue/impl/ChunkManagerFactory.java | 12 +-
.../background/BlockDeletingService.java | 27 +-
.../container/metadata/AbstractDatanodeStore.java | 40 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 2 +-
.../ozoneimpl/AbstractContainerScanner.java | 136 +++
...s.java => AbstractContainerScannerMetrics.java} | 42 +-
.../container/ozoneimpl/ContainerDataScanner.java | 135 +--
.../ozoneimpl/ContainerDataScannerMetrics.java | 69 ++
.../ozoneimpl/ContainerDataScrubberMetrics.java | 120 --
.../ozoneimpl/ContainerMetadataScanner.java | 112 +-
.../ozoneimpl/ContainerMetadataScannerMetrics.java | 44 +
...ion.java => ContainerScannerConfiguration.java} | 12 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 9 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 3 +-
.../protocol/commands/CommandForDatanode.java | 4 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 4 +-
.../webapps/hddsDatanode/dn-overview.html | 2 +
.../common/report/TestReportPublisher.java | 6 +-
.../common/statemachine/TestStateContext.java | 22 +-
.../container/common/utils/TestHddsVolumeUtil.java | 8 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 21 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 33 +-
.../TestVolumeIOStatsWithPrometheusSink.java | 92 ++
.../container/keyvalue/TestKeyValueContainer.java | 36 -
.../keyvalue/TestKeyValueContainerCheck.java | 15 +-
...java => TestContainerScannerConfiguration.java} | 28 +-
...trics.java => TestContainerScannerMetrics.java} | 20 +-
hadoop-hdds/docs/content/concept/Recon.md | 2 +-
hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 12 +-
.../docs/content/security/SecuringTDE.zh.md | 6 +-
hadoop-hdds/framework/pom.xml | 16 +-
...inerLocationProtocolClientSideTranslatorPB.java | 2 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 87 ++
.../hadoop/hdds/server/http/BaseHttpServer.java | 8 +-
.../hadoop/hdds/server/http/HttpServer2.java | 3 +
.../hdds/server/http/HttpServer2Metrics.java | 94 ++
.../hadoop/hdds/utils/DBCheckpointServlet.java | 11 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 30 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 22 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 19 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 57 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 7 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 18 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 25 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 45 +-
.../apache/hadoop/hdds/utils/db/TableConfig.java | 14 +-
.../db/managed/ManagedBlockBasedTableConfig.java | 74 ++
.../db/managed/ManagedColumnFamilyOptions.java | 37 +-
.../utils/db/managed/ManagedRocksObjectUtils.java | 2 +-
...dColumnFamilyOptions.java => ManagedSlice.java} | 28 +-
.../server/http/TestHttpServer2MetricsTest.java | 94 ++
hadoop-hdds/hadoop-dependency-client/pom.xml | 14 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 24 +
hadoop-hdds/pom.xml | 7 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 51 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 12 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 6 -
.../hdds/scm/container/ContainerReplica.java | 2 +-
.../scm/container/ContainerStateManagerImpl.java | 178 +--
.../scm/container/balancer/ContainerBalancer.java | 179 +--
.../balancer/ContainerBalancerMetrics.java | 29 +
.../algorithms/SCMContainerPlacementCapacity.java | 12 +-
.../algorithms/SCMContainerPlacementRackAware.java | 14 +-
.../SCMContainerPlacementRackScatter.java | 264 +++--
.../algorithms/SCMContainerPlacementRandom.java | 13 +-
.../replication/ContainerCheckRequest.java | 120 ++
.../replication/ContainerHealthCheck.java | 35 -
.../replication/ContainerHealthResult.java | 9 +-
.../replication/ECContainerHealthCheck.java | 86 --
.../replication/ECContainerReplicaCount.java | 48 +-
.../replication/ECOverReplicationHandler.java | 22 +-
.../replication/ECUnderReplicationHandler.java | 191 ++-
.../replication/LegacyReplicationManager.java | 4 +-
.../replication/OverReplicatedProcessor.java | 37 +-
.../container/replication/ReplicationManager.java | 252 ++--
.../container/replication/ReplicationQueue.java | 73 ++
.../replication/UnderReplicatedProcessor.java | 37 +-
.../replication/health/AbstractCheck.java | 71 ++
.../ClosedWithMismatchedReplicasHandler.java | 89 ++
.../health/ClosingContainerHandler.java | 62 +
.../health/ECReplicationCheckHandler.java | 134 +++
.../container/replication/health/HealthCheck.java | 56 +
.../replication/health/OpenContainerHandler.java | 81 ++
.../health/QuasiClosedContainerHandler.java | 131 ++
.../container/replication/health/package-info.java | 10 +-
.../scm/container/states/ContainerAttribute.java | 14 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 3 +
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 12 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 11 +-
.../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 2 +-
.../hdds/scm/ha/io/GeneratedMessageCodec.java | 12 +-
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 2 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 51 +-
.../hdds/scm/metadata/SCMMetadataStoreMetrics.java | 116 ++
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 10 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 5 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 2 +-
.../hdds/scm/server/SCMDBCheckpointServlet.java | 1 +
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 52 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 154 ++-
.../container/balancer/TestContainerBalancer.java | 189 +--
.../algorithms/TestContainerPlacementFactory.java | 4 +-
.../TestSCMContainerPlacementRackScatter.java | 76 +-
.../container/replication/ReplicationTestUtil.java | 56 +-
.../replication/TestECContainerReplicaCount.java | 84 +-
.../replication/TestECOverReplicationHandler.java | 5 +-
.../replication/TestECUnderReplicationHandler.java | 100 +-
.../replication/TestOverReplicatedProcessor.java | 7 +-
.../replication/TestReplicationManager.java | 102 +-
.../replication/TestUnderReplicatedProcessor.java | 7 +-
.../TestClosedWithMismatchedReplicasHandler.java | 217 ++++
.../health/TestClosingContainerHandler.java | 208 ++++
.../TestECReplicationCheckHandler.java} | 199 +++-
.../health/TestOpenContainerHandler.java | 168 +++
.../health/TestQuasiClosedContainerHandler.java | 238 ++++
.../container/replication/health/package-info.java | 11 +-
.../scm/metadata/TestSCMMetadataStoreImpl.java | 77 ++
hadoop-hdds/test-utils/pom.xml | 4 +-
.../org/apache/ozone/test/GenericTestUtils.java | 42 +
hadoop-hdds/tools/pom.xml | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 6 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 22 +-
hadoop-ozone/client/pom.xml | 6 +-
.../client/checksum/BaseFileChecksumHelper.java | 40 +-
.../checksum/ReplicatedFileChecksumHelper.java | 4 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 19 +-
.../client/io/BlockOutputStreamEntryPool.java | 20 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 16 +-
.../client/io/ECBlockOutputStreamEntryPool.java | 7 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 4 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 28 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 7 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 11 +-
hadoop-ozone/common/pom.xml | 12 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 39 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 11 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 14 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 20 +
hadoop-ozone/csi/pom.xml | 6 +-
.../dev-support/checks/_mvn_unit_report.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
.../compose/ozone-topology/docker-compose.yaml | 15 +
.../src/main/compose/ozone-topology/docker-config | 5 +
.../dist/src/main/compose/ozone-topology/test.sh | 2 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 10 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 12 +-
.../src/main/smoketest/admincli/pipeline.robot | 5 +-
.../dist/src/main/smoketest/ec/basic.robot | 6 +-
.../src/main/smoketest/freon/echoRPCLoad.robot | 42 +
.../dist/src/main/smoketest/recon/recon-api.robot | 14 +
.../main/smoketest/recon/recon-fso-nssummary.robot | 146 +++
.../dist/src/main/smoketest/s3/commonawslib.robot | 2 +
.../dist/src/main/smoketest/s3/freon.robot | 44 +
.../dist/src/main/smoketest/s3/objectputget.robot | 6 +
hadoop-ozone/dist/src/shell/ozone/ozone | 3 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 22 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 28 +-
hadoop-ozone/integration-test/pom.xml | 94 +-
.../fs/contract/AbstractContractUnbufferTest.java | 160 ---
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 80 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 203 ++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 247 ++++
.../hdds/scm/TestSCMDbCheckpointServlet.java | 31 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 32 +-
.../metrics/TestSCMContainerManagerMetrics.java | 81 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 42 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 40 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 22 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 33 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 35 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 21 +-
.../client/rpc/read/TestChunkInputStream.java | 20 +
.../ozone/container/TestECContainerRecovery.java | 84 +-
.../TestDataScanner.java} | 31 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 11 +-
.../om/{TestOmLDBCli.java => TestLDBCli.java} | 124 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 7 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 39 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 26 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 84 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 47 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 31 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 30 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 22 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 103 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 19 +-
.../scm/node/TestDecommissionAndMaintenance.java | 210 ++--
.../hadoop/ozone/scm/node/TestQueryNode.java | 29 +-
.../scm/pipeline/TestPipelineManagerMXBean.java | 35 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 28 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 69 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 21 +-
.../src/test/resources/testSequenceFile | Bin 0 -> 96 bytes
.../src/main/proto/OmClientProtocol.proto | 33 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 45 +-
hadoop-ozone/ozone-manager/pom.xml | 16 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 665 +----------
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 6 +-
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 +
.../hadoop/ozone/om/OMPerformanceMetrics.java | 92 ++
.../apache/hadoop/ozone/om/OmMetadataReader.java | 4 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 4 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 61 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 153 ++-
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 81 +-
.../om/hashcodegenerator/OMHashCodeGenerator.java | 2 +-
.../StringOMHashCodeGeneratorImpl.java | 2 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 38 +-
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 15 +-
.../ozone/om/lock/RegularBucketLockStrategy.java | 8 +-
.../RangerClientMultiTenantAccessController.java | 27 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 13 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../hadoop/ozone/om/request/OMEchoRPCRequest.java} | 25 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 11 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 6 +
.../request/s3/security/S3SecretRequestHelper.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 29 +
.../ozone/security/acl/OzoneNativeAuthorizer.java | 22 +-
.../webapps/ozoneManager/om-overview.html | 20 +
.../resources/webapps/ozoneManager/ozoneManager.js | 30 +
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 2 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 42 +
.../org/apache/hadoop/ozone/om/TestScmClient.java | 191 +++
.../om/request/key/TestOMKeyCreateRequest.java | 47 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 5 +
.../ozone/om/request/key/TestOMKeyRequest.java | 5 +-
.../s3/security/TestS3GetSecretRequest.java | 206 ++--
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../security/acl/TestOzoneAdministrators.java | 66 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 3 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 3 +-
hadoop-ozone/ozonefs-common/pom.xml | 5 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 138 ++-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 154 ++-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 6 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 6 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 +
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
hadoop-ozone/ozonefs/pom.xml | 5 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 +
.../hadoop/ozone/recon/ReconControllerModule.java | 4 +-
.../hadoop/ozone/recon/ReconRestServletModule.java | 2 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 929 +--------------
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 57 +-
.../ozone/recon/api/filters/ReconAdminFilter.java | 12 +-
.../recon/api/handlers/BucketEntityHandler.java | 152 +++
.../ozone/recon/api/handlers/BucketHandler.java | 207 ++++
.../recon/api/handlers/DirectoryEntityHandler.java | 150 +++
.../ozone/recon/api/handlers/EntityHandler.java | 342 ++++++
.../ozone/recon/api/handlers/FSOBucketHandler.java | 279 +++++
.../ozone/recon/api/handlers/KeyEntityHandler.java | 93 ++
.../recon/api/handlers/RootEntityHandler.java | 176 +++
.../recon/api/handlers/UnknownEntityHandler.java | 82 ++
.../recon/api/handlers/VolumeEntityHandler.java | 160 +++
.../EntityType.java => handlers/package-info.java} | 13 +-
.../ozone/recon/api/types/ContainerKeyPrefix.java | 9 +
.../hadoop/ozone/recon/api/types/EntityType.java | 85 +-
...ainerKeyPrefix.java => KeyPrefixContainer.java} | 50 +-
.../hadoop/ozone/recon/api/types/NSSummary.java | 7 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 10 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +-
.../scm/ReconStorageContainerManagerFacade.java | 11 +-
.../recon/spi/ReconContainerMetadataManager.java | 17 +
.../recon/spi/impl/KeyPrefixContainerCodec.java | 92 ++
.../impl/ReconContainerMetadataManagerImpl.java | 304 +++--
.../ozone/recon/spi/impl/ReconDBDefinition.java | 12 +-
.../ozone/recon/spi/impl/ReconDBProvider.java | 11 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 94 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 206 +---
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 217 ++++
.../webapps/recon/ozone-recon-web/api/db.json | 46 +-
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 44 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 8 +
.../src/views/datanodes/datanodes.tsx | 29 +-
.../src/views/overview/overview.tsx | 35 +-
.../src/views/pipelines/pipelines.less | 2 +-
.../src/views/pipelines/pipelines.tsx | 33 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 44 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 663 -----------
.../recon/api/TestNSSummaryEndpointWithFSO.java | 1249 ++++++++++++++++++++
.../ozone/recon/api/filters/TestAdminFilter.java | 23 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 11 +
.../TestReconContainerMetadataManagerImpl.java | 25 +
.../ozone/recon/tasks/TestNSSummaryTask.java | 541 ---------
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 604 ++++++++++
hadoop-ozone/s3gateway/pom.xml | 42 -
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 7 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 74 +-
.../ozone/s3/signature/StringToSignProducer.java | 1 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 153 ++-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 10 +
.../hadoop/ozone/debug/DBDefinitionFactory.java | 3 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 61 +-
.../ozone/debug/container/ContainerCommands.java | 12 +
.../ozone/debug/container/ExportSubcommand.java | 35 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 10 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 5 +-
.../ozone/freon/FreonReplicationOptions.java | 5 +-
.../hadoop/ozone/freon/OmRPCLoadGenerator.java | 120 ++
.../hadoop/ozone/freon/S3BucketGenerator.java | 73 ++
.../hadoop/ozone/freon/S3EntityGenerator.java | 65 +
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 47 +-
.../freon/containergenerator/GeneratorOm.java | 14 +-
.../hadoop/ozone/shell/ReplicationOptions.java | 18 +-
.../ozone/shell/ShellReplicationOptions.java | 3 +-
.../ozone/shell/bucket/InfoBucketHandler.java | 2 +-
.../ozone/shell/bucket/ListBucketHandler.java | 17 +-
pom.xml | 44 +-
393 files changed, 14348 insertions(+), 6595 deletions(-)
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index aba10935e9,4d059d2d60..c10344756a
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -3296,13 -3338,25 +3347,33 @@@
will create intermediate directories.
</description>
</property>
+ <property>
+ <name>ozone.om.snapshot.cache.max.size</name>
+ <value>10</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ Size of the OM Snapshot LRU cache. This is the maximum number of open OM Snapshot RocksDb instances
+ that will be held in memory at any time.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.om.container.location.cache.size</name>
+ <value>100000</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ The size of the container locations cache in Ozone Manager. This cache allows Ozone Manager to populate
+ block locations in key-read responses without calling SCM, thus increases Ozone Manager read performance.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.om.container.location.cache.ttl</name>
+ <value>360m</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ The time to live for container location cache in Ozone.
+ </description>
+ </property>
+
-
</configuration>
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
index fd90343ae4,aca30a7b20..5478e8658a
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
@@@ -90,7 -90,7 +90,8 @@@ public final class DBStoreBuilder
private RocksDBConfiguration rocksDBConfiguration;
// Flag to indicate if the RocksDB should be opened readonly.
private boolean openReadOnly = false;
+ private int maxFSSnapshots = 0;
+ private final DBProfile defaultCfProfile;
/**
* Create DBStoreBuilder from a generic DBDefinition.
@@@ -178,27 -175,26 +176,30 @@@
Set<TableConfig> tableConfigs = makeTableConfigs();
- if (rocksDBOption == null) {
- rocksDBOption = getDefaultDBOptions(tableConfigs);
- }
+ try {
+ if (rocksDBOption == null) {
+ rocksDBOption = getDefaultDBOptions(tableConfigs);
+ }
- ManagedWriteOptions writeOptions = new ManagedWriteOptions();
- writeOptions.setSync(rocksDBConfiguration.getSyncOption());
+ ManagedWriteOptions writeOptions = new ManagedWriteOptions();
+ writeOptions.setSync(rocksDBConfiguration.getSyncOption());
- File dbFile = getDBFile();
- if (!dbFile.getParentFile().exists()) {
- throw new IOException("The DB destination directory should exist.");
- }
+ File dbFile = getDBFile();
+ if (!dbFile.getParentFile().exists()) {
+ throw new IOException("The DB destination directory should exist.");
+ }
- return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs,
- registry, openReadOnly, maxFSSnapshots);
+ return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs,
- registry, openReadOnly, dbJmxBeanNameName);
++ registry, openReadOnly, maxFSSnapshots, dbJmxBeanNameName);
+ } finally {
+ tableConfigs.forEach(TableConfig::close);
+ }
}
+ public DBStoreBuilder setMaxFSSnapshots(int maxFSSnapshots) {
+ this.maxFSSnapshots = maxFSSnapshots;
+ return this;
+ }
-
public DBStoreBuilder setName(String name) {
dbname = name;
return this;
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
index 6835fe559a,943d58ef1d..f227ab81fd
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
@@@ -95,14 -92,8 +96,18 @@@ public class RDBCheckpointManager imple
return null;
}
+ /**
+ * Create RocksDB snapshot by saving a checkpoint to a directory.
+ *
+ * @param parentDir The directory where the checkpoint needs to be created.
+ * @return RocksDB specific Checkpoint information object.
+ */
+ public RocksDBCheckpoint createCheckpoint(String parentDir) {
+ return createCheckpoint(parentDir, null);
+ }
+
+ @Override
+ public void close() throws IOException {
+ checkpoint.close();
+ }
}
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index 4b98213af1,be8cfb9900..9fabf5e9df
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@@ -60,36 -57,29 +60,40 @@@ public class RDBStore implements DBStor
private ObjectName statMBeanName;
private final RDBCheckpointManager checkPointManager;
private final String checkpointsParentDir;
+ private final String snapshotsParentDir;
private final RDBMetrics rdbMetrics;
+ private final RocksDBCheckpointDiffer rocksDBCheckpointDiffer;
+ private final String dbJmxBeanName;
@VisibleForTesting
public RDBStore(File dbFile, ManagedDBOptions options,
Set<TableConfig> families) throws IOException {
this(dbFile, options, new ManagedWriteOptions(), families,
- new CodecRegistry(), false, 1000);
- new CodecRegistry(), false, null);
++ new CodecRegistry(), false, 1000, null);
}
++ @SuppressWarnings("parameternumber")
public RDBStore(File dbFile, ManagedDBOptions dbOptions,
ManagedWriteOptions writeOptions, Set<TableConfig> families,
- CodecRegistry registry, boolean readOnly, int maxFSSnapshots)
- throws IOException {
- CodecRegistry registry, boolean readOnly,
++ CodecRegistry registry, boolean readOnly, int maxFSSnapshots,
+ String dbJmxBeanNameName) throws IOException {
Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
Preconditions.checkNotNull(families);
Preconditions.checkArgument(!families.isEmpty());
codecRegistry = registry;
dbLocation = dbFile;
+ dbJmxBeanName = dbJmxBeanNameName == null ? dbFile.getName() :
+ dbJmxBeanNameName;
try {
+ rocksDBCheckpointDiffer =
+ new RocksDBCheckpointDiffer(
+ dbLocation.getAbsolutePath(), maxFSSnapshots,
+ Paths.get(dbLocation.getParent(), "db.checkpoints").toString(),
+ Paths.get(dbLocation.getParent(), "db.savedSSTFiles").toString(),
+ dbLocation.getAbsolutePath(), 0, "Snapshot_");
+ rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions);
+
db = RocksDatabase.open(dbFile, dbOptions, writeOptions,
families, readOnly);
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index caa61e8d69,65a2f67883..ab7bacb05b
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@@ -814,8 -800,37 +819,42 @@@ public final class OmUtils
return printString.toString();
}
+ // Key points to entire bucket's snapshot
+ public static boolean isBucketSnapshotIndicator(String key) {
+ return key.startsWith(OM_SNAPSHOT_INDICATOR) && key.split("/").length == 2;
+ }
++
+ public static String format(List<ServiceInfo> nodes, int port,
+ String leaderId) {
+ StringBuilder sb = new StringBuilder();
+ // Ensuring OM's are printed in correct order
+ List<ServiceInfo> omNodes = nodes.stream()
+ .filter(node -> node.getNodeType() == HddsProtos.NodeType.OM)
+ .sorted(Comparator.comparing(ServiceInfo::getHostname))
+ .collect(Collectors.toList());
+ int count = 0;
+ for (ServiceInfo info : omNodes) {
+ // Printing only the OM's running
+ if (info.getNodeType() == HddsProtos.NodeType.OM) {
+ String role =
+ info.getOmRoleInfo().getNodeId().equals(leaderId) ? "LEADER" :
+ "FOLLOWER";
+ sb.append(
+ String.format(
+ " { HostName: %s | Node-Id: %s | Ratis-Port : %d | Role: %s} ",
+ info.getHostname(),
+ info.getOmRoleInfo().getNodeId(),
+ port,
+ role
+ ));
+ count++;
+ }
+ }
+ // Print Stand-alone if only one OM exists
+ if (count == 1) {
+ return "STANDALONE";
+ } else {
+ return sb.toString();
+ }
+ }
}
diff --cc hadoop-ozone/dist/src/main/license/jar-report.txt
index 9581b6dbfb,ac5a3ab9a1..ba25a09545
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@@ -255,7 -245,7 +250,8 @@@ share/ozone/lib/ratis-thirdparty-misc.j
share/ozone/lib/ratis-tools.jar
share/ozone/lib/re2j.jar
share/ozone/lib/reflections.jar
+share/ozone/lib/rocksdb-checkpoint-differ.jar
+ share/ozone/lib/reload4j.jar
share/ozone/lib/rocksdbjni.jar
share/ozone/lib/simpleclient_common.jar
share/ozone/lib/simpleclient_dropwizard.jar
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index d46dda9ff3,88501120bf..5c9b3409a5
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -122,7 -122,7 +122,10 @@@ enum Type
SetRangerServiceVersion = 107;
RangerBGSync = 109;
- CreateSnapshot = 110;
++
+ EchoRPC = 110;
++
++ CreateSnapshot = 111;
}
message OMRequest {
@@@ -227,7 -227,7 +230,11 @@@
optional SetRangerServiceVersionRequest SetRangerServiceVersionRequest = 107;
optional RangerBGSyncRequest RangerBGSyncRequest = 109;
- optional CreateSnapshotRequest CreateSnapshotRequest = 110;
++
+ optional EchoRPCRequest EchoRPCRequest = 110;
++
++ optional CreateSnapshotRequest CreateSnapshotRequest = 111;
++
}
message OMResponse {
@@@ -325,7 -325,7 +332,11 @@@
optional SetRangerServiceVersionResponse SetRangerServiceVersionResponse = 107;
optional RangerBGSyncResponse RangerBGSyncResponse = 109;
- optional CreateSnapshotResponse CreateSnapshotResponse = 110;
++
+ optional EchoRPCResponse EchoRPCResponse = 110;
++
++ optional CreateSnapshotResponse CreateSnapshotResponse = 111;
++
}
enum Status {
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index d901336223,4ea8c52bf4..a2de68dcc2
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@@ -171,17 -161,37 +159,17 @@@ public class KeyManagerImpl implements
private BackgroundService openKeyCleanupService;
- @VisibleForTesting
- public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
- OMMetadataManager metadataManager, OzoneConfiguration conf,
- OzoneBlockTokenSecretManager secretManager,
- OMPerformanceMetrics metrics) {
- this(null, new ScmClient(scmBlockClient, null, conf), metadataManager,
- conf, secretManager, null, metrics);
- }
-
- @VisibleForTesting
- public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
- StorageContainerLocationProtocol scmContainerClient,
- OMMetadataManager metadataManager, OzoneConfiguration conf, String omId,
- OzoneBlockTokenSecretManager secretManager,
- OMPerformanceMetrics metrics) {
- this(null, new ScmClient(scmBlockClient, scmContainerClient, conf),
- metadataManager, conf, secretManager, null,
- metrics);
- }
-
public KeyManagerImpl(OzoneManager om, ScmClient scmClient,
- OzoneConfiguration conf, String omId) {
- this (om, scmClient, om.getMetadataManager(), conf, omId,
- om.getBlockTokenMgr(), om.getKmsProvider(), om.getPrefixManager());
+ OzoneConfiguration conf, OMPerformanceMetrics metrics) {
+ this (om, scmClient, om.getMetadataManager(), conf,
+ om.getBlockTokenMgr(), om.getKmsProvider(), metrics);
}
- @SuppressWarnings("parameternumber")
public KeyManagerImpl(OzoneManager om, ScmClient scmClient,
- OMMetadataManager metadataManager, OzoneConfiguration conf, String omId,
+ OMMetadataManager metadataManager, OzoneConfiguration conf,
OzoneBlockTokenSecretManager secretManager,
- KeyProviderCryptoExtension kmsProvider, PrefixManager prefixManager) {
+ KeyProviderCryptoExtension kmsProvider,
+ OMPerformanceMetrics metrics) {
this.scmBlockSize = (long) conf
.getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
StorageUnit.BYTES);
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index d3a14e267f,0000000000..42771f66f9
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@@ -1,499 -1,0 +1,501 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import java.io.IOException;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
++import org.apache.hadoop.hdds.server.OzoneAdmins;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.audit.AuditAction;
+import org.apache.hadoop.ozone.audit.AuditEventStatus;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditMessage;
+import org.apache.hadoop.ozone.audit.Auditor;
+import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
+import org.apache.hadoop.ozone.security.acl.RequestContext;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import java.net.InetAddress;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
+import static org.apache.hadoop.ozone.om.KeyManagerImpl.getRemoteUser;
+import static org.apache.hadoop.ozone.om.OzoneManager.getS3Auth;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
+import org.apache.hadoop.ozone.security.acl.OzoneAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType;
+import org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+
+/**
+ * OM Metadata Reading class for the OM and Snapshot managers.
+ *
+ * This abstraction manages all the metadata key/acl reading
+ * from a rocksDb instance, for both the OM and OM snapshots.
+ */
+public class OmMetadataReader implements IOmMetadataReader, Auditor {
+ private final KeyManager keyManager;
+ private final PrefixManager prefixManager;
+ private final VolumeManager volumeManager;
+ private final BucketManager bucketManager;
+ private final OzoneManager ozoneManager;
+ private final boolean isAclEnabled;
+ private final IAccessAuthorizer accessAuthorizer;
+ private final boolean isNativeAuthorizerEnabled;
+ private final OmMetadataReaderMetrics metrics;
+ private final Logger log;
+ private final AuditLogger audit;
+
+ public OmMetadataReader(KeyManager keyManager,
+ PrefixManager prefixManager,
+ OzoneManager ozoneManager,
+ Logger log,
+ AuditLogger audit,
+ OmMetadataReaderMetrics omMetadataReaderMetrics) {
+ this.keyManager = keyManager;
+ this.bucketManager = ozoneManager.getBucketManager();
+ this.volumeManager = ozoneManager.getVolumeManager();
+ this.prefixManager = prefixManager;
+ OzoneConfiguration configuration = ozoneManager.getConfiguration();
+ this.ozoneManager = ozoneManager;
+ this.isAclEnabled = ozoneManager.getAclsEnabled();
+ this.log = log;
+ this.audit = audit;
+ boolean allowListAllVolumes = ozoneManager.getAllowListAllVolumes();
+ metrics = omMetadataReaderMetrics;
+ if (isAclEnabled) {
+ accessAuthorizer = getACLAuthorizerInstance(configuration);
+ if (accessAuthorizer instanceof OzoneNativeAuthorizer) {
+ OzoneNativeAuthorizer authorizer =
+ (OzoneNativeAuthorizer) accessAuthorizer;
+ isNativeAuthorizerEnabled = true;
+ authorizer.setVolumeManager(volumeManager);
+ authorizer.setBucketManager(bucketManager);
+ authorizer.setKeyManager(keyManager);
+ authorizer.setPrefixManager(prefixManager);
- authorizer.setOzoneAdmins(ozoneManager.getOmAdminUsernames());
++ authorizer.setOzoneAdmins(
++ new OzoneAdmins(ozoneManager.getOmAdminUsernames()));
+ authorizer.setAllowListAllVolumes(allowListAllVolumes);
+ } else {
+ isNativeAuthorizerEnabled = false;
+ }
+ } else {
+ accessAuthorizer = null;
+ isNativeAuthorizerEnabled = false;
+ }
+ }
+
+ /**
+ * Lookup a key.
+ *
+ * @param args - attributes of the key.
+ * @return OmKeyInfo - the info about the requested key.
+ * @throws IOException
+ */
+ @Override
+ public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException {
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+
+ if (isAclEnabled) {
+ checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ,
+ bucket.realVolume(), bucket.realBucket(), args.getKeyName());
+ }
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ args = bucket.update(args);
+
+ try {
+ metrics.incNumKeyLookups();
+ return keyManager.lookupKey(args, getClientAddress());
+ } catch (Exception ex) {
+ metrics.incNumKeyLookupFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_KEY,
+ auditMap));
+ }
+ }
+ }
+
+ @Override
+ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries, boolean allowPartialPrefixes)
+ throws IOException {
+
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+
+ if (isAclEnabled) {
+ checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ,
+ bucket.realVolume(), bucket.realBucket(), args.getKeyName());
+ }
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ args = bucket.update(args);
+
+ try {
+ metrics.incNumListStatus();
+ return keyManager.listStatus(args, recursive, startKey, numEntries,
+ getClientAddress(), allowPartialPrefixes);
+ } catch (Exception ex) {
+ metrics.incNumListStatusFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_STATUS,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(
+ OMAction.LIST_STATUS, auditMap));
+ }
+ }
+ }
+
+ @Override
+ public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException {
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ args = bucket.update(args);
+
+ try {
+ metrics.incNumGetFileStatus();
+ return keyManager.getFileStatus(args, getClientAddress());
+ } catch (IOException ex) {
+ metrics.incNumGetFileStatusFails();
+ auditSuccess = false;
+ audit.logReadFailure(
+ buildAuditMessageForFailure(OMAction.GET_FILE_STATUS, auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(
+ buildAuditMessageForSuccess(OMAction.GET_FILE_STATUS, auditMap));
+ }
+ }
+ }
+
+ @Override
+ public OmKeyInfo lookupFile(OmKeyArgs args) throws IOException {
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+
+ if (isAclEnabled) {
+ checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ,
+ bucket.realVolume(), bucket.realBucket(), args.getKeyName());
+ }
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ args = bucket.update(args);
+
+ try {
+ metrics.incNumLookupFile();
+ return keyManager.lookupFile(args, getClientAddress());
+ } catch (Exception ex) {
+ metrics.incNumLookupFileFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.LOOKUP_FILE,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(
+ OMAction.LOOKUP_FILE, auditMap));
+ }
+ }
+ }
+
+ @Override
+ public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
+ String startKey, String keyPrefix, int maxKeys) throws IOException {
+
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(
+ Pair.of(volumeName, bucketName));
+
+ if (isAclEnabled) {
+ checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST,
+ bucket.realVolume(), bucket.realBucket(), keyPrefix);
+ }
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit();
+ auditMap.put(OzoneConsts.START_KEY, startKey);
+ auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys));
+ auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix);
+
+ try {
+ metrics.incNumKeyLists();
+ return keyManager.listKeys(bucket.realVolume(), bucket.realBucket(),
+ startKey, keyPrefix, maxKeys);
+ } catch (IOException ex) {
+ metrics.incNumKeyListFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_KEYS,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_KEYS,
+ auditMap));
+ }
+ }
+ }
+
+ /**
+ * Returns list of ACLs for given Ozone object.
+ *
+ * @param obj Ozone object.
+ * @throws IOException if there is error.
+ */
+ public List<OzoneAcl> getAcl(OzoneObj obj) throws IOException {
+ boolean auditSuccess = true;
+
+ try {
+ if (isAclEnabled) {
+ checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.READ_ACL,
+ obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
+ }
+ metrics.incNumGetAcl();
+ switch (obj.getResourceType()) {
+ case VOLUME:
+ return volumeManager.getAcl(obj);
+ case BUCKET:
+ return bucketManager.getAcl(obj);
+ case KEY:
+ return keyManager.getAcl(obj);
+ case PREFIX:
+ return prefixManager.getAcl(obj);
+
+ default:
+ throw new OMException("Unexpected resource type: " +
+ obj.getResourceType(), INVALID_REQUEST);
+ }
+ } catch (Exception ex) {
+ auditSuccess = false;
+ audit.logReadFailure(
+ buildAuditMessageForFailure(OMAction.GET_ACL, obj.toAuditMap(), ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(
+ buildAuditMessageForSuccess(OMAction.GET_ACL, obj.toAuditMap()));
+ }
+ }
+ }
+
+ /**
+ * Checks if current caller has acl permissions.
+ *
+ * @param resType - Type of ozone resource. Ex volume, bucket.
+ * @param store - Store type. i.e Ozone, S3.
+ * @param acl - type of access to be checked.
+ * @param vol - name of volume
+ * @param bucket - bucket name
+ * @param key - key
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied.
+ */
+ void checkAcls(ResourceType resType, StoreType store,
+ ACLType acl, String vol, String bucket, String key)
+ throws IOException {
+ UserGroupInformation user;
+ if (getS3Auth() != null) {
+ String principal =
+ OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId());
+ user = UserGroupInformation.createRemoteUser(principal);
+ } else {
+ user = ProtobufRpcEngine.Server.getRemoteUser();
+ }
+
+ InetAddress remoteIp = ProtobufRpcEngine.Server.getRemoteIp();
+ String volumeOwner = ozoneManager.getVolumeOwner(vol, acl, resType);
+ String bucketOwner = ozoneManager.getBucketOwner(vol, bucket, acl, resType);
+
+ OzoneAclUtils.checkAllAcls(this, resType, store, acl,
+ vol, bucket, key, volumeOwner, bucketOwner,
+ user != null ? user : getRemoteUser(),
+ remoteIp != null ? remoteIp :
+ ozoneManager.getOmRpcServerAddr().getAddress(),
+ remoteIp != null ? remoteIp.getHostName() :
+ ozoneManager.getOmRpcServerAddr().getHostName());
+ }
+
+
+ /**
+ * CheckAcls for the ozone object.
+ *
+ * @return true if permission granted, false if permission denied.
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied
+ * and throwOnPermissionDenied set to true.
+ */
+ @SuppressWarnings("parameternumber")
+ public boolean checkAcls(ResourceType resType, StoreType storeType,
+ ACLType aclType, String vol, String bucket, String key,
+ UserGroupInformation ugi, InetAddress remoteAddress, String hostName,
+ boolean throwIfPermissionDenied, String owner)
+ throws OMException {
+ OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
+ .setResType(resType)
+ .setStoreType(storeType)
+ .setVolumeName(vol)
+ .setBucketName(bucket)
+ .setKeyName(key).build();
+ RequestContext context = RequestContext.newBuilder()
+ .setClientUgi(ugi)
+ .setIp(remoteAddress)
+ .setHost(hostName)
+ .setAclType(ACLIdentityType.USER)
+ .setAclRights(aclType)
+ .setOwnerName(owner)
+ .build();
+
+ return checkAcls(obj, context, throwIfPermissionDenied);
+ }
+
+ /**
+ * CheckAcls for the ozone object.
+ *
+ * @return true if permission granted, false if permission denied.
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied
+ * and throwOnPermissionDenied set to true.
+ */
+ public boolean checkAcls(OzoneObj obj, RequestContext context,
+ boolean throwIfPermissionDenied)
+ throws OMException {
+
+ if (!accessAuthorizer.checkAccess(obj, context)) {
+ if (throwIfPermissionDenied) {
+ String volumeName = obj.getVolumeName() != null ?
+ "Volume:" + obj.getVolumeName() + " " : "";
+ String bucketName = obj.getBucketName() != null ?
+ "Bucket:" + obj.getBucketName() + " " : "";
+ String keyName = obj.getKeyName() != null ?
+ "Key:" + obj.getKeyName() : "";
+ log.warn("User {} doesn't have {} permission to access {} {}{}{}",
+ context.getClientUgi().getUserName(), context.getAclRights(),
+ obj.getResourceType(), volumeName, bucketName, keyName);
+ throw new OMException("User " + context.getClientUgi().getUserName() +
+ " doesn't have " + context.getAclRights() +
+ " permission to access " + obj.getResourceType() + " " +
+ volumeName + bucketName + keyName, ResultCodes.PERMISSION_DENIED);
+ }
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * Returns an instance of {@link IAccessAuthorizer}.
+ * Looks up the configuration to see if there is custom class specified.
+ * Constructs the instance by passing the configuration directly to the
+ * constructor to achieve thread safety using final fields.
+ *
+ * @param conf
+ * @return IAccessAuthorizer
+ */
+ private IAccessAuthorizer getACLAuthorizerInstance(OzoneConfiguration conf) {
+ Class<? extends IAccessAuthorizer> clazz = conf.getClass(
+ OZONE_ACL_AUTHORIZER_CLASS, OzoneAccessAuthorizer.class,
+ IAccessAuthorizer.class);
+ return ReflectionUtils.newInstance(clazz, conf);
+ }
+
+ private static String getClientAddress() {
+ String clientMachine = Server.getRemoteAddress();
+ if (clientMachine == null) { //not a RPC client
+ clientMachine = "";
+ }
+ return clientMachine;
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForSuccess(AuditAction op,
+ Map<String, String> auditMap) {
+
+ return new AuditMessage.Builder()
+ .setUser(getRemoteUserName())
+ .atIp(Server.getRemoteAddress())
+ .forOperation(op)
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.SUCCESS)
+ .build();
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForFailure(AuditAction op,
+ Map<String, String> auditMap, Throwable throwable) {
+
+ return new AuditMessage.Builder()
+ .setUser(getRemoteUserName())
+ .atIp(Server.getRemoteAddress())
+ .forOperation(op)
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.FAILURE)
+ .withException(throwable)
+ .build();
+ }
+
+ /**
+ * Returns true if OzoneNativeAuthorizer is enabled and false if otherwise.
+ *
+ * @return if native authorizer is enabled.
+ */
+ public boolean isNativeAuthorizerEnabled() {
+ return isNativeAuthorizerEnabled;
+ }
+
+ public IAccessAuthorizer getAccessAuthorizer() {
+ return accessAuthorizer;
+ }
+
+ private ResourceType getResourceType(OmKeyArgs args) {
+ if (args.getKeyName() == null || args.getKeyName().length() == 0) {
+ return ResourceType.BUCKET;
+ }
+ return ResourceType.KEY;
+ }
+
+
+}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index dcbed30bd4,0000000000..ffdd136877
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@@ -1,199 -1,0 +1,199 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+
+
+/**
+ * This class is used to manage/create OM snapshots.
+ */
+public final class OmSnapshotManager {
+ private final OzoneManager ozoneManager;
+ private final LoadingCache<String, OmSnapshot> snapshotCache;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OmSnapshotManager.class);
+
+ OmSnapshotManager(OzoneManager ozoneManager) {
+ this.ozoneManager = ozoneManager;
+
+ // size of lru cache
+ int cacheSize = ozoneManager.getConfiguration().getInt(
+ OzoneConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE,
+ OzoneConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE_DEFAULT);
+
+ CacheLoader<String, OmSnapshot> loader;
+ loader = new CacheLoader<String, OmSnapshot>() {
+ @Override
+
+ // load the snapshot into the cache if not already there
+ @Nonnull
+ public OmSnapshot load(@Nonnull String snapshotTableKey)
+ throws IOException {
+ SnapshotInfo snapshotInfo;
+ // see if the snapshot exists
+ snapshotInfo = getSnapshotInfo(snapshotTableKey);
+
+ // read in the snapshot
+ OzoneConfiguration conf = ozoneManager.getConfiguration();
+ OMMetadataManager snapshotMetadataManager;
+
+ // Create the snapshot metadata manager by finding the corresponding
+ // RocksDB instance, creating an OmMetadataManagerImpl instance based on
+ // that
+ try {
+ snapshotMetadataManager = OmMetadataManagerImpl
+ .createSnapshotMetadataManager(
+ conf, snapshotInfo.getCheckpointDirName());
+ } catch (IOException e) {
+ LOG.error("Failed to retrieve snapshot: {}, {}", snapshotTableKey, e);
+ throw e;
+ }
+
+ // create the other manager instances based on snapshot metadataManager
+ PrefixManagerImpl pm = new PrefixManagerImpl(snapshotMetadataManager,
+ false);
+ KeyManagerImpl km = new KeyManagerImpl(null,
- ozoneManager.getScmClient(), snapshotMetadataManager, conf, null,
++ ozoneManager.getScmClient(), snapshotMetadataManager, conf,
+ ozoneManager.getBlockTokenSecretManager(),
- ozoneManager.getKmsProvider(), pm);
++ ozoneManager.getKmsProvider(), ozoneManager.getPerfMetrics());
+
+ return new OmSnapshot(km, pm, ozoneManager,
+ snapshotInfo.getVolumeName(),
+ snapshotInfo.getBucketName(),
+ snapshotInfo.getName());
+ }
+ };
+
+ RemovalListener<String, OmSnapshot> removalListener
+ = notification -> {
+ try {
+ // close snapshot's rocksdb on eviction
+ notification.getValue().close();
+ } catch (IOException e) {
+ LOG.error("Failed to close snapshot: {} {}",
+ notification.getKey(), e);
+ }
+ };
+ // init LRU cache
+ snapshotCache = CacheBuilder.newBuilder()
+ .maximumSize(cacheSize)
+ .removalListener(removalListener)
+ .build(loader);
+ }
+
+ /**
+ * Creates snapshot checkpoint that corresponds to snapshotInfo.
+ * @param omMetadataManager the metadata manager
+ * @param snapshotInfo The metadata of snapshot to be created
+ * @return instance of DBCheckpoint
+ */
+ public static DBCheckpoint createOmSnapshotCheckpoint(
+ OMMetadataManager omMetadataManager, SnapshotInfo snapshotInfo)
+ throws IOException {
+ RDBStore store = (RDBStore) omMetadataManager.getStore();
+ return store.getSnapshot(snapshotInfo.getCheckpointDirName());
+ }
+
+ // Get OmSnapshot if the keyname has ".snapshot" key indicator
+ public IOmMetadataReader checkForSnapshot(String volumeName,
+ String bucketName, String keyname)
+ throws IOException {
+ if (keyname == null) {
+ return ozoneManager.getOmMetadataReader();
+ }
+
+ // see if key is for a snapshot
+ String[] keyParts = keyname.split("/");
+ if (isSnapshotKey(keyParts)) {
+ String snapshotName = keyParts[1];
+ if (snapshotName == null || snapshotName.isEmpty()) {
+ // don't allow snapshot indicator without snapshot name
+ throw new OMException(INVALID_KEY_NAME);
+ }
+ String snapshotTableKey = SnapshotInfo.getTableKey(volumeName,
+ bucketName, snapshotName);
+
+ // retrieve the snapshot from the cache
+ try {
+ return snapshotCache.get(snapshotTableKey);
+ } catch (ExecutionException e) {
+ throw new IOException(e.getCause());
+ }
+ } else {
+ return ozoneManager.getOmMetadataReader();
+ }
+ }
+
+ public SnapshotInfo getSnapshotInfo(String volumeName,
+ String bucketName, String snapshotName)
+ throws IOException {
+ return getSnapshotInfo(SnapshotInfo.getTableKey(volumeName,
+ bucketName, snapshotName));
+ }
+
+ private SnapshotInfo getSnapshotInfo(String key) throws IOException {
+ SnapshotInfo snapshotInfo;
+ try {
+ snapshotInfo = ozoneManager.getMetadataManager()
+ .getSnapshotInfoTable()
+ .get(key);
+ } catch (IOException e) {
+ LOG.error("Snapshot {}: not found: {}", key, e);
+ throw e;
+ }
+ if (snapshotInfo == null) {
+ throw new OMException(KEY_NOT_FOUND);
+ }
+ return snapshotInfo;
+ }
+
+ public static String getSnapshotPrefix(String snapshotName) {
+ return OM_SNAPSHOT_INDICATOR + OM_KEY_PREFIX +
+ snapshotName + OM_KEY_PREFIX;
+ }
+
+ public static boolean isSnapshotKey(String[] keyParts) {
+ return (keyParts.length > 1) &&
+ (keyParts[0].compareTo(OM_SNAPSHOT_INDICATOR) == 0);
+ }
+}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 9664e602f4,8d6ae64ddf..8013e1bd15
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -215,10 -222,9 +217,8 @@@ import static org.apache.hadoop.ozone.O
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED_DEFAULT;
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX;
@@@ -565,10 -574,18 +569,19 @@@ public final class OzoneManager extend
this.isS3MultiTenancyEnabled =
OMMultiTenantManager.checkAndEnableMultiTenancy(this, conf);
- // Get admin list
- omAdminUsernames = getOzoneAdminsFromConfig(configuration);
-
+ metrics = OMMetrics.create();
+ // Get admin list
+ Collection<String> omAdminUsernames =
+ OzoneConfigUtil.getOzoneAdminsFromConfig(configuration);
+ Collection<String> omAdminGroups =
+ OzoneConfigUtil.getOzoneAdminsGroupsFromConfig(configuration);
+ omAdmins = new OzoneAdmins(omAdminUsernames, omAdminGroups);
+
+ Collection<String> s3AdminUsernames =
+ OzoneConfigUtil.getS3AdminsFromConfig(configuration);
+ Collection<String> s3AdminGroups =
+ OzoneConfigUtil.getS3AdminsGroupsFromConfig(configuration);
+ s3OzoneAdmins = new OzoneAdmins(s3AdminUsernames, s3AdminGroups);
instantiateServices(false);
// Create special volume s3v which is required for S3G.
@@@ -704,11 -722,9 +717,12 @@@
}
prefixManager = new PrefixManagerImpl(metadataManager, isRatisEnabled);
+ perfMetrics = OMPerformanceMetrics.register();
keyManager = new KeyManagerImpl(this, scmClient, configuration,
- omStorage.getOmId());
- perfMetrics);
++ perfMetrics);
+ omMetadataReader = new OmMetadataReader(keyManager, prefixManager,
+ this, LOG, AUDIT, metrics);
+ omSnapshotManager = new OmSnapshotManager(this);
if (withNewSnapshot) {
Integer layoutVersionInDB = getLayoutVersionInDB();
@@@ -1423,6 -1464,6 +1437,10 @@@
return metrics;
}
++ public OMPerformanceMetrics getPerfMetrics() {
++ return perfMetrics;
++ }
++
/**
* Start service.
*/
@@@ -3805,24 -4109,22 +3834,13 @@@
* @param callerUgi Caller UserGroupInformation
*/
public boolean isAdmin(UserGroupInformation callerUgi) {
- if (callerUgi == null) {
- return false;
- } else {
- return isAdmin(callerUgi.getShortUserName())
- || isAdmin(callerUgi.getUserName());
- }
+ return callerUgi != null && omAdmins.isAdmin(callerUgi);
}
- @VisibleForTesting
- private boolean isAdmin(String username) {
- if (omAdminUsernames == null) {
- return false;
- } else {
- return omAdminUsernames.contains(OZONE_ADMINISTRATORS_WILDCARD) ||
- omAdminUsernames.contains(username);
- }
+ public boolean isS3Admin(UserGroupInformation callerUgi) {
+ return callerUgi != null && s3OzoneAdmins.isAdmin(callerUgi);
}
- /**
- * Returns true if OzoneNativeAuthorizer is enabled and false if otherwise.
- *
- * @return if native authorizer is enabled.
- */
- public boolean isNativeAuthorizerEnabled() {
- return isNativeAuthorizerEnabled;
- }
-
@VisibleForTesting
public boolean isRunning() {
return omState == State.RUNNING;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org