You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by du...@apache.org on 2023/05/10 05:27:08 UTC
[ozone] branch HDDS-7733-Symmetric-Tokens updated (783c90f72e -> ba51dbacdf)
This is an automated email from the ASF dual-hosted git repository.
duong pushed a change to branch HDDS-7733-Symmetric-Tokens
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit 783c90f72e HDDS-7945. Integrate secret keys to SCM snapshot (#4549)
omit 124345cfe7 HDDS-8003. E2E integration test cases for block tokens (#4547)
omit 84e260196f HDDS-7831. Use symmetric secret key to sign and verify token (#4417)
omit 3f6ef59e16 HDDS-7830. SCM API for OM and Datanode to get secret keys (#4345)
omit 89487e7fcb HDDS-7734. Implement symmetric SecretKeys lifescycle management in SCM (#4194)
add 7dc1dee2a8 HDDS-8099. Make unlimited length the default in ozone debug ldb. (#4404)
add c7de8d5871 HDDS-7741. [Snapshot] Delete keys in snapshot scope from deleteTable during createSnapshot to accommodate snapshot garbage collection (#4280)
add 07bb734e76 HDDS-8041. Let Ozone Client fail faster with wrong OM address in URI. (#4325)
add 29acb2b188 HDDS-7548. [Snapshot] SnapDiff pagination and partial diff report implementation (#4360)
add b25410d691 HDDS-8069. [Snapshot] Compaction DAG reconstruction logic potentially crashing OM on startup (#4401)
add dc03783161 HDDS-8145. ReadReplicas should close client (#4387)
add f093f71314 HDDS-8158. Replication Manager: Make all handlers send commands immediately instead of returning commands (#4399)
add ec62964e32 HDDS-8167. Inject MoveManager into ContainerBalancer (#4411)
add 0f1ba3f8a3 HDDS-7958. Ozone client not closed in integration tests (#4373)
add 6c18d95477 HDDS-8122. [Snapshot] Disable RocksDB column family auto compaction in SstFilteringService (#4400)
add 864ec707b7 HDDS-8183. Fix edge case where delimiter is empty string in BucketEndpoint#get (#4410)
add 3068a31f53 HDDS-8160. [Snapshot] Skip the compaction entry from log and DAG if compaction input and output files are exactly same. (#4416)
add 1c30a3528a HDDS-8149. Refactor the way to notify keyStoreFactory about certificate renewed (#4390)
add c14200c84b HDDS-8111. ReplicationManager: Add RatisMisReplicationHandler into rm.processUnderReplicatedContainer (#4414)
add 1236414c1c HDDS-8061. [Snapshot] Add SnapDiff client option to force a full diff (#4398)
add 9619759632 HDDS-8171. Replicate commands could be sent to dead maintenance nodes if the same index is being decommissioned (#4412)
add 8202b4184e HDDS-8204. Add testuser principals for all Ozone containers (#4422)
add b570ca7c33 HDDS-8187. ReplicationManager: Datanode commands should be sent to nodeManager directly (#4413)
add a76fb0ee58 HDDS-8205. Reorder OM nodes in HA acceptance tests (#4423)
add 9512c32c99 HDDS-8217. Fix OmBucketInfo#equals for comparing defaultReplicationConfig (#4426)
add 22823a0012 HDDS-8221. Fix misplaced OzoneKey javadoc comment (#4430)
add c144a7d457 HDDS-8054. Fix NPE in metrics for failed volume (#4340)
add da8da7435c HDDS-8216. EC: OzoneClientConfig is overwritten in ECKeyOutputStream (#4425)
add ceaaaae1a2 HDDS-8139. Datanodes should not drop block delete transactions based on transaction ID (#4384)
add 130b7b6587 HDDS-8068. Fix Exception: JMXJsonServlet, getting attribute RatisRoles of Hadoop:service=OzoneManager. (#4352)
add d0888381b7 HDDS-8222. EndpointBase#getBucket should handle BUCKET_NOT_FOUND (#4431)
add 65268cc2c1 HDDS-7918. EC: ECBlockReconstructedStripeInputStream should check for spare replicas before failing an index (#4441)
add 4a43f343ad HDDS-8168. Make deadlines inside MoveManager for move commands configurable (#4415)
add 184bfc2118 HDDS-7883. [Snapshot] Accommodate FSO, key renames and implement OMSnapshotPurgeRequest for SnapshotDeletingService (#4407)
add 030cf40fb1 HDDS-8172. ECUnderReplicationHandler should consider commands already sent when processing the container (#4435)
add dc6df43e34 HDDS-8152. Reduce S3 acceptance test setup time (#4393)
add 598de2f9e0 HDDS-7975. Rebalance acceptance tests (#4437)
add 21e8ef0fca HDDS-8173. Fix to remove enrties from RocksDB after container gets deleted. (#4445)
add 8e0c462c11 HDDS-8220. [Ozone-Streaming] Trigger volume check on IOException in StreamDataChannelBase (#4428)
add 6dd80eba2c HDDS-8233. ReplicationManager: Throttle delete container commands from over-replication handlers (#4447)
add 900f2db64a HDDS-8245. Info log for keyDeletingService when nonzero number of keys are deleted. (#4451)
add b801c6860e HDDS-8154. Perf: Reuse Mac instances in S3 token validation (#4433)
add 15117d8020 HDDS-8189. [Snapshot] renamedKeyTable should only track keys in buckets that has at least one active snapshot. (#4436)
add 209a2e785d HDDS-8042. Display certificate issuer in cert list command. (#4429)
add 3f5a80783a HDDS-7853. Add support for RemoveSCM in SCMRatisServer. (#4358)
add 5a90593f20 HDDS-8148. Improve log for Pipeline creation failure (#4385)
add d9a48cc95d HDDS-8155. SCM Block deleting service add transaction blocks for stale/dead/decommissioning DNs (#4403)
add cd37da0d08 HDDS-7874. Disable flaky unit test: TestHddsSecureDatanodeInit.testCertificateRotationRecoverableFailure
add e38e4721a2 HDDS-8241. Transfer leader command doesn't work in secure cluster (#4454)
add 823f56eaff HDDS-8098. Make Schema V3 as default in ozone debug ldb for container db. (#4434)
add 181558a21a HDDS-8230. Let ReplicationManager decide the timeout for commands in Datanodes (#4453)
add be6c20eb9b HDDS-8255. Remove unnecessary sleep at secure container startup (#4457)
add 89e6ee1ef3 HDDS-8251. RocksIterator is not closed properly in snapshot (#4461)
add 5756fd876f HDDS-7559. [Snapshot] Skip compaction tracking when no Ozone snapshot exists (#4418)
add 68eb26d70e HDDS-8265. [Snapshot] Close ManagedColumnFamilyOptions properly in OmSnapshotManager (#4465)
add 1eda443a1e HDDS-8176. [Snapshot] Rename variables to fix shadow declaration warning with gcc (#4409)
add 23e0ce72d2 HDDS-8223. SCM delete block service shoud run wait for safemode to exit. (#4432)
add a05543b1b0 HDDS-7122. Add validation for EC chunk size (#4463)
add b29efbe771 HDDS-8263. [JDK17] Bump guice to 5.1.0, maven-shade-plugin to 3.4.1, remove guice-multibindings (#4462)
add 55076744d7 HDDS-8279. [Snapshot] Switch RocksDB Tools JNI to release build (#4472)
add 1f29c4b5c3 HDDS-8244. Selective checks: handle change in junit.sh (#4450)
add 191bb5f111 HDDS-8219. Run HA secure tests from s3g container (#4444)
add 11355a46f0 HDDS-8240. Disable JaCoCo for PRs and in forks (#4449)
add 386d1be18b HDDS-8128. Deduplicate the ops in RDBBatchOperation. (#4424)
add f863d9046b HDDS-8281. Add IntelliJ code style scheme import recommendation in CONTRIBUTING.md (#4474)
add 50ca90e05e HDDS-7540. Document separate scheduled CI (#4392)
add 7357f752e4 HDDS-6064. Print proper JSON from `ozone debug ldb scan` (#4420)
add 0d7a042917 HDDS-7697. Restrict change of bucket properties to owner and admins in NativeACL (#4439)
add 5db68674de HDDS-5043. Consistently use user's short name in secure mode (#2466)
add 01b7dc639b HDDS-7573. Use keyManager and trustManager provided by keyStoreFactory in Ratis group (#4464)
add 31cc0bdc8a HDDS-8169. Delay Starting ContainerBalancer after SCM failover (#4458)
add e8fbdaaca2 HDDS-8285. Eliminate leftover Guava Optional from CacheValue (#4482)
add 33a0e61081 HDDS-8258. RocksIterator not closed properly in KeyValueHandler.logBlocksIfNonZero (#4471)
add 915b11bea6 HDDS-8275. ManagedWriteBatch is not closed properly in SCMHADBTransactionBuffer (#4476)
add 652459131a HDDS-8287. Add toString() method to ECContainerReplicaCount (#4481)
add 1508370494 HDDS-8288. Upgrade moment.js to 2.29.4 (#4483)
add 1ed9a73af0 HDDS-8272. DBStore not closed properly in ReconStorageContainerManagerFacade (#4469)
add 881a471c4e HDDS-8274. Intermittent timeout in acceptance-MR test setup (#4475)
add e79a221556 HDDS-8257. RocksIterator not closed properly in LegacyBucketHandler (#4468)
add 199da36ad6 HDDS-7842. java.lang.NullPointerException: hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService. (#4395)
add c74f19e8aa HDDS-8193. Ozone file systems to support Hadoop's PathCapabilities interface (#4452)
add d9a1662494 HDDS-7781. Change Get Key Info to return HSync info (#4460)
add 428fe1de25 HDDS-7952. [Snapshot] Server side changes to add job status and wait time to mimic the async behaviour for snapshot diff API. (#4438)
add 2a826133d6 HDDS-8195. RDBStore.getUpdatesSince() throws RocksDBException: Requested array size exceeds VM limit (#4459)
add 012ecd3316 HDDS-8178. CertificateClient and KeyStoresFactory support multiple Sub-CA certificates in the trust chain (#4442)
add 650e306c83 HDDS-6633. Refactor upgrade acceptance tests to reduce code duplication. (#4456)
add 5ae810a7d8 HDDS-7855. Add hsync metrics in OM. (#4479)
add 05c8dbb674 HDDS-7994. Expose information via the ClusterState endpoint on keys and Directories marked for deletion (#4406)
add cce3c7ea79 HDDS-8293. In Network Topology documentation remove DNS configuration from Topology table mapping file. (#4485)
add b5137a0c78 HDDS-8290. [Snapshot] Merge entries from snapshotRenamedKeyTable to next snapshot. (#4488)
add 4cb0e68efe HDDS-8276. EC file checksum calculation fails with topology-aware read enabled (#4495)
add e8e34c2a91 HDDS-7914. [Snapshot] Block FS API access to deleted (non-active) snapshots (#4419)
add 0a029def5a HDDS-8282. [Snapshot] Intermittent DB crash in RocksDBCheckpointDiffer (#4492)
add 06433c0da9 HDDS-6961. [Snapshot] Bootstrapping slow followers/new followers. (#3980)
add 48bfc26998 HDDS-8311. Fix list of supported units in Quota documentation
add 17e5349bc2 HDDS-8134. Certificate clients are not correctly closed. (#4380)
add 91227327b1 HDDS-8309. ReplicationManager: Basic Throttling of EC Reconstruction commands (#4496)
add 278c4cdce0 HDDS-8330. [Doc] Fix misplaced doc for GDPR.zh.md (#4502)
add 54e4ca3b39 HDDS-8313. Support rocksdb WAL ttl and size limit configuration (#4494)
add 57d66137fa HDDS-8323. [Snapshot] cancelAllBackgroundWork and close listeners in RDBStore#close (#4498)
add 612cfd631a HDDS-8320. Fix ranger jackson version conflict (#4504)
add f9a240da19 HDDS-5463. [FSO] Recon Container API does not work correctly with FSO. (#4182)
add 54a5a3c59c HDDS-8239. Backwards compatibility fix for new RATIS_DATASTREAM port (#4500)
add e9592d2654 HDDS-8273. EC: Validate replication config at server-side (#4507)
add 83969c0092 HDDS-8353. Mark TestBlockDeletion#testContainerStatisticsAfterDelete as flaky
add 7b570dfa7a HDDS-8185. Implement JSON output for datanode list (#4508)
add 3ecd87cd7a HDDS-8334. ReplicationManager: Add nodes to exclude list if they are overloaded (#4510)
add 3799bb228b HDDS-7817. Add HTTP and HTTPS ports to DatanodeDetails (#4369)
add d10bbf53c1 HDDS-8089. Disable flaky test TestOmSnapshot#checkKey
add d91f619f22 HDDS-6335. Disable flaky test TestParentAcl#testKeyAcl
add a83ae0f1bd HDDS-5319. Disable flaky test TestSCMUpdateServiceGrpcServer#testClientUpdateWithRevoke
add b509d37c08 HDDS-8355. Mark TestOMRatisSnapshots#testInstallSnapshot as flaky
add 0c84ff1cdb HDDS-8291. Upgrade to Hadoop 3.3.5 (#4484)
add 4a8844151a HDDS-8324. DN data cache gets removed randomly asking for data from disk (#4499)
add ac42bf0c77 HDDS-8354. Avoid WARNING about ObjectEndpoint#get (#4515)
add b7e08c1ed0 HDDS-8364. ReadReplicas may give wrong results with topology-aware read enabled (#4522)
add eab67e71ef HDDS-8127. Exclude deleted containers from Recon container count (#4440)
add e22a8f6103 HDDS-8135. Incorrect synchronization during certificate renewal in DefaultCertificateClient. (#4381)
add 910eef097d HDDS-8325. Consolidate and refine RocksDB metrics of services (#4506)
add 1a82bbd556 HDDS-8361. Useless object in TestOzoneBlockTokenIdentifier (#4517)
add c90c4cb1c5 HDDS-8359. ReplicationManager: Fix getContainerReplicationHealth() so that it builds ContainerCheckRequest correctly (#4528)
add 34de64fdae HDDS-8358. Fix the space usage comparator in ContainerBalancerSelectionCriteria (#4527)
add 4211d49eae HDDS-8368. ReplicationManager: Create ContainerReplicaOp with correct target Datanode (#4532)
add 3adae01b52 HDDS-7974. [Snapshot] KeyDeletingService to be aware of Ozone snapshots (#4486)
add 40e6875a7e HDDS-8304. [Snapshot] Reduce flakiness in testSkipTrackingWithZeroSnapshot (#4487)
add d4d90c7d1a HDDS-8039. Allow container inspector to run from ozone debug. (#4337)
add cb90fda213 HDDS-8035. Intermittent timeout in TestOzoneManagerHAWithData.testOMHAMetrics (#4362)
add 05d9d85b8e HDDS-8237. [Snapshot] loadDb() used by SstFiltering service creates extraneous directories. (#4446)
add 97f200f061 HDDS-8016. updated the ozone doc for linked bucket and deletion async limitation (#4526)
add d9d61719bd HDDS-8374. Disable flaky unit test: TestContainerStateCounts
add 53d8fe0e57 update readme (#4535)
add 9b49f264b9 HDDS-8394. Upgrade Spring Framework to 5.3.26 (#4550)
add e5133e8f29 HDDS-8397. openKey need not setKeyArgs twice (#4552)
add b54cc253a0 HDDS-8170. Let ContainerBalancer consider EC containers for balancing (#4542)
add 47d41da801 HDDS-8015. [Snapshot] Remove RequestValidations#createValidationRequest (#4513)
add 6326fd5cb8 HDDS-8351. ReplicationManager: Use RM exclude list when getting target nodes for reconstruction (#4520)
add bf35dc4cff HDDS-8337. ReplicationManager: MisReplicationHandler should throw an exception if partially successful (#4536)
add 6207d7c536 HDDS-8367. SetQuota should have a check on usedNamespace [NameSpacequota] (#4529)
add 45e8c5c45a HDDS-8383. Misreplication cannot be resolved with single rack (#4539)
add 40692dbe5d HDDS-7862. [Snapshot] Wrap up TestOMSnapshotDeleteResponse (#4548)
add 3873d889b6 HDDS-8286. Support SCM sub-ca certificate signed by leader SCM's sub-ca certificate (#4493)
add 6da7aa086a HDDS-8228. Add metric for out of order/retried delete transactions (#4533)
add eaf7f9a0c1 HDDS-8300. Enable topology aware replication and reads by default. (#4489)
add 96d9582898 HDDS-8352. OM crash with NPE in OMKeyCommitRequest due to missing user info (#4537)
add 03fdeaafaf HDDS-8302. Add a flag to disable hsync by default (#4512)
add 7075a967c4 HDDS-8346. ContainerBalancer should only move CLOSED replicas (#4560)
add bbcb8101bd HDDS-8374. Fix flaky TestContainerStateCounts in Recon (#4534)
add fc3f2861ef HDDS-7769. Implement client initiated lease recovery (#4234)
add f625a9f72c HDDS-8336. ReplicationManager: RatisUnderReplicationHandler should partially recover the container if not enough nodes (#4561)
add fc97e4893c HDDS-8406. Add OM DB update data size and sequence metrics (#4558)
add 974ebba7f7 HDDS-7758. Update Recon.zh.md doc for ozone.recon.task.safemode.wait.threshold configuration (#4562)
add 8515c13a1e HDDS-8345. [Snapshot] Remove snapshot from SnapshotChainManager in case of failure (#4525)
add 67f46e031e HDDS-8357. Exclude spring-jcl to avoid dependency conflict (#4544)
add dd003040a4 HDDS-8317. [Snapshot] DirectoryDeletingService should clean up Snapshot's deletedDirTable. (#4543)
add d34322f662 HDDS-8292. Inconsistent key name handling for FSO bucket files. (#4531)
add c9424217cd HDDS-8242. Rename operation not working with FSO bucket destination (#4478)
add 0991eab68b HDDS-8329. Performance Impact during container close when still Client is writing (#4505)
add 617ef338cd HDDS-8370. Intermittent failure in TestOzoneFileSystem#testListStatusWithIntermediateDir (#4564)
add c5912ff40d HDDS-8407. Intermittent failure in TestOzoneFileSystem#testTrash (#4566)
add 2560e64515 HDDS-8363. Make SCM Ratis roles available in /prom endpoint (#4521)
add e1ec24343b HDDS-8433. Mark TestOzoneManagerSnapshotAcl as flaky
add cffa3864b9 HDDS-7908. Support OM Metadata operation Generator in `Ozone freon` (#4251)
add 600bbd5ced HDDS-8147. Introduce latency metrics for S3 Gateway operations (#4383)
add b74faeff91 HDDS-8333. ReplicationManager: Allow partial EC reconstruction if insufficient nodes available (#4579)
add 7506842d83 HDDS-8400. Expose rocksdb last sequence number through metrics (#4557)
add dd489c7166 HDDS-8253. Set ozone.metadata.dirs to temporary dir if not defined in S3 Gateway (#4455)
add 93cdbfebab HDDS-8429. Checkpoint is not closed properly in OMDBCheckpointServlet (#4575)
add 0ee6f4a086 HDDS-8109. Seperate Ratis and EC MisReplication Handling (#4577)
add d58d529d8d HDDS-8270. Measure checkAccess latency for Ozone objects (#4467)
add 8f745fcc96 HDDS-8137. [Snapshot] SnapDiff to use tombstone entries in SST files (#4376)
add 35c64efa3d HDDS-8453. Mark TestDirectoryDeletingServiceWithFSO#testDirDeletedTableCleanUpForSnapshot as flaky
add c2c393a8ac HDDS-8424: Preserve legacy bucket getKeyInfo behavior (#4576)
add 97b18c5d27 HDDS-8385. Ozone can't process snapshot when service UID > 2097151 (#4580)
add 1cb935b064 HDDS-8458. Mark TestBlockDeletion#testBlockDeletion as flaky
add e61f348362 HDDS-7309. Enable by default GRPC between S3G and OM (#3820)
add f62476139c HDDS-8440. Ozone Manager crashed with ClassCastException when deleting FSO bucket. (#4582)
add cb201556be HDDS-8446. Selective checks: handle change in ci.yaml (#4587)
add 90fd5f2adc HDDS-8444. Increase timeout of CI build (#4586)
add 9d066ea265 HDDS-8355. Intermittent failure in TestOMRatisSnapshots#testInstallSnapshot (#4592)
add 52db80f920 HDDS-8335. ReplicationManager: EC Mis and Under replication handlers should handle overloaded exceptions (#4593)
add bee5454d7b HDDS-8445. Move PlacementPolicy back to SCM (#4588)
add 475963125b HDDS-7951. [Snapshot] Clean SnapDiff job and report table (#4490)
add 0231bd7d05 HDDS-8261. Fail integration tests if RocksDB objects are leaked (#4590)
add e7cd92c467 HDDS-8455. Om supports read only administrators. (#4565)
add cf3d0e7f8a HDDS-8017. Add check in ClosingContainerHandler for RM to handle Missing Containers (#4598)
add de4c4c7f45 HDDS-6113. Mark TestBlockOutputStreamWithFailuresFlushDelay#testWatchForCommitDatanodeFailure as flaky
add 531ed34b55 HDDS-8461. Bump jetty to 9.4.51.v20230217 (#4594)
add 99bab79cef HDDS-8416. ReplicationManager: RatisUnderReplicationHandler should filter sources by highest BCSID (#4599)
add efaa2b1656 HDDS-8474. XceiverClient not closed properly in TestContainerStateMachineFailures (#4605)
add 7573c0934c HDDS-8421. Use FileChannel.size() in FilePerBlockStrategy to reduce file operations (#4570)
add cf13e76520 HDDS-8478. Fix module name in ScmRoles IntelliJ run config (#4610)
add b6033460c5 HDDS-8480. DB not closed properly in ContainerMapper (#4612)
add 0038dd6610 HDDS-8479. XceiverClient not closed properly in tests (#4611)
add 5b1319a8c2 HDDS-8277. [Snapshot] Parsing Bug in SSTDumptool (#4470)
add 8d15d5bea7 HDDS-7789. [Snapshot] Provide API for snapDiff from OFS. (#4491)
add e74fccb8f8 HDDS-7907. [hsync] KeyOutputStream is not thread safe. (#4601)
add 70357d9c3f HDDS-8468. Fix native compilation for JDK 11 and 17 (#4603)
add cc1d2b3984 HDDS-7995. Support incremental snapshot on OM (#4294)
add c422d99d81 HDDS-8452. Scm server side RPC support for Scm Decommissioning. (#4595)
add de3877fe7d HDDS-8466. Disable LegacyReplicationManager by default (#4604)
add a44722f934 HDDS-7884. ofs: Recursive delete of path of symlink with empty source fails (#4405)
add 677b6465c3 HDDS-8459. Ratis under replication handling in a rack aware environment doesn't work (#4616)
add e50e0a6321 HDDS-8475. Unexpected IOException should be returned as INTERNAL_ERROR. (#4609)
add 73809175c6 HDDS-8482. Let Ozone shell support multiple OM services (#4613)
add 0221502ff1 HDDS-8433. Intermittent failure in TestOzoneManagerSnapshotAcl (#4619)
add 298e7eee13 HDDS-7586. Allow user to create bucket with non-s3-naming-convention (#4524)
add 6a7c0706cb HDDS-7828. Make Ozone fs rm symbolic links command support posix behaviour (#4246)
add c616e8d63f HDDS-8417. Cap on queue of commands at datanode (#4618)
add 7df18f2647 HDDS-7551. Test in ec/ozonefs.robot is not executed in CI (#4625)
add 14ab7783b9 HDDS-7412. Cannot delete empty OBS bucket if prefix of other, non-empty bucket (#4606)
add c20881a9ef HDDS-8314. [Snapshot] SnapDiff job and compaction DAG/SST file pruning synchronization (#4553)
add 16ee8c8b83 HDDS-8495. Fix permissions and path handling for ozone sh token get command (#4620)
add 94d8219eea HDDS-8269. [Snapshot] Enable CI native building and testing (#4596)
add c7797432b7 HDDS-8224. Ozone network topology does not create pipelines based on rack aware policy. (#4623)
add 53b2a252e8 HDDS-7226. Allow existing nodes to be specified for Rack Aware Policy. (#4614)
add fa831957aa HDDS-8458. Intermittent timeout in TestBlockDeletion#testBlockDeletion (#4624)
add 5657009a3d Data Rendering code on SCM UI (#4608)
add 6eebe4523c HDDS-8369. Decommissioning with rack aware placement policy does not replicate to correct rack. (#4556)
add ad2b6b6e52 HDDS-8471. Ensure replication processors use a single queue for each iteration (#4627)
add 3858cd102b HDDS-8322. [Snapshot] Created various config for Snapdiff and moved few configs from OzoneConfigKeys to OMConfigKeys (#4511)
add 5861766353 HDDS-8510. Intermittent timeout in TestRootedOzoneFileSystem#testSnapshotDiff (#4632)
add ad9358fdc4 HDDS-8506. LeaseRecovery failing with NullPointer exception. (#4631)
add df4df20371 HDDS-7965. Quota needs to be updated correctly for Hsync (#4629)
add 90f3bba78e HDDS-8512. Bump Spring Framework to 5.3.27 (#4635)
add 4578a06353 HDDS-8435. [Snapshot] Handle Directory renames for FSO Buckets. (#4607)
add be65ec1259 HDDS-8451. [Snapshot] Catch Exception and log it in generateSnapshotDiffReport (#4600)
add b9057a2dcb HDDS-8505. ReplicationManager: Add configurable global replication limit (#4636)
add 977a271b87 HDDS-8365. Admin SCM command to decommission SCM. (#4563)
add bc9258c0ea HDDS-7943. Link Datanode WebUIs from the Node Status table on SCM WebUI (#4546)
add 643982e1ee HDDS-8515. Update README to include GitHub Discussions. (#4641)
add 73d1ff104c HDDS-8243. [hsync] Add a CLI to recover lease (#4514)
add ad2d125d26 HDDS-8347. Investigate possible race conditions on ContainerInfo in ContainerBalancer (#4643)
add ea4b01b93b HDDS-7935. [Snapshot] LRU Cache entries may get evicted/closed during long running processes (#4568)
add ebff2f1705 HDDS-8496. S3 to return not found for object head/set when keyinfo indicates a directory. (#4647)
add c708259b56 HDDS-8215. Recon - SCM DB Insights. (#4569)
add 74adc9ac29 HDDS-8494. Adjust replication queue limits for out-of-service nodes (#4645)
add 46f0dc5c1d HDDS-8525. Provide more info in assertions (#4653)
add c78c07fda3 HDDS-8462. Key in deleteTable uniqueness (#4602)
add c4d348c30d HDDS-8393. [Recon] Include Roboto font using fontsource instead (#4551)
add 1fda945f75 HDDS-8340. [Recon] Show the number of deleted containers in UI (#4555)
add 8ae8e220a5 HDDS-8092. Mark TestRecursiveAclWithFSO#testKeyDeleteAndRenameWithoutPermission as flaky (#4652)
add ecaa76b669 HDDS-8527. Update Ofs (HCFS) doc to describe shared tmp dir (#4648)
add bf74f7857b HDDS-7546. [Snapshot] Throw exception if the SnapshotDiff size is huge (#4633)
add 5e5d04c9be HDDS-8531. Remove outdated/stale code around TrashRecoverRequest. (#4656)
add 4dfe040a49 HDDS-8058. Gracefully handle invalid S3 request and authorization headers (#4473)
add fe10a99bbb HDDS-8473. ReplicationManager: Clear ContainerReplicaPendingOps when RM goes to running state (#4644)
add cdd96b8baf HDDS-7632. Add ability in Recon to track various Container Sizes across Ozone cluster. (#4107)
add 69e3cf3505 HDDS-8541. CodecRegistry is not thread safe. (#4658)
add a455597815 HDDS-8548. Migrate tests in hdds-erasurecode to JUnit5 (#4662)
add 831f608d28 HDDS-8166. [Snapshot] Add a config to enable or disable Ozone snapshot feature on OM (#4642)
add 759433c2ec HDDS-8499. Add mechanism to notify threads when OM double buffer flushed. (#4621)
add 2feb7388aa HDDS-8554. Thread pool size needs to be decreased in different order in ReplicationSupervisor (#4669)
add 9fa304f502 HDDS-8550. Mark testOverWriteKeyWithAndWithOutVersioning as flaky
add 7f7ec2397a HDDS-8557. Disable testOverWriteKeyWithAndWithOutVersioning
add 75f5a28f37 HDDS-8542. In RDBTable, add a put method using RocksDB ByteBuffer APIs. (#4666)
add 2281e06e92 HDDS-8521. Fix SCM certificate count check during OM & Datanode startup. (#4654)
add cf6592d421 HDDS-7889. Mark TestOzoneSnapshotRestore as flaky
add 5d68ac93f9 HDDS-8484. Allow Ozone volume name to have special characters (#4630)
add 9f1b0fc51e HDDS-8493. Intermittent timeout in TestDecommissionAndMaintenance#testSCMHandlesRestartForMaintenanceNode (#4671)
add 72b5de7548 HDDS-8518. Admin SCM decommission command primordial node support (#4649)
add 2dfac8a0f2 HDDS-8533. Ozone Cluster components (SCM, OM, DNs) fails to start with JDK 17 (#4659)
add ec62011ab3 HDDS-8514. Improve ozone admin cert list subcommand count limits (#4646)
add de922e26b6 HDDS-8551. Fix the generic type of CodecRegistry.getCodec (#4667)
add 297c3c1eb3 HDDS-7814. Vault-based remote S3 secret store (#4389)
add 557449e916 HDDS-8546. Ozone debug chunkinfo command throws GroupMismatchException for closed container (#4661)
add 5765ba8851 HDDS-8390. [Snapshot] Added active snapshots check before returning diff report to client and in diff report generation to fail fast. (#4617)
add 740be38cfa HDDS-8390. [Snapshot] Added active snapshots check before returning diff report to client and in diff report generation to fail fast. (#4617)
add 13a4f81353 HDDS-8289: Speed up FSO ListKeys, skip skipToFirst (#4650)
add df43d549dd HDDS-8539. Container DB open, but not found in DatanodeStoreCache (#4672)
add 0fcfe212e1 HDDS-8179. Datanode decommissioning blocked due to non-empty replica of deleting container (#4685)
add 290e5f2ba1 HDDS-8550. Intermittent failure in testOverWriteKeyWithAndWithOutVersioning (#4686)
add 5e8b806f05 HDDS-8382. Upgrade Ratis to 2.5.1 (#4664)
add 81f1364e31 HDDS-8532. Add config for factor of scaling up replication queue/threads in decommissioning nodes (#4687)
add 71087dd092 HDDS-8064. [Snapshot] Clean up FSO deletedDirTable as well during snapshot creation (#4651)
new 43aaa9545a HDDS-7734. Implement symmetric SecretKeys lifescycle management in SCM (#4194)
new 42e2cb302e HDDS-7830. SCM API for OM and Datanode to get secret keys (#4345)
new f966961491 HDDS-7831. Use symmetric secret key to sign and verify token (#4417)
new b1880fffae HDDS-8003. E2E integration test cases for block tokens (#4547)
new ba51dbacdf HDDS-7945. Integrate secret keys to SCM snapshot (#4549)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (783c90f72e)
\
N -- N -- N refs/heads/HDDS-7733-Symmetric-Tokens (ba51dbacdf)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ci.md | 12 +-
.github/workflows/ci.yml | 16 +-
CONTRIBUTING.md | 9 +
README.md | 6 +-
dev-support/ci/selective_ci_checks.bats | 24 +
dev-support/ci/selective_ci_checks.sh | 4 +
.../hadoop/hdds/scm/client/HddsClientUtils.java | 27 +-
.../ozone/client/io/BadDataLocationException.java | 9 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 4 +-
.../io/ECBlockReconstructedStripeInputStream.java | 42 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 9 +-
.../TestECBlockReconstructedStripeInputStream.java | 76 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 13 -
.../java/org/apache/hadoop/hdds/StringUtils.java | 26 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 15 +-
.../hdds/client/ReplicationConfigValidator.java | 52 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 37 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 4 +-
.../apache/hadoop/hdds/scm/RemoveSCMRequest.java | 112 ++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +
.../hadoop/hdds/scm/container/ContainerInfo.java | 8 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 7 +
.../protocol/StorageContainerLocationProtocol.java | 6 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 23 -
.../hdds/upgrade/BelongsToHDDSLayoutVersion.java | 28 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 6 +-
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 20 +-
.../org/apache/hadoop/hdds/utils/db/Codec.java | 64 +-
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 162 +++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 38 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 32 +
.../apache/hadoop/ozone/lease/LeaseManager.java | 31 +
.../org/apache/hadoop/util/ClosableIterator.java | 27 +-
.../java/org/apache/hadoop/util/MetricUtil.java | 12 +
.../common/src/main/resources/ozone-default.xml | 216 +++-
.../hadoop/hdds/client/TestReplicationConfig.java | 10 +-
.../client/TestReplicationConfigValidator.java | 114 +-
.../org/apache/hadoop/hdds/utils/TestIOUtils.java | 18 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 21 +-
.../helpers/BlockDeletingServiceMetrics.java | 15 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/helpers/DatanodeIdYaml.java | 39 +-
.../ozone/container/common/impl/ContainerSet.java | 7 +-
.../common/statemachine/DatanodeConfiguration.java | 25 +-
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../common/statemachine/StateContext.java | 10 +
.../commandhandler/DeleteBlocksCommandHandler.java | 30 +-
.../DeleteContainerCommandHandler.java | 32 +-
.../ReconstructECContainersCommandHandler.java | 1 -
.../SetNodeOperationalStateCommandHandler.java | 14 +-
.../common/states/datanode/InitDatanodeState.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 20 +-
.../transport/server/ratis/XceiverServerRatis.java | 15 +-
.../container/common/utils/DatanodeStoreCache.java | 16 +
.../ozone/container/common/volume/DbVolume.java | 6 +-
.../ozone/container/common/volume/HddsVolume.java | 5 +-
.../container/common/volume/MutableVolumeSet.java | 41 +-
.../container/common/volume/StorageVolume.java | 48 +-
.../container/common/volume/VolumeInfoMetrics.java | 9 +-
.../KeyValueContainerMetadataInspector.java | 157 ++-
.../ozone/container/keyvalue/KeyValueHandler.java | 17 +-
.../container/keyvalue/TarContainerPacker.java | 4 +-
.../container/keyvalue/helpers/ChunkUtils.java | 51 +
.../keyvalue/impl/FilePerBlockStrategy.java | 12 +-
.../keyvalue/impl/StreamDataChannelBase.java | 33 +-
.../background/BlockDeletingService.java | 2 +-
.../StaleRecoveringContainerScrubbingService.java | 8 +-
.../ozone/container/metadata/DatanodeTable.java | 5 +
.../metadata/SchemaOneDeletedBlocksTable.java | 5 +
.../container/ozoneimpl/ContainerController.java | 4 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 8 +-
.../container/replication/ReplicationServer.java | 40 +
.../replication/ReplicationSupervisor.java | 173 ++-
.../protocol/commands/CloseContainerCommand.java | 30 +
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 11 +-
.../container/common/TestDatanodeStateMachine.java | 2 +-
.../common/helpers/TestContainerUtils.java | 17 +-
.../common/helpers/TestDatanodeIdYaml.java | 99 +-
.../common/impl/TestContainerPersistence.java | 43 +
.../common/statemachine/TestStateContext.java | 3 +-
.../TestDeleteContainerCommandHandler.java | 47 +-
.../container/common/volume/TestHddsVolume.java | 19 +
.../common/volume/TestReservedVolumeSpace.java | 25 +-
.../container/common/volume/TestVolumeSet.java | 3 +-
.../keyvalue/ContainerTestVersionInfo.java | 5 +
.../container/keyvalue/TestKeyValueContainer.java | 7 +
.../TestKeyValueContainerIntegrityChecks.java | 3 +-
.../TestKeyValueContainerMetadataInspector.java | 197 +++-
.../container/keyvalue/TestTarContainerPacker.java | 5 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 13 +
.../ReplicationSupervisorScheduling.java | 10 +-
.../replication/TestReplicationConfig.java | 14 +
.../replication/TestReplicationSupervisor.java | 135 ++-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 12 +-
hadoop-hdds/docs/content/concept/Recon.md | 2 +-
hadoop-hdds/docs/content/concept/Recon.zh.md | 1 +
hadoop-hdds/docs/content/feature/Quota.md | 13 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 13 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 16 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 14 +-
hadoop-hdds/docs/content/feature/Topology.md | 2 +-
hadoop-hdds/docs/content/interface/Ofs.md | 38 +-
hadoop-hdds/docs/content/security/GDPR.md | 1 -
hadoop-hdds/docs/content/security/GDPR.zh.md | 3 -
.../docs/content/security/SecuringDatanodes.zh.md | 5 +-
hadoop-hdds/docs/content/security/SecuringS3.zh.md | 2 +-
.../docs/content/security/SecuringTDE.zh.md | 2 +-
.../docs/content/security/SecurityWithRanger.zh.md | 2 +-
.../ozone/erasurecode/TestCodecRegistry.java | 8 +-
.../apache/ozone/erasurecode/TestCoderBase.java | 6 +-
.../rawcoder/TestCodecRawCoderMapping.java | 20 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 12 +-
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 10 +-
.../rawcoder/TestNativeXORRawCoder.java | 10 +-
.../ozone/erasurecode/rawcoder/TestRSRawCoder.java | 4 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 2 +-
.../erasurecode/rawcoder/TestRawCoderBase.java | 22 +-
.../rawcoder/TestRawErasureCoderBenchmark.java | 2 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 2 +-
.../framework/dev-support/findbugsExcludeFile.xml | 4 -
...inerLocationProtocolClientSideTranslatorPB.java | 17 +
.../hadoop/hdds/security/OzoneSecretManager.java | 3 +-
.../hdds/security/ssl/MonitoringTimerTask.java | 77 --
.../security/ssl/PemFileBasedKeyStoresFactory.java | 84 +-
.../hdds/security/ssl/ReloadingX509KeyManager.java | 29 +-
.../security/ssl/ReloadingX509TrustManager.java | 18 +-
.../symmetric/DefaultSecretKeySignerClient.java | 16 +-
.../x509/certificate/client/CertificateClient.java | 6 +
.../client/CertificateNotification.java | 6 +-
.../client/DefaultCertificateClient.java | 74 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 1 -
.../hadoop/hdds/utils/DBCheckpointMetrics.java | 21 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 59 +-
.../apache/hadoop/hdds/utils/FaultInjector.java | 38 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 53 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 32 +-
.../hdds/utils/PrometheusMetricsSinkUtil.java | 2 +-
.../hadoop/hdds/utils/RDBSnapshotProvider.java | 233 ++++
...sDBStoreMBean.java => RocksDBStoreMetrics.java} | 143 +--
.../hadoop/hdds/utils/db/ByteArrayCodec.java | 2 +-
.../hadoop/hdds/utils/db}/ByteStringCodec.java | 39 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 129 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 58 +-
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 27 +-
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 19 +-
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 227 +++-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 33 +-
.../hadoop/hdds/utils/db/RDBCheckpointUtils.java | 70 ++
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 19 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 159 +--
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 12 +-
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 42 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 40 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 23 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 9 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 26 +-
.../src/main/resources/webapps/static/ozone.css | 31 +-
.../ssl/TestPemFileBasedKeyStoresFactory.java | 5 -
.../security/ssl/TestReloadingX509KeyManager.java | 46 +-
.../ssl/TestReloadingX509TrustManager.java | 46 +-
.../token/TestOzoneBlockTokenSecretManager.java | 6 -
.../certificate/authority/TestDefaultCAServer.java | 264 ++---
.../client/CertificateClientTestImpl.java | 15 +-
.../client/TestDefaultCertificateClient.java | 117 +-
.../client/TestDnCertificateClientInit.java | 4 +-
.../hadoop/hdds/utils/TestRDBSnapshotProvider.java | 239 ++++
.../org/apache/hadoop/hdds/utils/db/TestCodec.java | 130 +++
.../hadoop/hdds/utils/db/TestCodecRegistry.java | 69 ++
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 66 ++
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 140 ++-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 111 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 22 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 61 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +
.../src/main/proto/ScmAdminProtocol.proto | 12 +
.../interface-client/src/main/proto/hdds.proto | 12 +
.../hdds/utils/db/managed/ManagedRocksDB.java | 7 -
.../db/managed/ManagedRocksObjectMetrics.java | 9 +-
hadoop-hdds/pom.xml | 2 +-
hadoop-hdds/rocks-native/pom.xml | 27 +-
hadoop-hdds/rocks-native/src/CMakeLists.txt | 4 +-
.../utils/db/managed/ManagedSSTDumpIterator.java | 186 ++-
.../hdds/utils/db/managed/ManagedSSTDumpTool.java | 17 +-
.../src/main/patches/rocks-native.patch | 83 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 9 +
.../ozone/rocksdb/util/ManagedSstFileReader.java | 208 +++-
.../org/apache/ozone/rocksdb/util/RdbUtil.java | 11 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 589 ++++++----
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 217 ++--
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 11 +-
.../hdds/scm/PlacementPolicyValidateProxy.java | 0
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 58 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 17 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 7 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 14 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 58 +-
.../container/AbstractContainerReportHandler.java | 21 +-
.../scm/container/CloseContainerEventHandler.java | 57 +-
.../scm/container/balancer/ContainerBalancer.java | 33 +-
.../balancer/ContainerBalancerConfiguration.java | 26 +-
.../ContainerBalancerSelectionCriteria.java | 64 +-
.../container/balancer/ContainerBalancerTask.java | 34 +-
.../hdds/scm/container/balancer/MoveManager.java | 88 +-
.../algorithms/SCMContainerPlacementRackAware.java | 255 +++-
.../algorithms/SCMContainerPlacementRandom.java | 3 +-
....java => CommandTargetOverloadedException.java} | 4 +-
.../replication/ContainerHealthResult.java | 7 +-
.../replication/ContainerReplicaPendingOps.java | 51 +-
.../replication/ECContainerReplicaCount.java | 112 +-
.../replication/ECMisReplicationHandler.java | 60 +-
.../replication/ECOverReplicationHandler.java | 75 +-
.../replication/ECUnderReplicationHandler.java | 343 ++++--
.../replication/LegacyReplicationManager.java | 1 +
.../replication/MisReplicationHandler.java | 110 +-
.../replication/OverReplicatedProcessor.java | 26 +-
.../replication/RatisContainerReplicaCount.java | 2 +-
.../replication/RatisMisReplicationHandler.java | 34 +-
.../replication/RatisOverReplicationHandler.java | 94 +-
.../replication/RatisUnderReplicationHandler.java | 151 ++-
.../container/replication/ReplicationManager.java | 599 +++++++---
.../replication/ReplicationManagerUtil.java | 93 ++
.../container/replication/ReplicationQueue.java | 34 +-
.../replication/UnderReplicatedProcessor.java | 24 +-
.../replication/UnhealthyReplicationHandler.java | 11 +-
.../replication/UnhealthyReplicationProcessor.java | 58 +-
.../health/ClosingContainerHandler.java | 7 +
.../health/ECReplicationCheckHandler.java | 7 +-
.../replication/health/EmptyContainerHandler.java | 13 +-
.../health/RatisReplicationCheckHandler.java | 50 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 69 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 8 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 5 +-
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 4 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 3 +
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 12 +-
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 13 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 31 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 19 +-
.../apache/hadoop/hdds/scm/ha/SCMHAMetrics.java | 116 ++
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 3 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 43 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 3 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 11 -
.../hdds/scm/metadata/SCMMetadataStoreMetrics.java | 105 --
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 8 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 19 +
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 3 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 66 +-
.../pipeline/InsufficientDatanodesException.java | 7 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 29 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 14 +-
...inerLocationProtocolServerSideTranslatorPB.java | 15 +
.../hdds/scm/server/SCMClientProtocolServer.java | 28 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 18 +
.../hdds/scm/server/SCMSecurityProtocolServer.java | 35 +-
.../hdds/scm/server/StorageContainerManager.java | 159 ++-
.../main/resources/webapps/scm/scm-overview.html | 10 +-
.../src/main/resources/webapps/scm/scm.js | 51 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 23 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 17 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 8 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 9 +-
.../container/TestCloseContainerEventHandler.java | 43 +-
.../scm/container/TestContainerReportHandler.java | 104 ++
.../TestIncrementalContainerReportHandler.java | 122 ++
.../container/balancer/TestContainerBalancer.java | 88 +-
.../balancer/TestContainerBalancerTask.java | 223 +++-
.../scm/container/balancer/TestMoveManager.java | 33 +-
.../algorithms/TestContainerPlacementFactory.java | 9 +-
.../TestSCMContainerPlacementRackAware.java | 209 ++++
.../container/replication/ReplicationTestUtil.java | 179 ++-
.../TestContainerReplicaPendingOps.java | 23 +
.../replication/TestECContainerReplicaCount.java | 16 +
.../replication/TestECMisReplicationHandler.java | 61 +-
.../replication/TestECOverReplicationHandler.java | 142 ++-
.../replication/TestECUnderReplicationHandler.java | 373 +++++-
.../replication/TestLegacyReplicationManager.java | 17 +-
.../replication/TestMisReplicationHandler.java | 124 +-
.../replication/TestOverReplicatedProcessor.java | 58 +-
.../TestRatisContainerReplicaCount.java | 23 +
.../TestRatisMisReplicationHandler.java | 27 +-
.../TestRatisOverReplicationHandler.java | 121 +-
.../TestRatisUnderReplicationHandler.java | 106 +-
.../replication/TestReplicationManager.java | 563 +++++++--
.../replication/TestUnderReplicatedProcessor.java | 121 +-
.../health/TestClosingContainerHandler.java | 49 +
.../health/TestRatisReplicationCheckHandler.java | 107 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 6 +
.../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 263 +++++
.../hadoop/hdds/scm/ha/TestSCMHAMetrics.java | 69 ++
.../apache/hadoop/hdds/scm/ha}/package-info.java | 7 +-
.../scm/metadata/TestSCMMetadataStoreImpl.java | 67 --
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 7 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 8 +
.../scm/pipeline/TestPipelinePlacementFactory.java | 41 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 43 +-
.../scm/server/TestSCMClientProtocolServer.java | 95 ++
.../server/TestSCMUpdateServiceGrpcServer.java | 1 +
.../testutils/ReplicationNodeManagerMock.java | 9 +-
.../org/apache/ozone/test/GenericTestUtils.java | 15 +
.../hdds/scm/cli/ContainerOperationClient.java | 9 +
.../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 19 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 16 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 8 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 9 +
.../hadoop/ozone/client/OzoneClientFactory.java | 1 +
.../org/apache/hadoop/ozone/client/OzoneKey.java | 11 +-
.../client/checksum/ECFileChecksumHelper.java | 3 +-
.../checksum/ReplicatedFileChecksumHelper.java | 2 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 15 +-
.../ozone/client/protocol/ClientProtocol.java | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneClient.java | 5 +
.../hadoop/ozone/client/TestOzoneECClient.java | 41 +-
hadoop-ozone/common/pom.xml | 4 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 38 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 82 +-
.../apache/hadoop/ozone/om/S3InMemoryCache.java | 82 ++
.../org/apache/hadoop/ozone/om/S3SecretCache.java | 13 +-
.../apache/hadoop/ozone/om/S3SecretManager.java | 10 +-
.../hadoop/ozone/om/exceptions/OMException.java | 2 +-
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 42 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 25 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 36 +-
.../hadoop/ozone/om/helpers/OmKeyRenameInfo.java | 91 --
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 14 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 16 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 34 +-
.../hadoop/ozone/snapshot/SnapshotDiffReport.java | 253 ----
.../ozone/snapshot/SnapshotDiffReportOzone.java | 194 ++++
.../ozone/snapshot/SnapshotDiffResponse.java | 6 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 6 +
.../dev-support/checks/_mvn_unit_report.sh | 18 +-
hadoop-ozone/dev-support/checks/build.sh | 19 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 6 +
hadoop-ozone/dev-support/checks/junit.sh | 11 +-
hadoop-ozone/dev-support/checks/native_check.sh | 50 +
hadoop-ozone/dev-support/intellij/ozone-site.xml | 11 +-
.../intellij/runConfigurations/ScmRoles.xml | 2 +-
hadoop-ozone/dist/pom.xml | 8 +-
.../dist/src/main/compose/ozone-ha/docker-config | 2 +
.../dist/src/main/compose/ozone-ha/test.sh | 15 +-
.../src/main/compose/ozone-om-ha/docker-config | 3 +
.../dist/src/main/compose/ozone/docker-config | 5 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 3 +-
.../compose/ozonesecure-ha/docker-compose.yaml | 53 +
.../src/main/compose/ozonesecure-ha/docker-config | 38 +-
.../load.sh => ozonesecure-ha/docker-config-scm4} | 13 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 52 +-
.../src/main/compose/ozonesecure-mr/docker-config | 4 +-
.../non-ha/load.sh => ozonesecure-vault/.env} | 19 +-
.../compose/ozonesecure-vault/docker-compose.yaml | 138 +++
.../docker-config | 12 +-
.../krb5.conf | 0
.../{ozonesecure-ha => ozonesecure-vault}/test.sh | 22 +-
.../src/main/compose/ozonesecure/docker-config | 10 +-
.../dist/src/main/compose/ozonesecure/test.sh | 8 -
hadoop-ozone/dist/src/main/compose/test-all.sh | 6 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 50 +
.../dist/src/main/compose/upgrade/README.md | 93 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 76 +-
.../main/compose/upgrade/compose/ha/docker-config | 19 +-
.../src/main/compose/upgrade/compose/ha/load.sh | 4 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 16 +-
.../compose/upgrade/compose/non-ha/docker-config | 4 +
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +
.../compose/upgrade/compose/{ha => om-ha}/.env | 0
.../compose/{ha => om-ha}/docker-compose.yaml | 16 +-
.../upgrade/compose/{ha => om-ha}/docker-config | 4 +
.../compose/upgrade/compose/{ha => om-ha}/load.sh | 4 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 12 +-
.../dist/src/main/compose/upgrade/testlib.sh | 139 +--
.../upgrade/upgrades/manual-upgrade/driver.sh | 4 +-
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 --
.../callbacks/1.3.0/callback.sh} | 19 +-
.../{1.1.0-1.2.0 => callbacks/common}/callback.sh | 48 +-
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 34 +-
hadoop-ozone/dist/src/main/keytabs/HTTP.keytab | Bin 580 -> 580 bytes
hadoop-ozone/dist/src/main/keytabs/dn.keytab | Bin 278 -> 576 bytes
hadoop-ozone/dist/src/main/keytabs/hadoop.keytab | Bin 146 -> 146 bytes
hadoop-ozone/dist/src/main/keytabs/httpfs.keytab | Bin 458 -> 616 bytes
hadoop-ozone/dist/src/main/keytabs/jhs.keytab | Bin 142 -> 142 bytes
hadoop-ozone/dist/src/main/keytabs/nm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/om.keytab | Bin 278 -> 576 bytes
hadoop-ozone/dist/src/main/keytabs/recon.keytab | Bin 296 -> 606 bytes
hadoop-ozone/dist/src/main/keytabs/rm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/s3g.keytab | Bin 434 -> 586 bytes
hadoop-ozone/dist/src/main/keytabs/scm.keytab | Bin 586 -> 586 bytes
hadoop-ozone/dist/src/main/keytabs/testuser.keytab | Bin 606 -> 908 bytes
.../dist/src/main/keytabs/testuser2.keytab | Bin 154 -> 920 bytes
.../dist/src/main/keytabs/update-keytabs.sh | 19 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 3 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 10 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
.../src/main/smoketest/admincli/datanode.robot | 6 +
.../admincli/{scmha.robot => scmrole.robot} | 2 +-
.../dist/src/main/smoketest/basic/links.robot | 11 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 8 +-
.../dist/src/main/smoketest/commonlib.robot | 15 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../src/main/smoketest/debug/ozone-debug.robot | 2 +-
.../dist/src/main/smoketest/ec/basic.robot | 44 +
.../dist/src/main/smoketest/ec/ozonefs.robot | 61 -
.../main/smoketest/omha/om-leader-transfer.robot | 73 ++
.../dist/src/main/smoketest/omha/testOMHA.robot | 61 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 6 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 38 +-
.../dist/src/main/smoketest/s3/objecthead.robot | 14 +
.../src/main/smoketest/scmha/primordial-scm.robot | 60 +
.../main/smoketest/scmha/scm-decommission.robot | 60 +
.../main/smoketest/scmha/scm-leader-transfer.robot | 48 +
.../src/main/smoketest/security/admin-cert.robot | 4 +-
.../main/smoketest/security/ozone-secure-fs.robot | 22 +-
.../smoketest/security/ozone-secure-tenant.robot | 2 +-
.../smoketest/security/ozone-secure-token.robot | 18 +-
.../main/smoketest/snapshot/snapshot-acls.robot | 2 +-
.../src/main/smoketest/snapshot/snapshot-sh.robot | 4 +-
.../smoketest/upgrade/check-finalization.robot | 50 +
.../src/main/smoketest/upgrade/check-mlv.robot | 41 -
.../dist/src/main/smoketest/upgrade/finalize.robot | 6 +-
.../dist/src/main/smoketest/upgrade/prepare.robot | 1 +
hadoop-ozone/dist/src/shell/ozone/ozone | 26 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 5 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 8 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 157 ++-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 97 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 155 +++
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 10 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 10 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 22 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 159 +--
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 64 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 105 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 7 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 7 +-
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 8 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 62 +-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 15 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 8 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 296 ++++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +
.../hadoop/fs/ozone/contract/OzoneContract.java | 13 +-
.../ozone/contract/rooted/RootedOzoneContract.java | 4 +
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 9 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 9 +-
.../metrics/TestSCMContainerManagerMetrics.java | 11 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 6 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 36 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 31 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../org/apache/hadoop/ozone/RatisTestHelper.java | 9 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 2 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 25 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 -
.../org/apache/hadoop/ozone/TestSecretKeysApi.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 298 ++---
.../hadoop/ozone/TestStorageContainerManager.java | 35 +-
.../ozone/TestStorageContainerManagerHelper.java | 40 +-
.../ozone/client/TestOzoneClientFactory.java | 8 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 2 +
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 2 +
.../client/rpc/TestBlockDataStreamOutput.java | 2 +
.../ozone/client/rpc/TestBlockOutputStream.java | 2 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 2 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 2 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 3 +-
.../rpc/TestCloseContainerHandlingByClient.java | 7 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 323 +++---
.../rpc/TestContainerReplicationEndToEnd.java | 15 +-
.../client/rpc/TestContainerStateMachine.java | 14 +-
.../TestContainerStateMachineFailureOnRead.java | 14 +-
.../rpc/TestContainerStateMachineFailures.java | 110 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +
.../rpc/TestContainerStateMachineStream.java | 2 +
.../client/rpc/TestDeleteWithSlowFollower.java | 31 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +
.../ozone/client/rpc/TestECKeyOutputStream.java | 4 +-
.../client/rpc/TestFailureHandlingByClient.java | 2 +
.../rpc/TestFailureHandlingByClientFlushDelay.java | 2 +
.../client/rpc/TestHybridPipelineOnDatanode.java | 2 +
.../rpc/TestMultiBlockWritesWithDnFailures.java | 2 +
.../client/rpc/TestOzoneAtRestEncryption.java | 101 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 7 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 2 +
.../rpc/TestOzoneClientRetriesOnExceptions.java | 2 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 198 ++--
.../TestOzoneRpcClientWithKeyLatestVersion.java | 5 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 6 +-
.../ozone/client/rpc/TestWatchForCommit.java | 215 ++--
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +-
.../ozone/container/TestContainerReplication.java | 2 +
.../ozone/container/TestECContainerRecovery.java | 11 +-
.../commandhandler/TestBlockDeletion.java | 89 +-
.../TestCloseContainerByPipeline.java | 2 +
.../commandhandler/TestCloseContainerHandler.java | 19 +-
.../commandhandler/TestDeleteContainerHandler.java | 5 +-
.../TestRefreshVolumeUsageHandler.java | 69 +-
.../container/ozoneimpl/TestOzoneContainer.java | 7 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 29 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 3 +-
.../container/server/TestContainerServer.java | 6 +
.../hadoop/ozone/debug/TestLeaseRecoverer.java | 146 +++
.../apache/hadoop/ozone/debug}/package-info.java | 17 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 4 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 7 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 8 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 74 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 7 +-
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 7 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 2 +
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 14 +-
.../ozone/om/TestBucketLayoutWithOlderClient.java | 9 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 138 +--
.../ozone/om/TestContainerReportWithKeys.java | 5 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 11 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 7 +-
.../org/apache/hadoop/ozone/om/TestLDBCli.java | 517 +++++----
.../hadoop/ozone/om/TestListKeysWithFSO.java | 7 +-
.../org/apache/hadoop/ozone/om/TestListStatus.java | 7 +-
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 8 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 370 +++++-
.../hadoop/ozone/om/TestOMEpochForNonRatis.java | 8 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 439 ++++++-
.../ozone/om/TestOMStartupWithBucketLayout.java | 17 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 8 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 8 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 39 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 15 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 23 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 11 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 117 +-
.../hadoop/ozone/om/TestOmSnapshotDisabled.java | 103 ++
.../hadoop/ozone/om/TestOmSnapshotFileSystem.java | 107 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 32 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 10 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 6 +-
.../hadoop/ozone/om/TestOzoneManagerHAMetrics.java | 104 ++
.../ozone/om/TestOzoneManagerHAWithData.java | 93 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 36 +-
.../om/TestOzoneManagerListVolumesSecure.java | 544 +++++++++
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 10 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 293 ++---
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 9 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 7 +
.../om/multitenant/TestMultiTenantVolume.java | 37 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 14 +-
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 56 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 10 +-
.../om/snapshot/TestOzoneSnapshotRestore.java | 21 +-
.../ozone/parser/TestOzoneHARatisLogParser.java | 8 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 7 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 29 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 17 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +
.../ozone/scm/TestStorageContainerManagerHA.java | 213 ++--
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 2 +
.../hadoop/ozone/scm/TestXceiverClientManager.java | 311 ++---
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 175 +--
.../scm/node/TestDecommissionAndMaintenance.java | 18 +-
.../org/apache/hadoop/ozone/scm}/package-info.java | 7 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 8 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 7 +-
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 181 +++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 77 +-
.../impl/StatemachineImplTestUtil.java | 17 +-
.../src/test/resources/log4j.properties | 1 +
.../src/test/resources/ozone-site.xml | 20 +
.../src/main/proto/OmClientProtocol.proto | 38 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 40 +-
.../ozone/om/codec/OmDBDiffReportEntryCodec.java | 28 +-
.../ozone/om/codec/OmKeyRenameInfoCodec.java | 55 -
hadoop-ozone/ozone-manager/pom.xml | 35 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 51 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 216 +++-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 43 +
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 5 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 6 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 332 ++++--
.../apache/hadoop/ozone/om/OmMetadataReader.java | 28 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 23 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 723 ++++++++++--
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 81 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 23 +-
.../hadoop/ozone/om/OzoneListStatusHelper.java | 12 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 272 +++--
.../hadoop/ozone/om/S3SecretManagerImpl.java | 21 +-
.../hadoop/ozone/om/SnapshotChainManager.java | 64 +-
.../hadoop/ozone/om/SstFilteringService.java | 9 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 24 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 74 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 32 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 42 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 34 +-
.../OmRatisSnapshotProvider.java} | 122 +-
.../{snapshot => ratis_snapshot}/package-info.java | 2 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 2 +-
.../ozone/om/request/OMClientRequestUtils.java | 52 +
.../om/request/bucket/OMBucketCreateRequest.java | 15 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 7 +-
.../om/request/bucket/OMBucketSetOwnerRequest.java | 3 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 28 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 3 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../request/file/OMFileCreateRequestWithFSO.java | 2 +-
.../om/request/file/OMRecoverLeaseRequest.java | 239 ++++
.../om/request/key/OMAllocateBlockRequest.java | 3 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 23 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 69 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 46 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 3 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 36 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 38 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 69 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 11 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 3 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 5 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 3 +-
.../om/request/key/OMTrashRecoverRequest.java | 143 ---
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 5 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 7 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 6 +-
.../S3InitiateMultipartUploadRequest.java | 7 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../multipart/S3MultipartUploadAbortRequest.java | 5 +-
.../S3MultipartUploadCommitPartRequest.java | 6 +-
.../S3MultipartUploadCompleteRequest.java | 14 +-
.../om/request/s3/security/OMSetSecretRequest.java | 2 +-
.../om/request/s3/security/S3GetSecretRequest.java | 3 +-
.../request/s3/security/S3RevokeSecretRequest.java | 3 +-
.../tenant/OMSetRangerServiceVersionRequest.java | 3 +-
.../s3/tenant/OMTenantAssignAdminRequest.java | 3 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 8 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 34 +-
.../request/s3/tenant/OMTenantDeleteRequest.java | 5 +-
.../s3/tenant/OMTenantRevokeAdminRequest.java | 3 +-
.../tenant/OMTenantRevokeUserAccessIdRequest.java | 15 +-
.../security/OMCancelDelegationTokenRequest.java | 3 +-
.../security/OMGetDelegationTokenRequest.java | 3 +-
.../security/OMRenewDelegationTokenRequest.java | 3 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 62 +-
.../request/snapshot/OMSnapshotDeleteRequest.java | 7 +-
.../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 13 +-
.../OMSnapshotPurgeRequest.java} | 54 +-
.../request/upgrade/OMFinalizeUpgradeRequest.java | 4 +-
.../om/request/validation/RequestValidations.java | 26 +-
.../om/request/volume/OMVolumeCreateRequest.java | 3 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 6 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 5 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 9 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 29 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 3 +-
.../om/response/file/OMRecoverLeaseResponse.java | 80 ++
.../response/key/AbstractOMKeyDeleteResponse.java | 22 +-
.../key/OMDirectoriesPurgeResponseWithFSO.java | 56 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 26 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 16 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 2 +
.../ozone/om/response/key/OMKeyPurgeResponse.java | 31 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 20 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 30 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 2 +
.../om/response/key/OMKeysRenameResponse.java | 17 +-
.../om/response/key/OMTrashRecoverResponse.java | 68 --
.../snapshot/OMSnapshotCreateResponse.java | 62 +-
.../OMSnapshotMoveDeletedKeysResponse.java | 44 +-
.../response/snapshot/OMSnapshotPurgeResponse.java | 145 +++
.../hadoop/ozone/om/s3/LocalS3StoreProvider.java | 21 +-
.../hadoop/ozone/om/s3/S3SecretCacheProvider.java | 57 +
.../om/s3/S3SecretStoreConfigurationKeys.java | 42 +-
.../hadoop/ozone/om/s3/S3SecretStoreProvider.java | 25 +-
.../apache/hadoop/ozone/om/s3/package-info.java} | 11 +-
.../om/service/AbstractKeyDeletingService.java | 457 ++++++++
.../ozone/om/service/DirectoryDeletingService.java | 298 +----
.../ozone/om/service/KeyDeletingService.java | 212 +---
.../ozone/om/service/OpenKeyCleanupService.java | 21 +-
.../ozone/om/service/SnapshotDeletingService.java | 369 +++++-
.../om/service/SnapshotDiffCleanupService.java | 254 ++++
.../hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 144 +++
.../hadoop/ozone/om/snapshot/PersistentList.java | 6 +-
.../hadoop/ozone/om/snapshot/PersistentMap.java | 7 +
.../hadoop/ozone/om/snapshot/PersistentSet.java | 4 +-
.../om/snapshot/RequireSnapshotFeatureState.java | 24 +-
.../RequireSnapshotFeatureStateAspect.java | 122 ++
.../ozone/om/snapshot/RocksDbPersistentList.java | 49 +-
.../ozone/om/snapshot/RocksDbPersistentMap.java | 71 +-
.../ozone/om/snapshot/RocksDbPersistentSet.java | 15 +-
.../hadoop/ozone/om/snapshot/SnapshotDiffJob.java | 207 ++++
.../ozone/om/snapshot/SnapshotDiffManager.java | 1226 +++++++++++++++-----
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 137 +++
.../hadoop/ozone/om/snapshot/package-info.java | 2 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../hadoop/ozone/security/AWSV4AuthValidator.java | 18 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 68 +-
.../src/main/resources/META-INF/aop.xml | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 8 +-
.../ozone/om/TestOMMultiTenantManagerImpl.java | 2 +-
.../hadoop/ozone/om/TestOMTenantCreateRequest.java | 182 +++
.../hadoop/ozone/om/TestOmMetadataManager.java | 5 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 131 ++-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 32 +-
.../hadoop/ozone/om/TestSstFilteringService.java | 4 +-
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 150 +++
...TestOzoneManagerDoubleBufferWithOMResponse.java | 8 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 4 +-
.../ozone/om/request/OMRequestTestUtils.java | 138 +--
.../request/TestOMClientRequestWithUserInfo.java | 14 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 4 +
.../request/bucket/TestOMBucketCreateRequest.java | 111 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 21 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 5 +-
.../om/request/file/TestOMRecoverLeaseRequest.java | 314 +++++
.../TestOMDirectoriesPurgeRequestAndResponse.java | 5 +-
.../om/request/key/TestOMKeyCommitRequest.java | 83 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 4 +
.../om/request/key/TestOMKeyCreateRequest.java | 10 +
.../key/TestOMKeyPurgeRequestAndResponse.java | 114 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 17 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 8 +
.../s3/security/TestS3GetSecretRequest.java | 10 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 179 +--
.../snapshot/TestOMSnapshotDeleteRequest.java | 19 +-
.../TestOMSnapshotPurgeRequestAndResponse.java | 312 +++++
.../request/volume/TestOMVolumeCreateRequest.java | 72 ++
.../volume/TestOMVolumeSetQuotaRequest.java | 27 +
.../ozone/om/response/TestCleanupTableInfo.java | 5 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 5 +-
.../om/response/key/TestOMKeyCommitResponse.java | 24 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 12 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 9 +-
.../om/response/key/TestOMKeyRenameResponse.java | 11 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 5 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 3 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 19 +-
.../s3/multipart/TestS3MultipartResponse.java | 5 +-
.../snapshot/OMSnapshotResponseTestUtil.java | 69 ++
.../snapshot/TestOMSnapshotCreateResponse.java | 171 ++-
.../snapshot/TestOMSnapshotDeleteResponse.java | 82 +-
.../ozone/om/service/TestKeyDeletingService.java | 71 ++
.../om/service/TestOpenKeyCleanupService.java | 24 +-
.../om/service/TestSnapshotDeletingService.java | 3 +
.../om/service/TestSnapshotDiffCleanupService.java | 389 +++++++
.../om/snapshot/SnapshotFeatureEnabledUtil.java | 35 +-
.../TestRequireSnapshotFeatureStateAspect.java | 59 +
.../om/snapshot/TestRocksDbPersistentList.java | 122 +-
.../om/snapshot/TestRocksDbPersistentMap.java | 129 +-
.../om/snapshot/TestRocksDbPersistentSet.java | 147 ++-
.../ozone/security/TestAWSV4AuthValidator.java | 29 +-
.../security/TestOmCertificateClientInit.java | 4 +-
.../TestOzoneDelegationTokenSecretManager.java | 1 +
.../security/acl/TestOzoneAdministrators.java | 80 ++
.../security/acl/TestOzoneNativeAuthorizer.java | 9 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 43 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 62 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 10 +
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 86 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 274 +++--
.../fs/ozone/CapableOzoneFSOutputStream.java | 9 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 5 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 23 +-
.../org/apache/hadoop/fs/ozone/OzoneFsDelete.java | 206 ++++
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 10 +
.../hadoop/fs/ozone/OzonePathCapabilities.java | 51 +
.../hadoop/fs/ozone/TestBasicOzoneFileSystems.java | 12 +
.../apache/hadoop/fs/ozone/TestOzoneFsShell.java | 68 ++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 16 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 16 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 16 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 16 +-
hadoop-ozone/pom.xml | 10 +-
hadoop-ozone/recon-codegen/pom.xml | 10 +-
.../recon/schema/UtilizationSchemaDefinition.java | 14 +
hadoop-ozone/recon/pom.xml | 9 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 17 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 5 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 8 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 30 +-
.../hadoop/ozone/recon/api/BlocksEndPoint.java | 174 +++
.../ozone/recon/api/ClusterStateEndpoint.java | 56 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 147 ++-
.../ozone/recon/api/UtilizationEndpoint.java | 55 +-
.../ozone/recon/api/handlers/BucketHandler.java | 6 +
.../ozone/recon/api/handlers/EntityHandler.java | 4 +-
.../recon/api/handlers/LegacyBucketHandler.java | 145 +--
.../recon/api/handlers/RootEntityHandler.java | 4 +-
.../recon/api/handlers/VolumeEntityHandler.java | 4 +-
.../recon/api/types/ClusterStateResponse.java | 52 +
.../api/types/ContainerBlocksInfoWrapper.java | 82 ++
.../recon/api/types/ContainerStateCounts.java | 62 +
.../recon/api/types/DeletedContainerInfo.java | 142 +++
.../api/types/FileSizeDistributionResponse.java | 8 +-
.../hadoop/ozone/recon/api/types/NSSummary.java | 6 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 18 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 9 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 9 +-
.../scm/ReconStorageContainerManagerFacade.java | 71 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 14 +-
.../impl/StorageContainerServiceProviderImpl.java | 10 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 38 +-
.../ozone/recon/tasks/ContainerSizeCountTask.java | 366 ++++++
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 4 +-
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 19 +
.../main/resources/webapps/recon/WEB-INF/web.xml | 4 +
.../webapps/recon/ozone-recon-web/api/db.json | 1 +
.../webapps/recon/ozone-recon-web/package.json | 1 +
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 18 +-
.../recon/ozone-recon-web/public/index.html | 1 -
.../ozone-recon-web/public/static/media/LICENSE | 15 +
.../webapps/recon/ozone-recon-web/src/index.tsx | 3 +
.../src/views/overview/overview.tsx | 11 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 52 +
.../hadoop/ozone/recon/api/TestBlocksEndPoint.java | 281 +++++
.../ozone/recon/api/TestContainerEndpoint.java | 489 +++++++-
.../ozone/recon/api/TestContainerStateCounts.java | 165 +++
.../hadoop/ozone/recon/api/TestEndpoints.java | 93 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 2 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 2 +-
.../recon/api/TestTotalOpenContainerCount.java | 396 -------
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../TestUtilizationSchemaDefinition.java | 24 +
.../impl/TestOzoneManagerServiceProviderImpl.java | 5 +
.../recon/tasks/TestContainerKeyMapperTask.java | 221 +++-
.../recon/tasks/TestContainerSizeCountTask.java | 157 +++
.../ozone/recon/tasks/TestNSSummaryTask.java | 15 +-
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 25 +-
.../recon/tasks/TestNSSummaryTaskWithLegacy.java | 27 +-
.../{recon-codegen => s3-secret-store}/pom.xml | 53 +-
.../S3SecretRemoteStoreConfigurationKeys.java | 59 +
.../hadoop/ozone/s3/remote/package-info.java} | 10 +-
.../remote/vault/VaultS3SecretStorageProvider.java | 23 +-
.../ozone/s3/remote/vault/VaultS3SecretStore.java | 210 ++++
.../s3/remote/vault/VaultS3SecretStoreBuilder.java | 174 +++
.../ozone/s3/remote/vault/auth/AppRoleAuth.java | 57 +
.../hadoop/ozone/s3/remote/vault/auth/Auth.java | 25 +-
.../ozone/s3/remote/vault/auth/AuthType.java | 51 +
.../s3/remote/vault/auth/DirectTokenAuth.java | 25 +-
.../ozone/s3/remote/vault/auth/package-info.java} | 10 +-
.../ozone/s3/remote/vault/package-info.java} | 11 +-
.../s3/remote/vault/VaultS3SecretStoreTest.java | 193 +++
hadoop-ozone/s3gateway/pom.xml | 5 +
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 32 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 5 +
.../ozone/s3/commontypes/EncodingTypeObject.java | 4 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 44 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 72 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 6 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 377 ++++--
.../s3/signature/AuthorizationV4HeaderParser.java | 41 +-
.../s3/signature/AuthorizationV4QueryParser.java | 135 ++-
.../ozone/s3/signature/StringToSignProducer.java | 97 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 3 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 31 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 29 +
.../hadoop/ozone/s3/endpoint/TestListParts.java | 8 +-
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 20 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 3 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 13 +-
.../signature/TestAuthorizationV4HeaderParser.java | 24 +
.../signature/TestAuthorizationV4QueryParser.java | 210 +++-
.../s3/signature/TestStringToSignProducer.java | 194 +++-
.../ozone/admin/scm/DecommissionScmSubcommand.java | 61 +
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 3 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 211 ++--
.../org/apache/hadoop/ozone/debug/DBScanner.java | 447 +++----
.../apache/hadoop/ozone/debug/LeaseRecoverer.java | 84 ++
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 7 +
.../apache/hadoop/ozone/debug/ReadReplicas.java | 167 +--
.../ozone/debug/container/ContainerCommands.java | 5 +
.../ozone/debug/container/InspectSubcommand.java | 74 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 118 +-
.../ozone/freon/ClosedContainerReplicator.java | 24 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 452 ++++++++
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 25 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 18 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 8 +-
.../hadoop/ozone/shell/token/GetTokenHandler.java | 9 +
.../hadoop/ozone/shell/token/TokenOption.java | 9 +-
.../ozone/shell/volume/ListVolumeHandler.java | 2 +-
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 2 +-
.../ozone/scm/TestDecommissionScmSubcommand.java | 108 ++
.../shell/TestOzoneAddressClientCreation.java | 12 +
pom.xml | 28 +-
905 files changed, 34893 insertions(+), 11284 deletions(-)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/RemoveSCMRequest.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentList.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/BelongsToHDDSLayoutVersion.java (63%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentSet.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java (55%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentSet.java => hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestIOUtils.java (75%)
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretCache.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/FaultInjector.java (59%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/{RocksDBStoreMBean.java => RocksDBStoreMetrics.java} (70%)
rename hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata => framework/src/main/java/org/apache/hadoop/hdds/utils/db}/ByteStringCodec.java (63%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestCodec.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestCodecRegistry.java
rename hadoop-hdds/{common => server-scm}/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java (91%)
rename hadoop-hdds/{common => server-scm}/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java (100%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{AllSourcesOverloadedException.java => CommandTargetOverloadedException.java} (88%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAMetrics.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreMetrics.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAMetrics.java
copy {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot => hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha}/package-info.java (87%)
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSCMMetadataStoreImpl.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3InMemoryCache.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
create mode 100644 hadoop-ozone/dev-support/checks/native_check.sh
copy hadoop-ozone/dist/src/main/compose/{upgrade/compose/non-ha/load.sh => ozonesecure-ha/docker-config-scm4} (73%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/compose/{upgrade/compose/non-ha/load.sh => ozonesecure-vault/.env} (73%)
mode change 100755 => 100644
create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-compose.yaml
copy hadoop-ozone/dist/src/main/compose/{ozonesecure => ozonesecure-vault}/docker-config (93%)
copy hadoop-ozone/dist/src/main/compose/{ozonesecure-ha => ozonesecure-vault}/krb5.conf (100%)
copy hadoop-ozone/dist/src/main/compose/{ozonesecure-ha => ozonesecure-vault}/test.sh (76%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha => om-ha}/.env (100%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha => om-ha}/docker-compose.yaml (94%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha => om-ha}/docker-config (93%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha => om-ha}/load.sh (89%)
delete mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
copy hadoop-ozone/dist/src/main/compose/upgrade/{compose/non-ha/load.sh => upgrades/non-rolling-upgrade/callbacks/1.3.0/callback.sh} (61%)
rename hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/{1.1.0-1.2.0 => callbacks/common}/callback.sh (52%)
rename hadoop-ozone/dist/src/main/smoketest/admincli/{scmha.robot => scmrole.robot} (98%)
delete mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/ozonefs.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/om-leader-transfer.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/scmha/primordial-scm.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/scmha/scm-decommission.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/scmha/scm-leader-transfer.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot
delete mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/check-mlv.robot
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
copy hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot => integration-test/src/test/java/org/apache/hadoop/ozone/debug}/package-info.java (74%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotDisabled.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetrics.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumesSecure.java
copy hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot => integration-test/src/test/java/org/apache/hadoop/ozone/scm}/package-info.java (87%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AllSourcesOverloadedException.java => hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java (67%)
delete mode 100644 hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java
rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{snapshot/OzoneManagerSnapshotProvider.java => ratis_snapshot/OmRatisSnapshotProvider.java} (65%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{snapshot => ratis_snapshot}/package-info.java (94%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/{key/OMKeyPurgeRequest.java => snapshot/OMSnapshotPurgeRequest.java} (54%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMRecoverLeaseResponse.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/LocalS3StoreProvider.java (58%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretCacheProvider.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretStoreConfigurationKeys.java (50%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AllSourcesOverloadedException.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretStoreProvider.java (65%)
copy hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx => ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/package-info.java} (82%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureState.java (63%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureStateAspect.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffJob.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMTenantCreateRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotResponseTestUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java
copy hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeXORRawCoder.java => hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/SnapshotFeatureEnabledUtil.java (56%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRequireSnapshotFeatureStateAspect.java
create mode 100644 hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsDelete.java
create mode 100644 hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzonePathCapabilities.java
create mode 100644 hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsShell.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/BlocksEndPoint.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerBlocksInfoWrapper.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DeletedContainerInfo.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerSizeCountTask.java
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/public/static/media/LICENSE
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestBlocksEndPoint.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerSizeCountTask.java
copy hadoop-ozone/{recon-codegen => s3-secret-store}/pom.xml (58%)
create mode 100644 hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/S3SecretRemoteStoreConfigurationKeys.java
copy hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx => s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/package-info.java} (82%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java => hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStorageProvider.java (65%)
create mode 100644 hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStore.java
create mode 100644 hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStoreBuilder.java
create mode 100644 hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/AppRoleAuth.java
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AllSourcesOverloadedException.java => hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/Auth.java (60%)
create mode 100644 hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/AuthType.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java => hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/DirectTokenAuth.java (57%)
copy hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx => s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/package-info.java} (82%)
copy hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx => s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/package-info.java} (82%)
create mode 100644 hadoop-ozone/s3-secret-store/src/test/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStoreTest.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/DecommissionScmSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/LeaseRecoverer.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java
copy hadoop-ozone/{dev-support/checks/build.sh => tools/src/main/resources/META-INF/services/org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory} (74%)
mode change 100755 => 100644
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestDecommissionScmSubcommand.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 05/05: HDDS-7945. Integrate secret keys to SCM snapshot (#4549)
Posted by du...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
duong pushed a commit to branch HDDS-7733-Symmetric-Tokens
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit ba51dbacdff19371c3d2649f34645442046b604c
Author: Duong Nguyen <du...@gmail.com>
AuthorDate: Tue May 9 11:08:24 2023 -0700
HDDS-7945. Integrate secret keys to SCM snapshot (#4549)
---
.../SCMSecurityProtocolFailoverProxyProvider.java | 2 +-
.../SingleSCMSecurityProtocolProxyProvider.java | 55 ++++
.../symmetric/DefaultSecretKeySignerClient.java | 16 +-
.../hdds/security/symmetric/SecretKeyManager.java | 4 +
.../hdds/security/symmetric/SecretKeyState.java | 5 +
.../security/symmetric/SecretKeyStateImpl.java | 9 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 11 +
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 8 +
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 26 ++
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 6 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 28 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 4 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 2 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 10 +-
.../org/apache/hadoop/ozone/TestSecretKeysApi.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 4 +-
.../TestSecretKeySnapshot.java} | 306 +++++++++------------
18 files changed, 309 insertions(+), 191 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java
index 18d1be6b43..5a95e1ffd7 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java
@@ -289,7 +289,7 @@ public class SCMSecurityProtocolFailoverProxyProvider implements
for (Map.Entry<String, ProxyInfo<SCMSecurityProtocolPB>> proxy :
scmProxies.entrySet()) {
if (proxy.getValue() != null) {
- RPC.stopProxy(proxy.getValue());
+ RPC.stopProxy(proxy.getValue().proxy);
}
scmProxies.remove(proxy.getKey());
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SingleSCMSecurityProtocolProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SingleSCMSecurityProtocolProxyProvider.java
new file mode 100644
index 0000000000..a69522d14f
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SingleSCMSecurityProtocolProxyProvider.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.proxy;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Proxy provider for
+ * {@link org.apache.hadoop.hdds.protocol.SCMSecurityProtocol} against a single
+ * SCM node (no fail-over).
+ */
+public class SingleSCMSecurityProtocolProxyProvider
+ extends SCMSecurityProtocolFailoverProxyProvider {
+ private final String scmNodeId;
+
+ public SingleSCMSecurityProtocolProxyProvider(
+ ConfigurationSource conf,
+ UserGroupInformation userGroupInformation,
+ String scmNodeId) {
+ super(conf, userGroupInformation);
+ this.scmNodeId = scmNodeId;
+ }
+
+ @Override
+ public synchronized String getCurrentProxySCMNodeId() {
+ return scmNodeId;
+ }
+
+ @Override
+ public synchronized void performFailover(SCMSecurityProtocolPB currentProxy) {
+ // do nothing.
+ }
+
+ @Override
+ public synchronized void performFailoverToAssignedLeader(String newLeader,
+ Exception e) {
+ // do nothing.
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java
index a1056f9139..e29254c753 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java
@@ -72,14 +72,16 @@ public class DefaultSecretKeySignerClient implements SecretKeySignerClient {
@Override
public void stop() {
- executorService.shutdown();
- try {
- if (executorService.awaitTermination(1, TimeUnit.MINUTES)) {
- executorService.shutdownNow();
+ if (executorService != null) {
+ executorService.shutdown();
+ try {
+ if (executorService.awaitTermination(1, TimeUnit.MINUTES)) {
+ executorService.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while shutting down executor service.", e);
+ Thread.currentThread().interrupt();
}
- } catch (InterruptedException e) {
- LOG.error("Interrupted while shutting down executor service.", e);
- Thread.currentThread().interrupt();
}
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
index f7a481cc05..8685a7fb52 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
@@ -143,6 +143,10 @@ public class SecretKeyManager implements SecretKeyClient {
return state.getSortedKeys();
}
+ public void reinitialize(List<ManagedSecretKey> secretKeys) {
+ state.reinitialize(secretKeys);
+ }
+
private boolean shouldRotate(ManagedSecretKey currentKey) {
Duration established = between(currentKey.getCreationTime(), Instant.now());
return established.compareTo(rotationDuration) >= 0;
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
index 7b510a10b2..43518b901a 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
@@ -52,4 +52,9 @@ public interface SecretKeyState {
*/
@Replicate
void updateKeys(List<ManagedSecretKey> newKeys) throws TimeoutException;
+
+ /**
+ * Update SecretKeys from a snapshot from SCM leader.
+ */
+ void reinitialize(List<ManagedSecretKey> secretKeys);
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
index 727b005d2b..b3f0ae55d9 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
@@ -106,6 +106,10 @@ public final class SecretKeyStateImpl implements SecretKeyState {
*/
@Override
public void updateKeys(List<ManagedSecretKey> newKeys) {
+ updateKeysInternal(newKeys);
+ }
+
+ private void updateKeysInternal(List<ManagedSecretKey> newKeys) {
LOG.info("Updating keys with {}", newKeys);
lock.writeLock().lock();
try {
@@ -127,4 +131,9 @@ public final class SecretKeyStateImpl implements SecretKeyState {
lock.writeLock().unlock();
}
}
+
+ @Override
+ public void reinitialize(List<ManagedSecretKey> secretKeys) {
+ updateKeysInternal(secretKeys);
+ }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
index f120ffad33..7c896c0bf1 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdds.scm.proxy.SCMClientConfig;
import org.apache.hadoop.hdds.scm.proxy.SCMSecurityProtocolFailoverProxyProvider;
+import org.apache.hadoop.hdds.scm.proxy.SingleSCMSecurityProtocolProxyProvider;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
@@ -447,6 +448,16 @@ public final class HddsServerUtil {
UserGroupInformation.getCurrentUser()));
}
+ /**
+ * Create a scm security client to interact with a specific SCM node.
+ */
+ public static SCMSecurityProtocolClientSideTranslatorPB
+ getScmSecurityClientSingleNode(ConfigurationSource conf, String scmNodeId,
+ UserGroupInformation ugi) throws IOException {
+ return new SCMSecurityProtocolClientSideTranslatorPB(
+ new SingleSCMSecurityProtocolProxyProvider(conf, ugi, scmNodeId));
+ }
+
public static SCMSecurityProtocolClientSideTranslatorPB
getScmSecurityClientWithMaxRetry(OzoneConfiguration conf,
UserGroupInformation ugi) throws IOException {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java
index 0a86de3b37..03f6ae293b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.hdds.scm.ha;
import org.apache.hadoop.hdds.scm.AddSCMRequest;
import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.ratis.server.protocol.TermIndex;
import java.io.IOException;
+import java.util.List;
/**
* SCMHAManager provides HA service for SCM.
@@ -87,6 +89,12 @@ public interface SCMHAManager extends AutoCloseable {
*/
DBCheckpoint downloadCheckpointFromLeader(String leaderId);
+ /**
+ * Get secret keys from SCM leader.
+ */
+ List<ManagedSecretKey> getSecretKeysFromLeader(String leaderID)
+ throws IOException;
+
/**
* Verify the SCM DB checkpoint downloaded from leader.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
index 6af0fb9919..c8efe7b261 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
@@ -20,12 +20,16 @@ package org.apache.hadoop.hdds.scm.ha;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.AddSCMRequest;
import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.metadata.SCMDBTransactionBufferImpl;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
+import org.apache.hadoop.hdds.scm.security.SecretKeyManagerService;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.utils.HAUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -36,6 +40,7 @@ import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.hdds.ExitManager;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.util.FileUtils;
import org.slf4j.Logger;
@@ -44,6 +49,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.List;
+
+import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientSingleNode;
/**
* SCMHAManagerImpl uses Apache Ratis for HA implementation. We will have 2N+1
@@ -60,6 +68,7 @@ public class SCMHAManagerImpl implements SCMHAManager {
private final SCMRatisServer ratisServer;
private final ConfigurationSource conf;
+ private final SecurityConfig securityConfig;
private final DBTransactionBuffer transactionBuffer;
private final SCMSnapshotProvider scmSnapshotProvider;
private final StorageContainerManager scm;
@@ -72,8 +81,10 @@ public class SCMHAManagerImpl implements SCMHAManager {
* Creates SCMHAManager instance.
*/
public SCMHAManagerImpl(final ConfigurationSource conf,
+ final SecurityConfig securityConfig,
final StorageContainerManager scm) throws IOException {
this.conf = conf;
+ this.securityConfig = securityConfig;
this.scm = scm;
this.exitManager = new ExitManager();
if (SCMHAUtils.isSCMHAEnabled(conf)) {
@@ -163,6 +174,21 @@ public class SCMHAManagerImpl implements SCMHAManager {
return dBCheckpoint;
}
+ @Override
+ public List<ManagedSecretKey> getSecretKeysFromLeader(String leaderID)
+ throws IOException {
+ if (!SecretKeyManagerService.isSecretKeyEnable(securityConfig)) {
+ return null;
+ }
+
+ LOG.info("Getting secret keys from leader {}.", leaderID);
+ try (SCMSecurityProtocolClientSideTranslatorPB securityProtocol =
+ getScmSecurityClientSingleNode(conf, leaderID,
+ UserGroupInformation.getLoginUser())) {
+ return securityProtocol.getAllSecretKeys();
+ }
+ }
+
@Override
public TermIndex verifyCheckpointFromLeader(String leaderId,
DBCheckpoint checkpoint) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerStub.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerStub.java
index 6382df3483..c15d805fb1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerStub.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerStub.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.hadoop.hdds.scm.AddSCMRequest;
import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.DBStore;
@@ -148,6 +149,11 @@ public final class SCMHAManagerStub implements SCMHAManager {
return null;
}
+ @Override
+ public List<ManagedSecretKey> getSecretKeysFromLeader(String leaderID) {
+ return null;
+ }
+
@Override
public TermIndex verifyCheckpointFromLeader(String leaderId,
DBCheckpoint checkpoint) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index c88331db98..9384661537 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.scm.ha;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.EnumMap;
+import java.util.List;
import java.util.Map;
import java.util.Collection;
import java.util.Optional;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
@@ -62,6 +64,8 @@ import org.apache.ratis.util.LifeCycle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.Objects.requireNonNull;
+
/**
* The SCMStateMachine is the state machine for SCMRatisServer. It is
* responsible for applying ratis committed transactions to
@@ -83,6 +87,7 @@ public class SCMStateMachine extends BaseStateMachine {
// ensures serializable between notifyInstallSnapshotFromLeader()
// and reinitialize().
private DBCheckpoint installingDBCheckpoint = null;
+ private List<ManagedSecretKey> installingSecretKeys = null;
private AtomicLong currentLeaderTerm = new AtomicLong(-1L);
private AtomicBoolean refreshedAfterLeaderReady = new AtomicBoolean(false);
@@ -243,12 +248,23 @@ public class SCMStateMachine extends BaseStateMachine {
return null;
}
+ List<ManagedSecretKey> secretKeys;
+ try {
+ secretKeys =
+ scm.getScmHAManager().getSecretKeysFromLeader(leaderNodeId);
+ LOG.info("Got secret keys from leaders {}", secretKeys);
+ } catch (IOException ex) {
+ LOG.error("Failed to get secret keys from SCM leader {}",
+ leaderNodeId, ex);
+ return null;
+ }
+
TermIndex termIndex =
scm.getScmHAManager().verifyCheckpointFromLeader(
leaderNodeId, checkpoint);
if (termIndex != null) {
- setInstallingDBCheckpoint(checkpoint);
+ setInstallingSnapshotData(checkpoint, secretKeys);
}
return termIndex;
},
@@ -381,9 +397,11 @@ public class SCMStateMachine extends BaseStateMachine {
public void reinitialize() throws IOException {
Preconditions.checkNotNull(installingDBCheckpoint);
DBCheckpoint checkpoint = installingDBCheckpoint;
+ List<ManagedSecretKey> secretKeys = installingSecretKeys;
// explicitly set installingDBCheckpoint to be null
installingDBCheckpoint = null;
+ installingSecretKeys = null;
TermIndex termIndex = null;
try {
@@ -402,6 +420,10 @@ public class SCMStateMachine extends BaseStateMachine {
LOG.error("Failed to unpause ", ioe);
}
+ if (secretKeys != null) {
+ requireNonNull(scm.getSecretKeyManager()).reinitialize(secretKeys);
+ }
+
getLifeCycle().transition(LifeCycle.State.STARTING);
getLifeCycle().transition(LifeCycle.State.RUNNING);
}
@@ -425,8 +447,10 @@ public class SCMStateMachine extends BaseStateMachine {
}
@VisibleForTesting
- public void setInstallingDBCheckpoint(DBCheckpoint checkpoint) {
+ public void setInstallingSnapshotData(DBCheckpoint checkpoint,
+ List<ManagedSecretKey> secretKeys) {
Preconditions.checkArgument(installingDBCheckpoint == null);
installingDBCheckpoint = checkpoint;
+ installingSecretKeys = secretKeys;
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 467a406f7e..a3f6f55493 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -632,7 +632,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
if (configurator.getSCMHAManager() != null) {
scmHAManager = configurator.getSCMHAManager();
} else {
- scmHAManager = new SCMHAManagerImpl(conf, this);
+ scmHAManager = new SCMHAManagerImpl(conf, securityConfig, this);
}
if (configurator.getLeaseManager() != null) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
index 11209217cc..8ca660491e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client
.CertificateClient;
import org.apache.hadoop.hdds.utils.TransactionInfo;
@@ -254,7 +255,8 @@ class TestSCMHAManagerImpl {
when(nodeDetails.getRatisHostPortStr()).thenReturn("localhost:" +
conf.get(ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY));
- final SCMHAManager manager = new SCMHAManagerImpl(conf, scm);
+ final SCMHAManager manager = new SCMHAManagerImpl(conf,
+ new SecurityConfig(conf), scm);
when(scm.getScmHAManager()).thenReturn(manager);
return scm;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
index cd8a7d2e68..407ae94664 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
@@ -151,7 +151,7 @@ public class TestSCMInstallSnapshot {
SCMStateMachine sm =
scm.getScmHAManager().getRatisServer().getSCMStateMachine();
sm.pause();
- sm.setInstallingDBCheckpoint(checkpoint);
+ sm.setInstallingSnapshotData(checkpoint, null);
sm.reinitialize();
Assert.assertNotNull(
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 26d31717e3..717964e122 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -171,6 +171,12 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
return this.scmhaService.getServiceByIndex(index);
}
+ public StorageContainerManager getScmLeader() {
+ return getStorageContainerManagers().stream()
+ .filter(StorageContainerManager::checkLeader)
+ .findFirst().orElse(null);
+ }
+
private OzoneManager getOMLeader(boolean waitForLeaderElection)
throws TimeoutException, InterruptedException {
if (waitForLeaderElection) {
@@ -701,9 +707,7 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
"127.0.0.1:" + blockPort);
- if (i <= numOfActiveSCMs) {
- scmPorts.release(scmNodeId);
- }
+ scmPorts.release(scmNodeId);
scmRpcPorts.release(scmNodeId);
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
index 217b08e728..c2bfbb0e15 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
@@ -85,7 +85,7 @@ public final class TestSecretKeysApi {
.getLogger(TestSecretKeysApi.class);
@Rule
- public Timeout timeout = Timeout.seconds(180);
+ public Timeout timeout = Timeout.seconds(500);
private MiniKdc miniKdc;
private OzoneConfiguration conf;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
index 3b8ad5faf3..74868bee2a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
@@ -279,8 +279,8 @@ public class TestSCMInstallSnapshotWithHA {
s == LifeCycle.State.NEW || s.isPausingOrPaused());
// Verify correct reloading
- followerSM.setInstallingDBCheckpoint(
- new RocksDBCheckpoint(checkpointBackup.toPath()));
+ followerSM.setInstallingSnapshotData(
+ new RocksDBCheckpoint(checkpointBackup.toPath()), null);
followerSM.reinitialize();
Assert.assertEquals(followerSM.getLastAppliedTermIndex(),
leaderCheckpointTrxnInfo.getTermIndex());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSecretKeySnapshot.java
similarity index 51%
copy from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
copy to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSecretKeySnapshot.java
index 217b08e728..410fc5bd65 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSecretKeySnapshot.java
@@ -15,35 +15,37 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone;
+package org.apache.hadoop.ozone.scm;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.ha.SCMStateMachine;
import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager;
import org.apache.hadoop.minikdc.MiniKdc;
-import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.util.ExitUtils;
-import org.jetbrains.annotations.NotNull;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
@@ -61,7 +63,6 @@ import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBER
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
-import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.SECRET_KEY_NOT_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
@@ -69,36 +70,36 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRI
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
- * Integration test to verify symmetric SecretKeys APIs in a secure cluster.
+ * Integration test to verify that symmetric secret keys are correctly
+ * synchronized from leader to follower during snapshot installation.
*/
-@InterfaceAudience.Private
-public final class TestSecretKeysApi {
+@Timeout(500)
+public final class TestSecretKeySnapshot {
private static final Logger LOG = LoggerFactory
- .getLogger(TestSecretKeysApi.class);
-
- @Rule
- public Timeout timeout = Timeout.seconds(180);
+ .getLogger(TestSecretKeySnapshot.class);
+ private static final long SNAPSHOT_THRESHOLD = 100;
+ private static final int LOG_PURGE_GAP = 100;
+ public static final int ROTATE_CHECK_DURATION_MS = 1_000;
+ public static final int ROTATE_DURATION_MS = 30_000;
+ public static final int EXPIRY_DURATION_MS = 61_000;
private MiniKdc miniKdc;
private OzoneConfiguration conf;
private File workDir;
private File ozoneKeytab;
private File spnegoKeytab;
- private File testUserKeytab;
- private String testUserPrincipal;
+ private String host;
private String clusterId;
private String scmId;
private MiniOzoneHAClusterImpl cluster;
- @Before
+ @BeforeEach
public void init() throws Exception {
conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
@@ -112,9 +113,32 @@ public final class TestSecretKeysApi {
startMiniKdc();
setSecureConfig();
createCredentialsInKDC();
+
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_PURGE_ENABLED, true);
+ conf.setInt(ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_PURGE_GAP, LOG_PURGE_GAP);
+ conf.setLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD,
+ SNAPSHOT_THRESHOLD);
+
+ conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION,
+ ROTATE_CHECK_DURATION_MS + "ms");
+ conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, ROTATE_DURATION_MS + "ms");
+ conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, EXPIRY_DURATION_MS + "ms");
+
+ MiniOzoneCluster.Builder builder = MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setSCMServiceId("TestSecretKeySnapshot")
+ .setScmId(scmId)
+ .setSCMServiceId("SCMServiceId")
+ .setNumDatanodes(1)
+ .setNumOfStorageContainerManagers(3)
+ .setNumOfActiveSCMs(2)
+ .setNumOfOzoneManagers(1);
+
+ cluster = (MiniOzoneHAClusterImpl) builder.build();
+ cluster.waitForClusterToBeReady();
}
- @After
+ @AfterEach
public void stop() {
miniKdc.stop();
if (cluster != null) {
@@ -129,7 +153,6 @@ public final class TestSecretKeysApi {
conf.getObject(SCMHTTPServerConfig.class);
createPrincipal(ozoneKeytab, scmConfig.getKerberosPrincipal());
createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal());
- createPrincipal(testUserKeytab, testUserPrincipal);
}
private void createPrincipal(File keytab, String... principal)
@@ -145,7 +168,7 @@ public final class TestSecretKeysApi {
private void setSecureConfig() throws IOException {
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- String host = InetAddress.getLocalHost().getCanonicalHostName()
+ host = InetAddress.getLocalHost().getCanonicalHostName()
.toLowerCase();
conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name());
@@ -163,8 +186,6 @@ public final class TestSecretKeysApi {
ozoneKeytab = new File(workDir, "scm.keytab");
spnegoKeytab = new File(workDir, "http.keytab");
- testUserKeytab = new File(workDir, "testuser.keytab");
- testUserPrincipal = "test@" + realm;
conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
ozoneKeytab.getAbsolutePath());
@@ -176,152 +197,93 @@ public final class TestSecretKeysApi {
spnegoKeytab.getAbsolutePath());
conf.set(DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY,
ozoneKeytab.getAbsolutePath());
- }
- /**
- * Test secret key apis in happy case.
- */
- @Test
- public void testSecretKeyApiSuccess() throws Exception {
- enableBlockToken();
- // set a low rotation period, of 1s, expiry is 3s, expect 3 active keys
- // at any moment.
- conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "100ms");
- conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1s");
- conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "3000ms");
-
- startCluster();
- SCMSecurityProtocol securityProtocol = getScmSecurityProtocol();
-
- // start the test when keys are full.
- GenericTestUtils.waitFor(() -> {
- try {
- return securityProtocol.getAllSecretKeys().size() >= 3;
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- }, 100, 4_000);
-
- ManagedSecretKey initialKey = securityProtocol.getCurrentSecretKey();
- assertNotNull(initialKey);
- List<ManagedSecretKey> initialKeys = securityProtocol.getAllSecretKeys();
- assertEquals(initialKey, initialKeys.get(0));
- ManagedSecretKey lastKey = initialKeys.get(initialKeys.size() - 1);
-
- LOG.info("Initial active key: {}", initialKey);
- LOG.info("Initial keys: {}", initialKeys);
-
- // wait for the next rotation.
- GenericTestUtils.waitFor(() -> {
- try {
- ManagedSecretKey newCurrentKey = securityProtocol.getCurrentSecretKey();
- return !newCurrentKey.equals(initialKey);
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- }, 100, 1500);
- ManagedSecretKey updatedKey = securityProtocol.getCurrentSecretKey();
- List<ManagedSecretKey> updatedKeys = securityProtocol.getAllSecretKeys();
-
- LOG.info("Updated active key: {}", updatedKey);
- LOG.info("Updated keys: {}", updatedKeys);
-
- assertEquals(updatedKey, updatedKeys.get(0));
- assertEquals(initialKey, updatedKeys.get(1));
- // ensure the last key from the previous cycle no longer managed.
- assertTrue(lastKey.isExpired());
- assertFalse(updatedKeys.contains(lastKey));
-
- // assert getSecretKey by ID.
- ManagedSecretKey keyById = securityProtocol.getSecretKey(
- updatedKey.getId());
- assertNotNull(keyById);
- ManagedSecretKey nonExisting = securityProtocol.getSecretKey(
- UUID.randomUUID());
- assertNull(nonExisting);
+ conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);
}
- /**
- * Verify API behavior when block token is not enable.
- */
@Test
- public void testSecretKeyApiNotEnabled() throws Exception {
- startCluster();
- SCMSecurityProtocol securityProtocol = getScmSecurityProtocol();
-
- SCMSecurityException ex = assertThrows(SCMSecurityException.class,
- securityProtocol::getCurrentSecretKey);
- assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode());
+ public void testInstallSnapshot() throws Exception {
+ // Get the leader SCM
+ StorageContainerManager leaderSCM = cluster.getScmLeader();
+ assertNotNull(leaderSCM);
+ // Find the inactive SCM
+ String followerId = cluster.getInactiveSCM().next().getSCMNodeId();
+
+ StorageContainerManager followerSCM = cluster.getSCM(followerId);
+
+ // wait until leader SCM got enough secret keys.
+ SecretKeyManager leaderSecretKeyManager = leaderSCM.getSecretKeyManager();
+ GenericTestUtils.waitFor(
+ () -> leaderSecretKeyManager.getSortedKeys().size() >= 2,
+ ROTATE_CHECK_DURATION_MS, EXPIRY_DURATION_MS);
+
+ writeToIncreaseLogIndex(leaderSCM, 200);
+ ManagedSecretKey currentKeyInLeader =
+ leaderSecretKeyManager.getCurrentSecretKey();
+
+ // Start the inactive SCM. Install Snapshot will happen as part
+ // of setConfiguration() call to ratis leader and the follower will catch
+ // up
+ LOG.info("Starting follower...");
+ cluster.startInactiveSCM(followerId);
+
+ // The recently started should be lagging behind the leader .
+ SCMStateMachine followerSM =
+ followerSCM.getScmHAManager().getRatisServer().getSCMStateMachine();
+
+ // Wait & retry for follower to update transactions to leader
+ // snapshot index.
+ // Timeout error if follower does not load update within 3s
+ GenericTestUtils.waitFor(() ->
+ followerSM.getLastAppliedTermIndex().getIndex() >= 200,
+ 100, 3000);
+ long followerLastAppliedIndex =
+ followerSM.getLastAppliedTermIndex().getIndex();
+ assertTrue(followerLastAppliedIndex >= 200);
+ assertFalse(followerSM.getLifeCycleState().isPausingOrPaused());
+
+ // Verify that the follower has the secret keys created
+ // while it was inactive.
+ SecretKeyManager followerSecretKeyManager =
+ followerSCM.getSecretKeyManager();
+ assertTrue(followerSecretKeyManager.isInitialized());
+ List<ManagedSecretKey> followerKeys =
+ followerSecretKeyManager.getSortedKeys();
+ LOG.info("Follower secret keys after snapshot: {}", followerKeys);
+ assertTrue(followerKeys.size() >= 2);
+ assertTrue(followerKeys.contains(currentKeyInLeader));
+ assertEquals(leaderSecretKeyManager.getSortedKeys(), followerKeys);
+
+ // Wait for the next rotation, assert that the updates can be synchronized
+ // normally post snapshot.
+ ManagedSecretKey currentKeyPostSnapshot =
+ leaderSecretKeyManager.getCurrentSecretKey();
+ GenericTestUtils.waitFor(() ->
+ !leaderSecretKeyManager.getCurrentSecretKey()
+ .equals(currentKeyPostSnapshot),
+ ROTATE_CHECK_DURATION_MS, ROTATE_DURATION_MS);
+ assertEquals(leaderSecretKeyManager.getSortedKeys(),
+ followerSecretKeyManager.getSortedKeys());
- ex = assertThrows(SCMSecurityException.class,
- () -> securityProtocol.getSecretKey(UUID.randomUUID()));
- assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode());
-
- ex = assertThrows(SCMSecurityException.class,
- securityProtocol::getAllSecretKeys);
- assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode());
}
- /**
- * Verify API behavior when SCM leader fails.
- */
- @Test
- public void testSecretKeyAfterSCMFailover() throws Exception {
- enableBlockToken();
- // set a long duration period, so that no rotation happens during SCM
- // leader change.
- conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "10m");
- conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1d");
- conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "7d");
-
- startCluster();
- SCMSecurityProtocol securityProtocol = getScmSecurityProtocol();
- List<ManagedSecretKey> keysInitial = securityProtocol.getAllSecretKeys();
- LOG.info("Keys before fail over: {}.", keysInitial);
-
- // turn the current SCM leader off.
- StorageContainerManager activeSCM = cluster.getActiveSCM();
- cluster.shutdownStorageContainerManager(activeSCM);
- // wait for
- cluster.waitForSCMToBeReady();
-
- List<ManagedSecretKey> keysAfter = securityProtocol.getAllSecretKeys();
- LOG.info("Keys after fail over: {}.", keysAfter);
-
- assertEquals(keysInitial.size(), keysAfter.size());
- for (int i = 0; i < keysInitial.size(); i++) {
- assertEquals(keysInitial.get(i), keysAfter.get(i));
+ private List<ContainerInfo> writeToIncreaseLogIndex(
+ StorageContainerManager scm, long targetLogIndex)
+ throws IOException, InterruptedException, TimeoutException {
+ List<ContainerInfo> containers = new ArrayList<>();
+ SCMStateMachine stateMachine =
+ scm.getScmHAManager().getRatisServer().getSCMStateMachine();
+ long logIndex = scm.getScmHAManager().getRatisServer().getSCMStateMachine()
+ .getLastAppliedTermIndex().getIndex();
+ while (logIndex <= targetLogIndex) {
+ containers.add(scm.getContainerManager()
+ .allocateContainer(
+ RatisReplicationConfig.getInstance(ReplicationFactor.ONE),
+ this.getClass().getName()));
+ Thread.sleep(100);
+ logIndex = stateMachine.getLastAppliedTermIndex().getIndex();
}
+ return containers;
}
- private void startCluster()
- throws IOException, TimeoutException, InterruptedException {
- OzoneManager.setTestSecureOmFlag(true);
- MiniOzoneCluster.Builder builder = MiniOzoneCluster.newHABuilder(conf)
- .setClusterId(clusterId)
- .setSCMServiceId("TestSecretKey")
- .setScmId(scmId)
- .setNumDatanodes(3)
- .setNumOfStorageContainerManagers(3)
- .setNumOfOzoneManagers(1);
-
- cluster = (MiniOzoneHAClusterImpl) builder.build();
- cluster.waitForClusterToBeReady();
- }
-
- @NotNull
- private SCMSecurityProtocol getScmSecurityProtocol() throws IOException {
- UserGroupInformation ugi =
- UserGroupInformation.loginUserFromKeytabAndReturnUGI(
- testUserPrincipal, testUserKeytab.getCanonicalPath());
- ugi.setAuthenticationMethod(KERBEROS);
- SCMSecurityProtocol scmSecurityProtocolClient =
- HddsServerUtil.getScmSecurityClient(conf, ugi);
- assertNotNull(scmSecurityProtocolClient);
- return scmSecurityProtocolClient;
- }
-
- private void enableBlockToken() {
- conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 03/05: HDDS-7831. Use symmetric secret key to sign and verify token (#4417)
Posted by du...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
duong pushed a commit to branch HDDS-7733-Symmetric-Tokens
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit f966961491648a7fb47fd1389fc8fce4353f013c
Author: Duong Nguyen <du...@gmail.com>
AuthorDate: Wed Apr 5 18:24:44 2023 -0700
HDDS-7831. Use symmetric secret key to sign and verify token (#4417)
---
.../security/token/ContainerTokenIdentifier.java | 17 +-
.../security/token/OzoneBlockTokenIdentifier.java | 26 +-
.../security/token/ShortLivedTokenIdentifier.java | 24 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 4 +
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 1 +
.../token/TestOzoneBlockTokenIdentifier.java | 306 ---------------------
.../apache/hadoop/ozone/HddsDatanodeService.java | 19 +-
.../common/statemachine/DatanodeStateMachine.java | 12 +-
.../ECReconstructionCoordinator.java | 10 +-
.../container/ec/reconstruction/TokenHelper.java | 44 +--
.../ozone/container/ozoneimpl/OzoneContainer.java | 17 +-
.../ozone/container/common/ContainerTestUtils.java | 2 +-
.../container/common/TestDatanodeStateMachine.java | 11 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 3 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 10 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 8 +-
.../security/symmetric/DefaultSecretKeyClient.java | 74 +++++
.../symmetric/DefaultSecretKeySignerClient.java | 130 +++++++++
.../symmetric/DefaultSecretKeyVerifierClient.java | 99 +++++++
.../hdds/security/symmetric/ManagedSecretKey.java | 42 ++-
.../hdds/security/symmetric/SecretKeyClient.java | 26 ++
.../hdds/security/symmetric/SecretKeyConfig.java | 26 +-
.../hdds/security/symmetric/SecretKeyManager.java | 8 +-
.../security/symmetric/SecretKeySignerClient.java | 43 +++
.../security/symmetric/SecretKeyStateImpl.java | 3 +-
.../symmetric/SecretKeyVerifierClient.java | 30 ++
.../hdds/security/token/BlockTokenVerifier.java | 7 +-
.../token/ContainerTokenSecretManager.java | 14 +-
.../security/token/ContainerTokenVerifier.java | 6 +-
.../token/OzoneBlockTokenSecretManager.java | 66 +----
.../token/ShortLivedTokenSecretManager.java | 75 ++---
.../security/token/ShortLivedTokenVerifier.java | 66 ++---
.../hadoop/hdds/security/token/TokenVerifier.java | 9 +-
.../security/symmetric/ManagedSecretKeyTest.java | 75 +++++
.../security/symmetric/SecretKeyManagerTest.java | 13 +-
.../hdds/security/symmetric/SecretKeyTestUtil.java | 52 ++++
.../security/token/TestBlockTokenVerifier.java | 18 +-
.../security/token/TestContainerTokenVerifier.java | 12 +-
.../token/TestOzoneBlockTokenIdentifier.java | 103 +++++++
.../token/TestOzoneBlockTokenSecretManager.java | 245 ++++-------------
.../hdds/security/token/TokenVerifierTests.java | 168 ++++++-----
.../org.mockito.plugins.MockMaker | 16 ++
.../src/main/proto/ScmAdminProtocol.proto | 3 +-
.../interface-client/src/main/proto/hdds.proto | 3 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 55 +---
.../ozone/container/common/TestEndPoint.java | 12 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 28 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 7 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 11 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 8 +-
.../org/apache/hadoop/ozone/TestSecretKeysApi.java | 1 -
.../hadoop/ozone/TestSecureOzoneCluster.java | 252 -----------------
.../hadoop/ozone/client/SecretKeyTestClient.java | 73 +++++
.../client/rpc/TestContainerStateMachine.java | 2 +
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +
.../client/rpc/TestOzoneAtRestEncryption.java | 2 +
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 8 +-
.../container/ozoneimpl/TestOzoneContainer.java | 5 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 20 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 18 +-
.../server/TestSecureContainerServer.java | 17 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 64 ++---
.../ozone/security/TestOzoneManagerBlockToken.java | 251 -----------------
65 files changed, 1261 insertions(+), 1531 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenIdentifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenIdentifier.java
index ebb0709d4b..7cd8c279ba 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenIdentifier.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenIdentifier.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerTokenSecretProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.ProtobufUtils;
import java.io.DataInput;
import java.io.DataInputStream;
@@ -28,6 +29,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.time.Instant;
import java.util.Objects;
+import java.util.UUID;
/**
* Token identifier for container operations, similar to block token.
@@ -43,11 +45,18 @@ public class ContainerTokenIdentifier extends ShortLivedTokenIdentifier {
}
public ContainerTokenIdentifier(String ownerId, ContainerID containerID,
- String certSerialId, Instant expiryDate) {
- super(ownerId, expiryDate, certSerialId);
+ Instant expiryDate) {
+ super(ownerId, expiryDate);
this.containerID = containerID;
}
+ public ContainerTokenIdentifier(String ownerId, ContainerID containerID,
+ UUID secretKeyId,
+ Instant expiryDate) {
+ this(ownerId, containerID, expiryDate);
+ setSecretKeyId(secretKeyId);
+ }
+
@Override
public Text getKind() {
return KIND;
@@ -58,7 +67,7 @@ public class ContainerTokenIdentifier extends ShortLivedTokenIdentifier {
ContainerTokenSecretProto.Builder builder = ContainerTokenSecretProto
.newBuilder()
.setOwnerId(getOwnerId())
- .setCertSerialId(getCertSerialId())
+ .setSecretKeyId(ProtobufUtils.toProtobuf(getSecretKeyId()))
.setExpiryDate(getExpiry().toEpochMilli())
.setContainerId(containerID.getProtobuf());
out.write(builder.build().toByteArray());
@@ -72,7 +81,7 @@ public class ContainerTokenIdentifier extends ShortLivedTokenIdentifier {
}
ContainerTokenSecretProto proto =
ContainerTokenSecretProto.parseFrom((DataInputStream) in);
- setCertSerialId(proto.getCertSerialId());
+ setSecretKeyId(ProtobufUtils.fromProtobuf(proto.getSecretKeyId()));
setExpiry(Instant.ofEpochMilli(proto.getExpiryDate()));
setOwnerId(proto.getOwnerId());
this.containerID = ContainerID.getFromProtobuf(proto.getContainerId());
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
index dcd75d6334..be18f90a07 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token.TrivialRenewer;
+import org.apache.hadoop.util.ProtobufUtils;
import java.io.DataInput;
import java.io.DataInputStream;
@@ -59,16 +60,14 @@ public class OzoneBlockTokenIdentifier extends ShortLivedTokenIdentifier {
}
public OzoneBlockTokenIdentifier(String ownerId, BlockID blockId,
- Set<AccessModeProto> modes, long expiryDate, String omCertSerialId,
- long maxLength) {
- this(ownerId, getTokenService(blockId), modes, expiryDate, omCertSerialId,
+ Set<AccessModeProto> modes, long expiryDate, long maxLength) {
+ this(ownerId, getTokenService(blockId), modes, expiryDate,
maxLength);
}
public OzoneBlockTokenIdentifier(String ownerId, String blockId,
- Set<AccessModeProto> modes, long expiryDate, String omCertSerialId,
- long maxLength) {
- super(ownerId, Instant.ofEpochMilli(expiryDate), omCertSerialId);
+ Set<AccessModeProto> modes, long expiryDate, long maxLength) {
+ super(ownerId, Instant.ofEpochMilli(expiryDate));
this.blockId = blockId;
this.modes = modes == null
? EnumSet.noneOf(AccessModeProto.class) : EnumSet.copyOf(modes);
@@ -136,7 +135,7 @@ public class OzoneBlockTokenIdentifier extends ShortLivedTokenIdentifier {
BlockTokenSecretProto.parseFrom((DataInputStream) in);
setOwnerId(token.getOwnerId());
setExpiry(Instant.ofEpochMilli(token.getExpiryDate()));
- setCertSerialId(token.getOmCertSerialId());
+ setSecretKeyId(ProtobufUtils.fromProtobuf(token.getSecretKeyId()));
this.blockId = token.getBlockId();
this.modes = EnumSet.copyOf(token.getModesList());
this.maxLength = token.getMaxLength();
@@ -147,10 +146,13 @@ public class OzoneBlockTokenIdentifier extends ShortLivedTokenIdentifier {
throws IOException {
BlockTokenSecretProto token =
BlockTokenSecretProto.parseFrom((DataInputStream) in);
- return new OzoneBlockTokenIdentifier(token.getOwnerId(),
- token.getBlockId(), EnumSet.copyOf(token.getModesList()),
- token.getExpiryDate(), token.getOmCertSerialId(),
- token.getMaxLength());
+ OzoneBlockTokenIdentifier tokenId =
+ new OzoneBlockTokenIdentifier(token.getOwnerId(),
+ token.getBlockId(), EnumSet.copyOf(token.getModesList()),
+ token.getExpiryDate(),
+ token.getMaxLength());
+ tokenId.setSecretKeyId(ProtobufUtils.fromProtobuf(token.getSecretKeyId()));
+ return tokenId;
}
@Override
@@ -158,7 +160,7 @@ public class OzoneBlockTokenIdentifier extends ShortLivedTokenIdentifier {
BlockTokenSecretProto.Builder builder = BlockTokenSecretProto.newBuilder()
.setBlockId(blockId)
.setOwnerId(getOwnerId())
- .setOmCertSerialId(getCertSerialId())
+ .setSecretKeyId(ProtobufUtils.toProtobuf(getSecretKeyId()))
.setExpiryDate(getExpiryDate())
.setMaxLength(maxLength);
// Add access mode allowed
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenIdentifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenIdentifier.java
index 7475fa5042..dbd168f96a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenIdentifier.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenIdentifier.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import java.time.Instant;
import java.util.Objects;
+import java.util.UUID;
/**
* Base class for short-lived tokens (block, container).
@@ -33,18 +34,16 @@ public abstract class ShortLivedTokenIdentifier extends TokenIdentifier {
private String ownerId;
private Instant expiry;
- private String certSerialId;
+ private UUID secretKeyId;
public abstract String getService();
protected ShortLivedTokenIdentifier() {
}
- protected ShortLivedTokenIdentifier(String ownerId, Instant expiry,
- String certSerialId) {
+ protected ShortLivedTokenIdentifier(String ownerId, Instant expiry) {
this.ownerId = ownerId;
this.expiry = expiry;
- this.certSerialId = certSerialId;
}
@Override
@@ -67,22 +66,23 @@ public abstract class ShortLivedTokenIdentifier extends TokenIdentifier {
this.expiry = expiry;
}
- protected void setCertSerialId(String certSerialId) {
- this.certSerialId = certSerialId;
+ public void setSecretKeyId(UUID secretKeyId) {
+ this.secretKeyId = secretKeyId;
}
public Instant getExpiry() {
return expiry;
}
- public String getCertSerialId() {
- return certSerialId;
- }
public String getOwnerId() {
return ownerId;
}
+ public UUID getSecretKeyId() {
+ return secretKeyId;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -95,18 +95,18 @@ public abstract class ShortLivedTokenIdentifier extends TokenIdentifier {
ShortLivedTokenIdentifier that = (ShortLivedTokenIdentifier) o;
return Objects.equals(ownerId, that.ownerId) &&
Objects.equals(expiry, that.expiry) &&
- Objects.equals(certSerialId, that.certSerialId);
+ Objects.equals(secretKeyId, that.secretKeyId);
}
@Override
public int hashCode() {
- return Objects.hash(ownerId, expiry, certSerialId);
+ return Objects.hash(ownerId, expiry, secretKeyId);
}
@Override
public String toString() {
return "ownerId=" + ownerId +
", expiry=" + expiry +
- ", certSerialId=" + certSerialId;
+ ", secretKeyId=" + secretKeyId;
}
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 6a8a91e7b2..12a4de1c68 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -487,4 +487,8 @@ public class SecurityConfig {
OzoneConfigKeys.OZONE_S3_AUTHINFO_MAX_LIFETIME_KEY_DEFAULT,
TimeUnit.MICROSECONDS);
}
+
+ public boolean isTokenEnabled() {
+ return blockTokenEnabled || containerTokenEnabled;
+ }
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
index 21e6ffbb41..db258e03e5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.commons.validator.routines.InetAddressValidator;
+
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_SECURITY_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
deleted file mode 100644
index c834e8c89c..0000000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.security.token;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.Signature;
-import java.security.SignatureException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import javax.crypto.KeyGenerator;
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.security.token.Token;
-import org.apache.ozone.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Assumptions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test class for {@link OzoneBlockTokenIdentifier}.
- */
-class TestOzoneBlockTokenIdentifier {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TestOzoneBlockTokenIdentifier.class);
- private static final String BASEDIR = GenericTestUtils
- .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
- private static final String KEYSTORES_DIR =
- new File(BASEDIR).getAbsolutePath();
- private static long expiryTime;
- private static KeyPair keyPair;
- private static X509Certificate cert;
-
- @BeforeAll
- static void setUp() throws Exception {
- File base = new File(BASEDIR);
- FileUtil.fullyDelete(base);
- Assumptions.assumeTrue(base.mkdirs());
- expiryTime = Time.monotonicNow() + 60 * 60 * 24;
-
- // Create Ozone Master key pair.
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- // Create Ozone Master certificate (SCM CA issued cert) and key store.
- cert = KeyStoreTestUtil
- .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
- }
-
- @AfterEach
- void cleanUp() {
- // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
- }
-
- @Test
- void testSignToken() throws GeneralSecurityException, IOException {
- String keystore = new File(KEYSTORES_DIR, "keystore.jks")
- .getAbsolutePath();
- String truststore = new File(KEYSTORES_DIR, "truststore.jks")
- .getAbsolutePath();
- String trustPassword = "trustPass";
- String keyStorePassword = "keyStorePass";
- String keyPassword = "keyPass";
-
-
- KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
- "OzoneMaster", keyPair.getPrivate(), cert);
-
- // Create trust store and put the certificate in the trust store
- Map<String, X509Certificate> certs = Collections.singletonMap("server",
- cert);
- KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
-
- // Sign the OzoneMaster Token with Ozone Master private key
- PrivateKey privateKey = keyPair.getPrivate();
- OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
- "testUser", "84940",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString(), 128L);
- byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
-
- // Verify a valid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
- LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
-
- // Verify an invalid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- tokenId = new OzoneBlockTokenIdentifier("", "",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString(), 128L);
- LOG.info("Unsigned token {} is {}", tokenId,
- verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
-
- }
-
- @Test
- void testTokenSerialization() throws GeneralSecurityException,
- IOException {
- String keystore = new File(KEYSTORES_DIR, "keystore.jks")
- .getAbsolutePath();
- String truststore = new File(KEYSTORES_DIR, "truststore.jks")
- .getAbsolutePath();
- String trustPassword = "trustPass";
- String keyStorePassword = "keyStorePass";
- String keyPassword = "keyPass";
- long maxLength = 128L;
-
- KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
- "OzoneMaster", keyPair.getPrivate(), cert);
-
- // Create trust store and put the certificate in the trust store
- Map<String, X509Certificate> certs = Collections.singletonMap("server",
- cert);
- KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
-
- // Sign the OzoneMaster Token with Ozone Master private key
- PrivateKey privateKey = keyPair.getPrivate();
- OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
- "testUser", "84940",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString(), maxLength);
- byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
-
-
- Token<OzoneBlockTokenIdentifier> token = new Token<>(tokenId.getBytes(),
- signedToken, tokenId.getKind(), new Text("host:port"));
-
- String encodeToUrlString = token.encodeToUrlString();
-
- Token<OzoneBlockTokenIdentifier>decodedToken = new Token<>();
- decodedToken.decodeFromUrlString(encodeToUrlString);
-
- OzoneBlockTokenIdentifier decodedTokenId = new OzoneBlockTokenIdentifier();
- decodedTokenId.readFields(new DataInputStream(
- new ByteArrayInputStream(decodedToken.getIdentifier())));
-
- Assertions.assertEquals(tokenId, decodedTokenId);
- Assertions.assertEquals(maxLength, decodedTokenId.getMaxLength());
-
- // Verify a decoded signed Token with public key(certificate)
- boolean isValidToken = verifyTokenAsymmetric(decodedTokenId, decodedToken
- .getPassword(), cert);
- LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
- }
-
-
- private byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- PrivateKey privateKey) throws NoSuchAlgorithmException,
- InvalidKeyException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initSign(privateKey);
- rsaSignature.update(tokenId.getBytes());
- return rsaSignature.sign();
- }
-
- private boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- byte[] signature, Certificate certificate) throws InvalidKeyException,
- NoSuchAlgorithmException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initVerify(certificate);
- rsaSignature.update(tokenId.getBytes());
- return rsaSignature.verify(signature);
- }
-
- private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
- Mac mac, SecretKey key) {
- try {
- mac.init(key);
- } catch (InvalidKeyException ike) {
- throw new IllegalArgumentException("Invalid key to HMAC computation",
- ike);
- }
- return mac.doFinal(identifier.getBytes());
- }
-
- private OzoneBlockTokenIdentifier generateTestToken() {
- return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
- RandomStringUtils.randomAlphabetic(5),
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString(), 1024768L);
- }
-
- @Test
- void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
- CertificateEncodingException, NoSuchProviderException,
- InvalidKeyException, SignatureException {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordAsym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
-
- // Create Ozone Master certificate (SCM CA issued cert) and key store
- X509Certificate certificate;
- certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
- kp, 30, "SHA256withRSA");
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordAsym.add(
- signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
- duration / testTokenCount);
-
- startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
- certificate);
- }
- duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
- + "is {} ns", duration / testTokenCount);
- }
-
- @Test
- void testSymmetricTokenPerf() {
- String hmacSHA1 = "HmacSHA1";
- String hmacSHA256 = "HmacSHA256";
-
- testSymmetricTokenPerfHelper(hmacSHA1, 64);
- testSymmetricTokenPerfHelper(hmacSHA256, 1024);
- }
-
- private void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordSym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyGenerator keyGen;
- try {
- keyGen = KeyGenerator.getInstance(hmacAlgorithm);
- keyGen.init(keyLen);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- Mac mac;
- try {
- mac = Mac.getInstance(hmacAlgorithm);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- SecretKey secretKey = keyGen.generateKey();
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordSym.add(
- signTokenSymmetric(tokenIds.get(i), mac, secretKey));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
- hmacAlgorithm, keyLen, duration / tokenPasswordSym.size());
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index e06c2565a7..16a2e219ae 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStoreImpl;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.security.symmetric.DefaultSecretKeyClient;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient;
@@ -99,6 +101,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
private DatanodeStateMachine datanodeStateMachine;
private List<ServicePlugin> plugins;
private CertificateClient dnCertClient;
+ private SecretKeyClient secretKeyClient;
private String component;
private HddsDatanodeHttpServer httpServer;
private boolean printBanner;
@@ -293,9 +296,14 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
dnCertClient = initializeCertificateClient(dnCertClient);
+
+ if (secConf.isTokenEnabled()) {
+ secretKeyClient = DefaultSecretKeyClient.create(conf);
+ secretKeyClient.start(conf);
+ }
}
datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf,
- dnCertClient, this::terminateDatanode, dnCRLStore);
+ dnCertClient, secretKeyClient, this::terminateDatanode, dnCRLStore);
try {
httpServer = new HddsDatanodeHttpServer(conf);
httpServer.start();
@@ -561,6 +569,10 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
LOG.error("Datanode CRL store stop failed", ex);
}
RatisDropwizardExports.clear(ratisMetricsMap, ratisReporterList);
+
+ if (secretKeyClient != null) {
+ secretKeyClient.stop();
+ }
}
}
@@ -603,6 +615,11 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
dnCertClient = client;
}
+ @VisibleForTesting
+ public void setSecretKeyClient(SecretKeyClient client) {
+ this.secretKeyClient = client;
+ }
+
@Override
public void printError(Throwable error) {
LOG.error("Exception in HddsDatanodeService.", error);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 73d064f04b..956290bbcf 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.IOUtils;
@@ -139,6 +140,7 @@ public class DatanodeStateMachine implements Closeable {
public DatanodeStateMachine(DatanodeDetails datanodeDetails,
ConfigurationSource conf,
CertificateClient certClient,
+ SecretKeyClient secretKeyClient,
HddsDatanodeStopService hddsDatanodeStopService,
DatanodeCRLStore crlStore) throws IOException {
DatanodeConfiguration dnConf =
@@ -171,7 +173,7 @@ public class DatanodeStateMachine implements Closeable {
constructionLock.writeLock().lock();
try {
container = new OzoneContainer(this.datanodeDetails,
- conf, context, certClient);
+ conf, context, certClient, secretKeyClient);
} finally {
constructionLock.writeLock().unlock();
}
@@ -209,7 +211,7 @@ public class DatanodeStateMachine implements Closeable {
ecReconstructionMetrics = ECReconstructionMetrics.create();
ecReconstructionCoordinator = new ECReconstructionCoordinator(
- conf, certClient, context, ecReconstructionMetrics);
+ conf, certClient, secretKeyClient, context, ecReconstructionMetrics);
// This is created as an instance variable as Mockito needs to access it in
// a test. The test mocks it in a running mini-cluster.
@@ -252,6 +254,12 @@ public class DatanodeStateMachine implements Closeable {
queueMetrics = DatanodeQueueMetrics.create(this);
}
+ @VisibleForTesting
+ public DatanodeStateMachine(DatanodeDetails datanodeDetails,
+ ConfigurationSource conf) throws IOException {
+ this(datanodeDetails, conf, null, null, null, null);
+ }
+
private int getEndPointTaskThreadPoolSize() {
// TODO(runzhiwang): current only support one recon, if support multiple
// recon in future reconServerCount should be the real number of recon
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
index a7e5bfb92d..7d258819eb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.storage.BlockLocationInfo;
import org.apache.hadoop.hdds.scm.storage.BufferPool;
import org.apache.hadoop.hdds.scm.storage.ECBlockOutputStream;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.utils.IOUtils;
@@ -110,9 +111,9 @@ public class ECReconstructionCoordinator implements Closeable {
private final ECReconstructionMetrics metrics;
private final StateContext context;
- public ECReconstructionCoordinator(ConfigurationSource conf,
- CertificateClient certificateClient,
- StateContext context,
+ public ECReconstructionCoordinator(
+ ConfigurationSource conf, CertificateClient certificateClient,
+ SecretKeySignerClient secretKeyClient, StateContext context,
ECReconstructionMetrics metrics) throws IOException {
this.context = context;
this.containerOperationClient = new ECContainerOperationClient(conf,
@@ -128,7 +129,7 @@ public class ECReconstructionCoordinator implements Closeable {
new ThreadPoolExecutor.CallerRunsPolicy());
this.blockInputStreamFactory = BlockInputStreamFactoryImpl
.getInstance(byteBufferPool, () -> ecReconstructExecutor);
- tokenHelper = new TokenHelper(conf, certificateClient);
+ tokenHelper = new TokenHelper(conf, secretKeyClient);
this.clientMetrics = ContainerClientMetrics.acquire();
this.metrics = metrics;
}
@@ -390,7 +391,6 @@ public class ECReconstructionCoordinator implements Closeable {
if (containerOperationClient != null) {
containerOperationClient.close();
}
- tokenHelper.stop();
}
private Pipeline rebuildInputPipeline(ECReplicationConfig repConfig,
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java
index ffd7632804..682b9dc147 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java
@@ -22,17 +22,16 @@ import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import java.io.IOException;
-import java.time.Duration;
import java.util.EnumSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -52,7 +51,7 @@ class TokenHelper {
private static final Set<AccessModeProto> MODES =
EnumSet.of(READ, WRITE, DELETE);
- TokenHelper(ConfigurationSource conf, CertificateClient certClient)
+ TokenHelper(ConfigurationSource conf, SecretKeySignerClient secretKeyClient)
throws IOException {
SecurityConfig securityConfig = new SecurityConfig(conf);
@@ -61,7 +60,7 @@ class TokenHelper {
// checking certClient != null instead of securityConfig.isSecurityEnabled()
// to allow integration test without full kerberos etc. setup
- boolean securityEnabled = certClient != null;
+ boolean securityEnabled = secretKeyClient != null;
if (securityEnabled && (blockTokenEnabled || containerTokenEnabled)) {
user = UserGroupInformation.getCurrentUser().getShortUserName();
@@ -70,29 +69,17 @@ class TokenHelper {
HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME,
HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT,
TimeUnit.MILLISECONDS);
- long certificateGracePeriod = Duration.parse(
- conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION,
- HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT))
- .toMillis();
- if (expiryTime > certificateGracePeriod) {
- throw new IllegalArgumentException("Certificate grace period " +
- HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION +
- " should be greater than maximum block/container token lifetime " +
- HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME);
- }
if (blockTokenEnabled) {
- blockTokenMgr = new OzoneBlockTokenSecretManager(
- securityConfig, expiryTime);
- blockTokenMgr.start(certClient);
+ blockTokenMgr = new OzoneBlockTokenSecretManager(expiryTime,
+ secretKeyClient);
} else {
blockTokenMgr = null;
}
if (containerTokenEnabled) {
- containerTokenMgr = new ContainerTokenSecretManager(
- securityConfig, expiryTime);
- containerTokenMgr.start(certClient);
+ containerTokenMgr = new ContainerTokenSecretManager(expiryTime,
+ secretKeyClient);
} else {
containerTokenMgr = null;
}
@@ -103,23 +90,6 @@ class TokenHelper {
}
}
- void stop() {
- if (blockTokenMgr != null) {
- try {
- blockTokenMgr.stop();
- } catch (IOException ignored) {
- // no threads involved, cannot really happen
- }
- }
- if (containerTokenMgr != null) {
- try {
- containerTokenMgr.stop();
- } catch (IOException ignored) {
- // no threads involved, cannot really happen
- }
- }
- }
-
Token<OzoneBlockTokenIdentifier> getBlockToken(BlockID blockID, long length) {
return blockTokenMgr != null
? blockTokenMgr.generateToken(user, blockID, MODES, length)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index d276b617d5..779676932c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
@@ -135,7 +136,8 @@ public class OzoneContainer {
*/
public OzoneContainer(
DatanodeDetails datanodeDetails, ConfigurationSource conf,
- StateContext context, CertificateClient certClient) throws IOException {
+ StateContext context, CertificateClient certClient,
+ SecretKeyVerifierClient secretKeyClient) throws IOException {
config = conf;
this.datanodeDetails = datanodeDetails;
this.context = context;
@@ -189,7 +191,8 @@ public class OzoneContainer {
SecurityConfig secConf = new SecurityConfig(conf);
hddsDispatcher = new HddsDispatcher(config, containerSet, volumeSet,
- handlers, context, metrics, TokenVerifier.create(secConf, certClient));
+ handlers, context, metrics,
+ TokenVerifier.create(secConf, secretKeyClient));
/*
* ContainerController is the control plane
@@ -260,6 +263,16 @@ public class OzoneContainer {
new AtomicReference<>(InitializingStatus.UNINITIALIZED);
}
+ /**
+ * Shorthand constructor used for testing in non-secure context.
+ */
+ @VisibleForTesting
+ public OzoneContainer(
+ DatanodeDetails datanodeDetails, ConfigurationSource conf,
+ StateContext context) throws IOException {
+ this(datanodeDetails, conf, context, null, null);
+ }
+
public GrpcTlsConfig getTlsClientConfig() {
return tlsClientConfig;
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
index 31d5000acb..9109be4627 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
@@ -107,7 +107,7 @@ public final class ContainerTestUtils {
StateContext context = Mockito.mock(StateContext.class);
Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
Mockito.when(context.getParent()).thenReturn(stateMachine);
- return new OzoneContainer(datanodeDetails, conf, context, null);
+ return new OzoneContainer(datanodeDetails, conf, context);
}
public static DatanodeDetails createDatanodeDetails() {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 1fc56b2963..97903fb13b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -157,8 +157,7 @@ public class TestDatanodeStateMachine {
public void testStartStopDatanodeStateMachine() throws IOException,
InterruptedException, TimeoutException {
try (DatanodeStateMachine stateMachine =
- new DatanodeStateMachine(getNewDatanodeDetails(), conf, null, null,
- null)) {
+ new DatanodeStateMachine(getNewDatanodeDetails(), conf)) {
stateMachine.startDaemon();
SCMConnectionManager connectionManager =
stateMachine.getConnectionManager();
@@ -220,8 +219,7 @@ public class TestDatanodeStateMachine {
datanodeDetails.setPort(port);
ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath, conf);
try (DatanodeStateMachine stateMachine =
- new DatanodeStateMachine(datanodeDetails, conf, null, null,
- null)) {
+ new DatanodeStateMachine(datanodeDetails, conf)) {
DatanodeStateMachine.DatanodeStates currentState =
stateMachine.getContext().getState();
Assertions.assertEquals(DatanodeStateMachine.DatanodeStates.INIT,
@@ -343,8 +341,7 @@ public class TestDatanodeStateMachine {
datanodeDetails.setPort(port);
try (DatanodeStateMachine stateMachine =
- new DatanodeStateMachine(datanodeDetails, conf, null, null,
- null)) {
+ new DatanodeStateMachine(datanodeDetails, conf)) {
DatanodeStateMachine.DatanodeStates currentState =
stateMachine.getContext().getState();
Assertions.assertEquals(DatanodeStateMachine.DatanodeStates.INIT,
@@ -402,7 +399,7 @@ public class TestDatanodeStateMachine {
perTestConf.setStrings(entry.getKey(), entry.getValue());
LOG.info("Test with {} = {}", entry.getKey(), entry.getValue());
try (DatanodeStateMachine stateMachine = new DatanodeStateMachine(
- getNewDatanodeDetails(), perTestConf, null, null, null)) {
+ getNewDatanodeDetails(), perTestConf)) {
DatanodeStateMachine.DatanodeStates currentState =
stateMachine.getContext().getState();
Assertions.assertEquals(DatanodeStateMachine.DatanodeStates.INIT,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index cceed79390..f06eea2ec1 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -178,7 +178,7 @@ public class TestOzoneContainer {
// loaded into the containerSet.
// Also expected to initialize committed space for each volume.
OzoneContainer ozoneContainer = new
- OzoneContainer(datanodeDetails, conf, context, null);
+ OzoneContainer(datanodeDetails, conf, context);
ContainerSet containerset = ozoneContainer.getContainerSet();
assertEquals(numTestContainers, containerset.containerCount());
@@ -213,7 +213,7 @@ public class TestOzoneContainer {
// loaded into the containerSet.
// Also expected to initialize committed space for each volume.
OzoneContainer ozoneContainer = new
- OzoneContainer(datanodeDetails, conf, context, null);
+ OzoneContainer(datanodeDetails, conf, context);
Assert.assertEquals(volumeSet.getVolumesList().size(),
ozoneContainer.getNodeReport().getStorageReportList().size());
Assert.assertEquals(3,
@@ -234,7 +234,7 @@ public class TestOzoneContainer {
// loaded into the containerSet.
// Also expected to initialize committed space for each volume.
OzoneContainer ozoneContainer = new
- OzoneContainer(datanodeDetails, conf, context, null);
+ OzoneContainer(datanodeDetails, conf, context);
Assert.assertEquals(volumeSet.getVolumesList().size(),
ozoneContainer.getNodeReport().getStorageReportList().size());
Assert.assertEquals(1,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDataNodeStartupSlvLessThanMlv.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDataNodeStartupSlvLessThanMlv.java
index 306cdb77b3..82a9db0377 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDataNodeStartupSlvLessThanMlv.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDataNodeStartupSlvLessThanMlv.java
@@ -65,8 +65,7 @@ public class TestDataNodeStartupSlvLessThanMlv {
HddsProtos.NodeType.DATANODE, mlv);
try {
- new DatanodeStateMachine(getNewDatanodeDetails(), conf, null,
- null, null);
+ new DatanodeStateMachine(getNewDatanodeDetails(), conf);
Assert.fail("Expected IOException due to incorrect MLV on DataNode " +
"creation.");
} catch (IOException e) {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java
index 1670c1ebc0..7669a577cf 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java
@@ -222,7 +222,7 @@ public class TestDatanodeUpgradeToSchemaV3 {
HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT.layoutVersion());
layoutStorage.initialize();
dsm = new DatanodeStateMachine(
- ContainerTestUtils.createDatanodeDetails(), conf, null, null, null);
+ ContainerTestUtils.createDatanodeDetails(), conf);
HddsVolume dataVolume = (
HddsVolume) dsm.getContainer().getVolumeSet().getVolumesList().get(0);
// Format HddsVolume to mimic the real cluster upgrade situation
@@ -493,7 +493,7 @@ public class TestDatanodeUpgradeToSchemaV3 {
HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT.layoutVersion());
layoutStorage.initialize();
dsm = new DatanodeStateMachine(
- ContainerTestUtils.createDatanodeDetails(), conf, null, null, null);
+ ContainerTestUtils.createDatanodeDetails(), conf);
HddsVolume dataVolume = (
HddsVolume) dsm.getContainer().getVolumeSet().getVolumesList().get(0);
// Format HddsVolume to mimic the real cluster upgrade situation
@@ -592,8 +592,7 @@ public class TestDatanodeUpgradeToSchemaV3 {
// Build and start the datanode.
DatanodeDetails dd = ContainerTestUtils.createDatanodeDetails();
- DatanodeStateMachine newDsm = new DatanodeStateMachine(dd,
- conf, null, null, null);
+ DatanodeStateMachine newDsm = new DatanodeStateMachine(dd, conf);
int actualMlv = newDsm.getLayoutVersionManager().getMetadataLayoutVersion();
Assert.assertEquals(
HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT.layoutVersion(),
@@ -613,8 +612,7 @@ public class TestDatanodeUpgradeToSchemaV3 {
dsm.close();
// Start new datanode with the same configuration.
- dsm = new DatanodeStateMachine(dd,
- conf, null, null, null);
+ dsm = new DatanodeStateMachine(dd, conf);
int mlv = dsm.getLayoutVersionManager().getMetadataLayoutVersion();
if (exactMatch) {
Assert.assertEquals(expectedMlv, mlv);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
index 3ff347fbdf..79f55001c7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
@@ -516,9 +516,7 @@ public class TestDatanodeUpgradeToScmHA {
// Build and start the datanode.
DatanodeDetails dd = ContainerTestUtils.createDatanodeDetails();
- DatanodeStateMachine newDsm = new DatanodeStateMachine(dd,
- conf, null, null,
- null);
+ DatanodeStateMachine newDsm = new DatanodeStateMachine(dd, conf);
int actualMlv = newDsm.getLayoutVersionManager().getMetadataLayoutVersion();
Assert.assertEquals(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(),
actualMlv);
@@ -534,9 +532,7 @@ public class TestDatanodeUpgradeToScmHA {
dsm.close();
// Start new datanode with the same configuration.
- dsm = new DatanodeStateMachine(dd,
- conf, null, null,
- null);
+ dsm = new DatanodeStateMachine(dd, conf);
int mlv = dsm.getLayoutVersionManager().getMetadataLayoutVersion();
if (exactMatch) {
Assert.assertEquals(expectedMlv, mlv);
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyClient.java
new file mode 100644
index 0000000000..d77fee778e
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyClient.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient;
+
+/**
+ * A composition of {@link DefaultSecretKeySignerClient} and
+ * {@link DefaultSecretKeyVerifierClient} for components need both APIs.
+ */
+public class DefaultSecretKeyClient implements SecretKeyClient {
+ private final SecretKeySignerClient signerClientDelegate;
+ private final SecretKeyVerifierClient verifierClientDelegate;
+
+
+ DefaultSecretKeyClient(SecretKeySignerClient signerClientDelegate,
+ SecretKeyVerifierClient verifierClientDelegate) {
+ this.signerClientDelegate = signerClientDelegate;
+ this.verifierClientDelegate = verifierClientDelegate;
+ }
+
+
+ @Override
+ public ManagedSecretKey getCurrentSecretKey() {
+ return signerClientDelegate.getCurrentSecretKey();
+ }
+
+ @Override
+ public void start(ConfigurationSource conf) throws IOException {
+ signerClientDelegate.start(conf);
+ }
+
+ @Override
+ public void stop() {
+ signerClientDelegate.stop();
+ }
+
+ @Override
+ public ManagedSecretKey getSecretKey(UUID id) throws SCMSecurityException {
+ return verifierClientDelegate.getSecretKey(id);
+ }
+
+ public static SecretKeyClient create(ConfigurationSource conf)
+ throws IOException {
+ SCMSecurityProtocol securityProtocol = getScmSecurityClient(conf);
+ SecretKeySignerClient singerClient =
+ new DefaultSecretKeySignerClient(securityProtocol);
+ SecretKeyVerifierClient verifierClient =
+ new DefaultSecretKeyVerifierClient(securityProtocol, conf);
+ return new DefaultSecretKeyClient(singerClient, verifierClient);
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java
new file mode 100644
index 0000000000..a1056f9139
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient;
+
+/**
+ * Default implementation of {@link SecretKeySignerClient} that fetches
+ * secret keys from SCM. This client implements a background thread that
+ * periodically check and get the latest current secret key from SCM.
+ */
+public class DefaultSecretKeySignerClient implements SecretKeySignerClient {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultSecretKeySignerClient.class);
+
+ private final SCMSecurityProtocol scmSecurityProtocol;
+ private final AtomicReference<ManagedSecretKey> cache =
+ new AtomicReference<>();
+ private ScheduledExecutorService executorService;
+
+ public DefaultSecretKeySignerClient(
+ SCMSecurityProtocol scmSecurityProtocol) {
+ this.scmSecurityProtocol = scmSecurityProtocol;
+ }
+
+ @Override
+ public ManagedSecretKey getCurrentSecretKey() {
+ return requireNonNull(cache.get(),
+ "SecretKey client must have been initialized already.");
+ }
+
+ @Override
+ public void start(ConfigurationSource conf) throws IOException {
+ final ManagedSecretKey initialKey =
+ scmSecurityProtocol.getCurrentSecretKey();
+ LOG.info("Initial secret key fetched from SCM: {}.", initialKey);
+ cache.set(initialKey);
+ scheduleSecretKeyPoller(conf, initialKey.getCreationTime());
+ }
+
+ @Override
+ public void stop() {
+ executorService.shutdown();
+ try {
+ if (executorService.awaitTermination(1, TimeUnit.MINUTES)) {
+ executorService.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while shutting down executor service.", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void scheduleSecretKeyPoller(ConfigurationSource conf,
+ Instant initialCreation) {
+ Duration rotateDuration = SecretKeyConfig.parseRotateDuration(conf);
+ Instant nextRotate = initialCreation.plus(rotateDuration);
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("SecretKeyPoller")
+ .setDaemon(true)
+ .build();
+ executorService = Executors.newScheduledThreadPool(1, threadFactory);
+ Duration interval = SecretKeyConfig.parseRotateCheckDuration(conf);
+ Duration initialDelay = Duration.between(Instant.now(), nextRotate);
+
+ LOG.info("Scheduling SecretKeyPoller with initial delay of {} " +
+ "and interval of {}", initialDelay, interval);
+ executorService.scheduleAtFixedRate(() -> checkAndRefresh(rotateDuration),
+ initialDelay.toMillis(), interval.toMillis(),
+ TimeUnit.MILLISECONDS);
+ }
+
+ private void checkAndRefresh(Duration rotateDuration) {
+ ManagedSecretKey current = cache.get();
+ Instant nextRotate = current.getCreationTime().plus(rotateDuration);
+ // when the current key passes the rotation cycle, fetch the next one
+ // from SCM.
+ if (nextRotate.isBefore(Instant.now())) {
+ try {
+ ManagedSecretKey newKey = scmSecurityProtocol.getCurrentSecretKey();
+ if (!newKey.equals(current)) {
+ cache.set(newKey);
+ LOG.info("New secret key fetched from SCM: {}.", newKey);
+ }
+ } catch (IOException e) {
+ // TODO: emic failure metrics.
+ throw new UncheckedIOException(
+ "Error fetching current key from SCM", e);
+ }
+ }
+ }
+
+ public static DefaultSecretKeySignerClient create(ConfigurationSource conf)
+ throws IOException {
+ SCMSecurityProtocol securityProtocol = getScmSecurityClient(conf);
+ return new DefaultSecretKeySignerClient(securityProtocol);
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java
new file mode 100644
index 0000000000..56478793cb
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.security.symmetric.SecretKeyConfig.parseExpiryDuration;
+import static org.apache.hadoop.hdds.security.symmetric.SecretKeyConfig.parseRotateDuration;
+import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient;
+
+/**
+ * Default implementation of {@link SecretKeyVerifierClient} that fetches
+ * SecretKeys remotely via {@link SCMSecurityProtocol}.
+ */
+public class DefaultSecretKeyVerifierClient implements SecretKeyVerifierClient {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultSecretKeyVerifierClient.class);
+
+ private final LoadingCache<UUID, ManagedSecretKey> cache;
+
+ DefaultSecretKeyVerifierClient(SCMSecurityProtocol scmSecurityProtocol,
+ ConfigurationSource conf) {
+ Duration expiryDuration = parseExpiryDuration(conf);
+ Duration rotateDuration = parseRotateDuration(conf);
+ long cacheSize = expiryDuration.toMillis() / rotateDuration.toMillis() + 1;
+
+ CacheLoader<UUID, ManagedSecretKey> loader =
+ new CacheLoader<UUID, ManagedSecretKey>() {
+ @Override
+ public ManagedSecretKey load(UUID id) throws Exception {
+ ManagedSecretKey secretKey = scmSecurityProtocol.getSecretKey(id);
+ LOG.info("Secret key fetched from SCM: {}", secretKey);
+ return secretKey;
+ }
+ };
+
+ LOG.info("Initializing secret key cache with size {}, TTL {}",
+ cacheSize, expiryDuration);
+ cache = CacheBuilder.newBuilder()
+ .maximumSize(cacheSize)
+ .expireAfterWrite(expiryDuration.toMillis(), TimeUnit.MILLISECONDS)
+ .recordStats()
+ .build(loader);
+ }
+
+ @Override
+ public ManagedSecretKey getSecretKey(UUID id) throws SCMSecurityException {
+ try {
+ return cache.get(id);
+ } catch (ExecutionException e) {
+ // handle cache load exception.
+ if (e.getCause() instanceof IOException) {
+ IOException cause = (IOException) e.getCause();
+ if (cause instanceof SCMSecurityException) {
+ throw (SCMSecurityException) cause;
+ } else {
+ throw new SCMSecurityException(
+ "Error fetching secret key " + id + " from SCM", cause);
+ }
+ }
+ throw new IllegalStateException("Unexpected exception fetching secret " +
+ "key " + id + " from SCM", e.getCause());
+ }
+ }
+
+ public static DefaultSecretKeyVerifierClient create(ConfigurationSource conf)
+ throws IOException {
+ SCMSecurityProtocol securityProtocol = getScmSecurityClient(conf);
+ return new DefaultSecretKeyVerifierClient(securityProtocol, conf);
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
index 3128265e9a..78e4fc0b90 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
@@ -20,11 +20,15 @@ package org.apache.hadoop.hdds.security.symmetric;
import com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ProtobufUtils;
+import javax.crypto.Mac;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
-import java.io.Serializable;
+import java.security.InvalidKeyException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.UUID;
@@ -32,11 +36,12 @@ import java.util.UUID;
* Enclosed a symmetric {@link SecretKey} with additional data for life-cycle
* management.
*/
-public final class ManagedSecretKey implements Serializable {
+public final class ManagedSecretKey {
private final UUID id;
private final Instant creationTime;
private final Instant expiryTime;
private final SecretKey secretKey;
+ private final ThreadLocal<Mac> macInstances;
public ManagedSecretKey(UUID id,
Instant creationTime,
@@ -46,6 +51,16 @@ public final class ManagedSecretKey implements Serializable {
this.creationTime = creationTime;
this.expiryTime = expiryTime;
this.secretKey = secretKey;
+
+ // This help reuse Mac instances for the same thread.
+ macInstances = ThreadLocal.withInitial(() -> {
+ try {
+ return Mac.getInstance(secretKey.getAlgorithm());
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalArgumentException(
+ "Invalid algorithm " + secretKey.getAlgorithm(), e);
+ }
+ });
}
public boolean isExpired() {
@@ -88,6 +103,29 @@ public final class ManagedSecretKey implements Serializable {
+ creationTime + ", expire at: " + expiryTime + ")";
}
+ public byte[] sign(byte[] data) {
+ try {
+ Mac mac = macInstances.get();
+ mac.init(secretKey);
+ return mac.doFinal(data);
+ } catch (InvalidKeyException e) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation", e);
+ }
+ }
+
+ public byte[] sign(TokenIdentifier tokenId) {
+ return sign(tokenId.getBytes());
+ }
+
+ public boolean isValidSignature(byte[] data, byte[] signature) {
+ byte[] expectedSignature = sign(data);
+ return MessageDigest.isEqual(expectedSignature, signature);
+ }
+
+ public boolean isValidSignature(TokenIdentifier tokenId, byte[] signature) {
+ return isValidSignature(tokenId.getBytes(), signature);
+ }
+
/**
* @return the protobuf message to deserialize this object.
*/
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyClient.java
new file mode 100644
index 0000000000..a71b14dc3f
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyClient.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
+
+/**
+ * Composite client for those components that need to perform both signing
+ * and verifying.
+ */
+public interface SecretKeyClient extends SecretKeySignerClient,
+ SecretKeyVerifierClient {
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java
index f2a9181051..a833ba0137 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java
@@ -58,23 +58,33 @@ public class SecretKeyConfig {
HDDS_SECRET_KEY_FILE_DEFAULT);
localSecretKeyFile = Paths.get(metadataDir, component, keyDir, fileName);
- long rotateDurationInMs = conf.getTimeDuration(
- HDDS_SECRET_KEY_ROTATE_DURATION,
- HDDS_SECRET_KEY_ROTATE_DURATION_DEFAULT, TimeUnit.MILLISECONDS);
- this.rotateDuration = Duration.ofMillis(rotateDurationInMs);
+ this.rotateDuration = parseRotateDuration(conf);
+ this.expiryDuration = parseExpiryDuration(conf);
+ this.rotationCheckDuration = parseRotateCheckDuration(conf);
+
+ this.algorithm = conf.get(HDDS_SECRET_KEY_ALGORITHM,
+ HDDS_SECRET_KEY_ALGORITHM_DEFAULT);
+ }
+ public static Duration parseExpiryDuration(ConfigurationSource conf) {
long expiryDurationInMs = conf.getTimeDuration(
HDDS_SECRET_KEY_EXPIRY_DURATION,
HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT, TimeUnit.MILLISECONDS);
- this.expiryDuration = Duration.ofMillis(expiryDurationInMs);
+ return Duration.ofMillis(expiryDurationInMs);
+ }
- this.algorithm = conf.get(HDDS_SECRET_KEY_ALGORITHM,
- HDDS_SECRET_KEY_ALGORITHM_DEFAULT);
+ public static Duration parseRotateDuration(ConfigurationSource conf) {
+ long rotateDurationInMs = conf.getTimeDuration(
+ HDDS_SECRET_KEY_ROTATE_DURATION,
+ HDDS_SECRET_KEY_ROTATE_DURATION_DEFAULT, TimeUnit.MILLISECONDS);
+ return Duration.ofMillis(rotateDurationInMs);
+ }
+ public static Duration parseRotateCheckDuration(ConfigurationSource conf) {
long rotationCheckInMs = conf.getTimeDuration(
HDDS_SECRET_KEY_ROTATE_CHECK_DURATION,
HDDS_SECRET_KEY_ROTATE_CHECK_DURATION_DEFAULT, TimeUnit.MILLISECONDS);
- this.rotationCheckDuration = Duration.ofMillis(rotationCheckInMs);
+ return Duration.ofMillis(rotationCheckInMs);
}
public Path getLocalSecretKeyFile() {
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
index cb529e10d1..f7a481cc05 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
@@ -37,7 +37,7 @@ import static java.util.stream.Collectors.toList;
* This component manages symmetric SecretKey life-cycle, including generation,
* rotation and destruction.
*/
-public class SecretKeyManager {
+public class SecretKeyManager implements SecretKeyClient {
private static final Logger LOG =
LoggerFactory.getLogger(SecretKeyManager.class);
@@ -129,11 +129,13 @@ public class SecretKeyManager {
return false;
}
- public ManagedSecretKey getCurrentKey() {
+ @Override
+ public ManagedSecretKey getCurrentSecretKey() {
return state.getCurrentKey();
}
- public ManagedSecretKey getKey(UUID id) {
+ @Override
+ public ManagedSecretKey getSecretKey(UUID id) {
return state.getKey(id);
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeySignerClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeySignerClient.java
new file mode 100644
index 0000000000..d05786db5c
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeySignerClient.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+
+import java.io.IOException;
+
+/**
+ * Define the client-side API that the token signers (like OM) uses to retrieve
+ * the secret key to sign data.
+ */
+public interface SecretKeySignerClient {
+ ManagedSecretKey getCurrentSecretKey();
+
+ /**
+ * This is where the actual implementation can prefetch the current
+ * secret key or initialize ay necessary resources, e.g. cache or executors.
+ */
+ default void start(ConfigurationSource conf) throws IOException {
+ }
+
+ /**
+ * Give a chance for the implementation to clean up acquired resources.
+ */
+ default void stop() {
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
index b1d66e1186..727b005d2b 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
@@ -127,5 +127,4 @@ public final class SecretKeyStateImpl implements SecretKeyState {
lock.writeLock().unlock();
}
}
-
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java
new file mode 100644
index 0000000000..59f49f72f1
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
+import java.util.UUID;
+
+/**
+ * Define the client-side API that the token verifiers (or datanodes) use to
+ * retrieve the relevant secret key to validate token authority.
+ */
+public interface SecretKeyVerifierClient {
+ ManagedSecretKey getSecretKey(UUID id) throws SCMSecurityException;
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
index 996ed7ae68..c9999d253b 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
@@ -26,8 +26,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProtoOrBuilder;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,8 +54,9 @@ public class BlockTokenVerifier extends
return String.valueOf(blockID);
}
- public BlockTokenVerifier(SecurityConfig conf, CertificateClient caClient) {
- super(conf, caClient);
+ public BlockTokenVerifier(SecurityConfig conf,
+ SecretKeyVerifierClient secretKeyClient) {
+ super(conf, secretKeyClient);
}
@Override
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java
index e9f37f3de1..4cee87696f 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java
@@ -20,11 +20,9 @@ package org.apache.hadoop.hdds.security.token;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -38,17 +36,15 @@ public class ContainerTokenSecretManager
extends ShortLivedTokenSecretManager<ContainerTokenIdentifier>
implements ContainerTokenGenerator {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerTokenSecretManager.class);
-
- public ContainerTokenSecretManager(SecurityConfig conf, long tokenLifetime) {
- super(conf, tokenLifetime, LOG);
+ public ContainerTokenSecretManager(long tokenLifetime,
+ SecretKeySignerClient secretKeyClient) {
+ super(tokenLifetime, secretKeyClient);
}
public ContainerTokenIdentifier createIdentifier(String user,
ContainerID containerID) {
return new ContainerTokenIdentifier(user, containerID,
- getCertSerialId(), getTokenExpiryTime());
+ getTokenExpiryTime());
}
@Override
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenVerifier.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenVerifier.java
index 941160a042..7e4d186c32 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenVerifier.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenVerifier.java
@@ -21,16 +21,16 @@ import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProtoOrBuilder;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
/** Verifier for container tokens. */
public class ContainerTokenVerifier extends
ShortLivedTokenVerifier<ContainerTokenIdentifier> {
public ContainerTokenVerifier(SecurityConfig conf,
- CertificateClient caClient) {
- super(conf, caClient);
+ SecretKeyVerifierClient secretKeyClient) {
+ super(conf, secretKeyClient);
}
@Override
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java
index 3dc7a395a1..1192377e19 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java
@@ -21,11 +21,10 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,24 +39,19 @@ import java.util.Set;
@InterfaceStability.Unstable
public class OzoneBlockTokenSecretManager extends
ShortLivedTokenSecretManager<OzoneBlockTokenIdentifier> {
-
private static final Logger LOG = LoggerFactory
.getLogger(OzoneBlockTokenSecretManager.class);
- public OzoneBlockTokenSecretManager(SecurityConfig conf, long tokenLifetime) {
- super(conf, tokenLifetime, LOG);
- }
-
- @Override
- public OzoneBlockTokenIdentifier createIdentifier() {
- throw new SecurityException("Ozone block token can't be created "
- + "without owner and access mode information.");
+ public OzoneBlockTokenSecretManager(long tokenLifetime,
+ SecretKeySignerClient passwordManager) {
+ super(tokenLifetime, passwordManager);
}
public OzoneBlockTokenIdentifier createIdentifier(String owner,
BlockID blockID, Set<AccessModeProto> modes, long maxLength) {
return new OzoneBlockTokenIdentifier(owner, blockID, modes,
- getTokenExpiryTime().toEpochMilli(), getCertSerialId(), maxLength);
+ getTokenExpiryTime().toEpochMilli(),
+ maxLength);
}
/**
@@ -73,8 +67,9 @@ public class OzoneBlockTokenSecretManager extends
LOG.info("Issued delegation token -> expiryTime:{}, tokenId:{}",
Instant.ofEpochMilli(expiryTime), tokenIdentifier);
}
+ byte[] password = createPassword(tokenIdentifier);
return new Token<>(tokenIdentifier.getBytes(),
- createPassword(tokenIdentifier), tokenIdentifier.getKind(),
+ password, tokenIdentifier.getKind(),
new Text(tokenIdentifier.getService()));
}
@@ -88,49 +83,4 @@ public class OzoneBlockTokenSecretManager extends
return generateToken(userID, blockId, modes, maxLength);
}
- @Override
- public byte[] retrievePassword(OzoneBlockTokenIdentifier identifier)
- throws InvalidToken {
- validateToken(identifier);
- return createPassword(identifier);
- }
-
- @Override
- public long renewToken(Token<OzoneBlockTokenIdentifier> token,
- String renewer) {
- throw new UnsupportedOperationException("Renew token operation is not " +
- "supported for ozone block tokens.");
- }
-
- @Override
- public OzoneBlockTokenIdentifier cancelToken(Token<OzoneBlockTokenIdentifier>
- token, String canceller) {
- throw new UnsupportedOperationException("Cancel token operation is not " +
- "supported for ozone block tokens.");
- }
-
- /**
- * Find the OzoneBlockTokenInfo for the given token id, and verify that if the
- * token is not expired.
- */
- @Override
- public boolean validateToken(OzoneBlockTokenIdentifier identifier)
- throws InvalidToken {
- long now = Time.now();
- if (identifier.getExpiryDate() < now) {
- throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
- "expired, current time: " + Time.formatTime(now) +
- " expiry time: " + identifier.getExpiryDate());
- }
- return true;
- }
-
- /**
- * Validates if given hash is valid.
- */
- public boolean verifySignature(OzoneBlockTokenIdentifier identifier,
- byte[] password) {
- throw new UnsupportedOperationException("This operation is not " +
- "supported for block tokens.");
- }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java
index 966cf1cf5f..981ea77a41 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java
@@ -19,11 +19,10 @@ package org.apache.hadoop.hdds.security.token;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.hdds.security.OzoneSecretManager;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token;
-import org.slf4j.Logger;
import java.time.Instant;
@@ -33,54 +32,21 @@ import java.time.Instant;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public abstract class
- ShortLivedTokenSecretManager<T extends ShortLivedTokenIdentifier>
- extends OzoneSecretManager<T> {
+public abstract class ShortLivedTokenSecretManager
+ <T extends ShortLivedTokenIdentifier> {
+ private final long tokenMaxLifetime;
+ private SecretKeySignerClient secretKeyClient;
- private static final Text SERVICE = new Text("HDDS_SERVICE");
-
- protected ShortLivedTokenSecretManager(SecurityConfig conf,
- long tokenLifetime, Logger logger) {
- super(conf, tokenLifetime, tokenLifetime, SERVICE, logger);
- }
-
- @Override
- public T createIdentifier() {
- throw new SecurityException("Short-lived token requires additional " +
- "information (owner, etc.).");
- }
-
- @Override
- public long renewToken(Token<T> token, String renewer) {
- throw new UnsupportedOperationException("Renew token operation is not " +
- "supported for short-lived tokens.");
- }
-
- @Override
- public T cancelToken(Token<T> token, String canceller) {
- throw new UnsupportedOperationException("Cancel token operation is not " +
- "supported for short-lived tokens.");
- }
-
- @Override
- public byte[] retrievePassword(T identifier) throws InvalidToken {
- validateToken(identifier);
- return createPassword(identifier);
+ protected ShortLivedTokenSecretManager(
+ long tokenLifetime, SecretKeySignerClient secretKeyClient) {
+ this.tokenMaxLifetime = tokenLifetime;
+ this.secretKeyClient = secretKeyClient;
}
- /**
- * Find the OzoneBlockTokenInfo for the given token id, and verify that if the
- * token is not expired.
- */
- protected boolean validateToken(T identifier) throws InvalidToken {
- Instant now = Instant.now();
- if (identifier.isExpired(now)) {
- throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
- "expired, current time: " + now +
- " expiry time: " + identifier.getExpiry());
- }
-
- return true;
+ protected byte[] createPassword(T tokenId) {
+ ManagedSecretKey secretKey = secretKeyClient.getCurrentSecretKey();
+ tokenId.setSecretKeyId(secretKey.getId());
+ return secretKey.sign(tokenId);
}
/**
@@ -89,12 +55,21 @@ public abstract class
* @return Expiry time.
*/
protected Instant getTokenExpiryTime() {
- return Instant.now().plusMillis(getTokenMaxLifetime());
+ return Instant.now().plusMillis(tokenMaxLifetime);
}
public Token<T> generateToken(T tokenIdentifier) {
+ byte[] password = createPassword(tokenIdentifier);
return new Token<>(tokenIdentifier.getBytes(),
- createPassword(tokenIdentifier), tokenIdentifier.getKind(),
+ password, tokenIdentifier.getKind(),
new Text(tokenIdentifier.getService()));
}
+
+ /**
+ * Allows integration-test to inject a custom implementation of
+ * SecretKeyClient to test without fully setting up a working secure cluster.
+ */
+ public void setSecretKeyClient(SecretKeySignerClient client) {
+ this.secretKeyClient = client;
+ }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java
index 92b643e285..4731f9149c 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java
@@ -20,17 +20,15 @@ package org.apache.hadoop.hdds.security.token;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProtoOrBuilder;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
-import java.security.cert.CertificateExpiredException;
-import java.security.cert.CertificateNotYetValidException;
-import java.security.cert.X509Certificate;
import java.time.Instant;
import java.util.Objects;
@@ -42,13 +40,13 @@ public abstract class
ShortLivedTokenVerifier<T extends ShortLivedTokenIdentifier>
implements TokenVerifier {
- private final CertificateClient caClient;
private final SecurityConfig conf;
+ private final SecretKeyVerifierClient secretKeyClient;
protected ShortLivedTokenVerifier(SecurityConfig conf,
- CertificateClient caClient) {
+ SecretKeyVerifierClient secretKeyClient) {
this.conf = conf;
- this.caClient = caClient;
+ this.secretKeyClient = secretKeyClient;
}
/** Whether the specific kind of token is required for {@code cmdType}. */
@@ -75,11 +73,6 @@ public abstract class
return;
}
- if (caClient == null) {
- throw new SCMSecurityException("Certificate client not available " +
- "to validate token");
- }
-
T tokenId = createTokenIdentifier();
try {
tokenId.readFields(new DataInputStream(new ByteArrayInputStream(
@@ -88,31 +81,9 @@ public abstract class
throw new BlockTokenException("Failed to decode token : " + token);
}
- UserGroupInformation tokenUser = tokenId.getUser();
- X509Certificate signerCert =
- caClient.getCertificate(tokenId.getCertSerialId());
-
- if (signerCert == null) {
- throw new BlockTokenException("Can't find signer certificate " +
- "(CertSerialId: " + tokenId.getCertSerialId() +
- ") of the token for user: " + tokenUser);
- }
-
- try {
- signerCert.checkValidity();
- } catch (CertificateExpiredException exExp) {
- throw new BlockTokenException("Token can't be verified due to " +
- "expired certificate " + tokenId.getCertSerialId());
- } catch (CertificateNotYetValidException exNyv) {
- throw new BlockTokenException("Token can't be verified due to " +
- "not yet valid certificate " + tokenId.getCertSerialId());
- }
-
- if (!caClient.verifySignature(tokenId.getBytes(), token.getPassword(),
- signerCert)) {
- throw new BlockTokenException("Invalid token for user: " + tokenUser);
- }
+ verifyTokenPassword(tokenId, token.getPassword());
+ UserGroupInformation tokenUser = tokenId.getUser();
// check expiration
if (tokenId.isExpired(Instant.now())) {
throw new BlockTokenException("Expired token for user: " + tokenUser);
@@ -132,4 +103,27 @@ public abstract class
protected SecurityConfig getConf() {
return conf;
}
+
+ private void verifyTokenPassword(
+ ShortLivedTokenIdentifier tokenId, byte[] password)
+ throws SCMSecurityException {
+
+ ManagedSecretKey secretKey = secretKeyClient.getSecretKey(
+ tokenId.getSecretKeyId());
+ if (secretKey == null) {
+ throw new BlockTokenException("Can't find the signer secret key " +
+ tokenId.getSecretKeyId() + " of the token for user: " +
+ tokenId.getUser());
+ }
+
+ if (secretKey.isExpired()) {
+ throw new BlockTokenException("Token can't be verified due to " +
+ "expired secret key " + tokenId.getSecretKeyId());
+ }
+
+ if (!secretKey.isValidSignature(tokenId, password)) {
+ throw new BlockTokenException("Invalid token for user: " +
+ tokenId.getUser());
+ }
+ }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
index dbf79d5482..3301b68fcc 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProtoOrBuilder;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.security.token.Token;
import java.io.IOException;
@@ -73,15 +73,14 @@ public interface TokenVerifier {
/** Create appropriate token verifier based on the configuration. */
static TokenVerifier create(SecurityConfig conf,
- CertificateClient certClient) {
-
+ SecretKeyVerifierClient secretKeyClient) throws IOException {
if (!conf.isBlockTokenEnabled() && !conf.isContainerTokenEnabled()) {
return new NoopTokenVerifier();
}
List<TokenVerifier> list = new LinkedList<>();
- list.add(new BlockTokenVerifier(conf, certClient));
- list.add(new ContainerTokenVerifier(conf, certClient));
+ list.add(new BlockTokenVerifier(conf, secretKeyClient));
+ list.add(new ContainerTokenVerifier(conf, secretKeyClient));
return new CompositeTokenVerifier(list);
}
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKeyTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKeyTest.java
new file mode 100644
index 0000000000..6db83186ac
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKeyTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.junit.jupiter.api.Test;
+
+import static com.google.common.collect.ImmutableSet.of;
+import static java.time.Duration.ofDays;
+import static java.time.Instant.now;
+import static org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey.fromProtobuf;
+import static org.apache.hadoop.hdds.security.symmetric.SecretKeyTestUtil.generateHmac;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Simple test cases for {@link ManagedSecretKey}.
+ */
+public class ManagedSecretKeyTest {
+
+ @Test
+ public void testSignAndVerifySuccess() throws Exception {
+ // Data can be signed and verified by same key.
+ byte[] data = RandomUtils.nextBytes(100);
+ ManagedSecretKey secretKey = generateHmac(now(), ofDays(1));
+ byte[] signature = secretKey.sign(data);
+ assertTrue(secretKey.isValidSignature(data, signature));
+
+ // Data can be signed and verified by same key transferred via network.
+ ManagedSecretKey transferredKey = fromProtobuf(secretKey.toProtobuf());
+ assertTrue(transferredKey.isValidSignature(data, signature));
+
+ // Token can be sign and verified by the same key.
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier("owner",
+ new BlockID(1L, 1L), of(AccessModeProto.READ), 0L, 1L);
+ tokenId.setSecretKeyId(secretKey.getId());
+
+ signature = secretKey.sign(tokenId);
+ assertTrue(secretKey.isValidSignature(tokenId, signature));
+
+ // Token can be signed and verified by same key transferred via network.
+ assertTrue(transferredKey.isValidSignature(tokenId, signature));
+ }
+
+ @Test
+ public void testVerifyFailure() throws Exception {
+ byte[] data = RandomUtils.nextBytes(100);
+ ManagedSecretKey secretKey = generateHmac(now(), ofDays(1));
+ // random signature is not valid.
+ assertFalse(secretKey.isValidSignature(data, RandomUtils.nextBytes(100)));
+
+ // Data sign by one key can't be verified by another key.
+ byte[] signature = secretKey.sign(data);
+ ManagedSecretKey secretKey1 = generateHmac(now(), ofDays(1));
+ assertFalse(secretKey1.isValidSignature(data, signature));
+ }
+}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java
index 053148e28d..e7fd24082c 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java
@@ -26,13 +26,10 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
-import javax.crypto.KeyGenerator;
-import javax.crypto.SecretKey;
import java.time.Duration;
import java.time.Instant;
import java.util.Collection;
import java.util.List;
-import java.util.UUID;
import java.util.concurrent.TimeoutException;
import java.util.stream.Stream;
@@ -203,13 +200,7 @@ public class SecretKeyManagerTest {
private static ManagedSecretKey generateKey(Instant creationTime)
throws Exception {
- KeyGenerator keyGen = KeyGenerator.getInstance(ALGORITHM);
- SecretKey secretKey = keyGen.generateKey();
- return new ManagedSecretKey(
- UUID.randomUUID(),
- creationTime,
- creationTime.plus(VALIDITY_DURATION),
- secretKey
- );
+ return SecretKeyTestUtil.generateKey(ALGORITHM, creationTime,
+ VALIDITY_DURATION);
}
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyTestUtil.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyTestUtil.java
new file mode 100644
index 0000000000..272bedad54
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyTestUtil.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.UUID;
+
+/**
+ * Contains utility to test secret key logic.
+ */
+public final class SecretKeyTestUtil {
+ private SecretKeyTestUtil() {
+ }
+
+ public static ManagedSecretKey generateKey(
+ String algorithm, Instant creationTime, Duration validDuration)
+ throws NoSuchAlgorithmException {
+ KeyGenerator keyGen = KeyGenerator.getInstance(algorithm);
+ SecretKey secretKey = keyGen.generateKey();
+ return new ManagedSecretKey(
+ UUID.randomUUID(),
+ creationTime,
+ creationTime.plus(validDuration),
+ secretKey
+ );
+ }
+
+ public static ManagedSecretKey generateHmac(
+ Instant creationTime, Duration validDuration)
+ throws NoSuchAlgorithmException {
+ return generateKey("HmacSHA256", creationTime, validDuration);
+ }
+}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestBlockTokenVerifier.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestBlockTokenVerifier.java
index c108a2670d..91825b09cc 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestBlockTokenVerifier.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestBlockTokenVerifier.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerC
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import java.io.IOException;
@@ -46,8 +46,8 @@ public class TestBlockTokenVerifier
@Override
protected TokenVerifier newTestSubject(SecurityConfig secConf,
- CertificateClient caClient) {
- return new BlockTokenVerifier(secConf, caClient);
+ SecretKeyVerifierClient secretKeyClient) {
+ return new BlockTokenVerifier(secConf, secretKeyClient);
}
@Override
@@ -67,10 +67,12 @@ public class TestBlockTokenVerifier
@Override
protected OzoneBlockTokenIdentifier newTokenId() {
- return new OzoneBlockTokenIdentifier("any user",
- new BlockID(1, 0),
- EnumSet.allOf(AccessModeProto.class),
- Instant.now().plusSeconds(3600).toEpochMilli(),
- CERT_ID, 100);
+ OzoneBlockTokenIdentifier tokenId =
+ new OzoneBlockTokenIdentifier("any user",
+ new BlockID(1, 0),
+ EnumSet.allOf(AccessModeProto.class),
+ Instant.now().plusSeconds(3600).toEpochMilli(), 100);
+ tokenId.setSecretKeyId(SECRET_KEY_ID);
+ return tokenId;
}
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestContainerTokenVerifier.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestContainerTokenVerifier.java
index 25616a62ec..1704226e5a 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestContainerTokenVerifier.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestContainerTokenVerifier.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerC
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import java.io.IOException;
import java.time.Instant;
@@ -48,8 +48,8 @@ public class TestContainerTokenVerifier
@Override
protected TokenVerifier newTestSubject(SecurityConfig secConf,
- CertificateClient caClient) {
- return new ContainerTokenVerifier(secConf, caClient);
+ SecretKeyVerifierClient secretKeyClient) {
+ return new ContainerTokenVerifier(secConf, secretKeyClient);
}
@Override
@@ -69,8 +69,10 @@ public class TestContainerTokenVerifier
@Override
protected ContainerTokenIdentifier newTokenId() {
- return new ContainerTokenIdentifier("any user",
- ContainerID.valueOf(CONTAINER_ID.incrementAndGet()), "123",
+ ContainerTokenIdentifier tokenId = new ContainerTokenIdentifier("any user",
+ ContainerID.valueOf(CONTAINER_ID.incrementAndGet()),
Instant.now().plusSeconds(3600));
+ tokenId.setSecretKeyId(SECRET_KEY_ID);
+ return tokenId;
}
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000000..fbedf2de1a
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyTestUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.EnumSet;
+
+import static java.time.Duration.ofDays;
+import static java.time.Instant.now;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test class for {@link OzoneBlockTokenIdentifier}.
+ */
+public class TestOzoneBlockTokenIdentifier {
+ private long expiryTime;
+ private ManagedSecretKey secretKey;
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+ secretKey = SecretKeyTestUtil.generateHmac(now(), ofDays(1));
+ }
+
+ @Test
+ public void testSignToken() {
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(AccessModeProto.class),
+ expiryTime, 128L);
+ tokenId.setSecretKeyId(secretKey.getId());
+ byte[] signedToken = secretKey.sign(tokenId);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master.
+ assertTrue(secretKey.isValidSignature(tokenId.getBytes(), signedToken));
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master.
+ assertFalse(secretKey.isValidSignature(tokenId.getBytes(),
+ RandomUtils.nextBytes(128)));
+ }
+
+ @Test
+ public void testTokenSerialization() throws
+ IOException {
+ long maxLength = 128L;
+
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(AccessModeProto.class),
+ expiryTime, maxLength);
+ tokenId.setSecretKeyId(secretKey.getId());
+ byte[] signedToken = secretKey.sign(tokenId);
+
+ Token<OzoneBlockTokenIdentifier> token = new Token<>(tokenId.getBytes(),
+ signedToken, tokenId.getKind(), new Text("host:port"));
+
+ String encodeToUrlString = token.encodeToUrlString();
+
+ Token<OzoneBlockTokenIdentifier>decodedToken = new Token<>();
+ decodedToken.decodeFromUrlString(encodeToUrlString);
+
+ OzoneBlockTokenIdentifier decodedTokenId = new OzoneBlockTokenIdentifier();
+ decodedTokenId.readFields(new DataInputStream(
+ new ByteArrayInputStream(decodedToken.getIdentifier())));
+
+ Assertions.assertEquals(tokenId, decodedTokenId);
+ Assertions.assertEquals(maxLength, decodedTokenId.getMaxLength());
+
+ // Verify a decoded signed Token
+ assertTrue(secretKey.isValidSignature(decodedTokenId,
+ decodedToken.getPassword()));
+ }
+}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java
index 1c93dcc180..d8c2271323 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java
@@ -18,19 +18,6 @@
package org.apache.hadoop.hdds.security.token;
-import static org.apache.hadoop.ozone.container.ContainerTestHelper.getBlockRequest;
-import static org.apache.hadoop.ozone.container.ContainerTestHelper.getReadChunkRequest;
-import static org.apache.hadoop.ozone.container.ContainerTestHelper.newPutBlockRequestBuilder;
-import static org.apache.hadoop.ozone.container.ContainerTestHelper.newWriteChunkRequestBuilder;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.AdditionalAnswers.delegatesTo;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -38,48 +25,36 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerC
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyTestUtil;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.token.Token;
import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ozone.test.LambdaTestUtils;
-import org.apache.hadoop.util.Time;
-import org.bouncycastle.asn1.x500.X500Name;
-import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
-import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.X509v1CertificateBuilder;
-import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
-import org.bouncycastle.crypto.util.PrivateKeyFactory;
-import org.bouncycastle.operator.ContentSigner;
-import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder;
-import org.bouncycastle.operator.DefaultSignatureAlgorithmIdentifierFinder;
-import org.bouncycastle.operator.OperatorCreationException;
-import org.bouncycastle.operator.bc.BcRSAContentSignerBuilder;
-import org.bouncycastle.pkcs.PKCS10CertificationRequest;
-import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.slf4j.Logger;
+import org.mockito.Mockito;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.nio.file.Path;
-import java.security.KeyPair;
-import java.security.SecureRandom;
-import java.security.Signature;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.time.Instant;
-import java.util.Date;
+import java.security.NoSuchAlgorithmException;
import java.util.EnumSet;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import static java.time.Duration.ofDays;
+import static java.time.Instant.now;
+import static org.apache.hadoop.ozone.container.ContainerTestHelper.getBlockRequest;
+import static org.apache.hadoop.ozone.container.ContainerTestHelper.getReadChunkRequest;
+import static org.apache.hadoop.ozone.container.ContainerTestHelper.newPutBlockRequestBuilder;
+import static org.apache.hadoop.ozone.container.ContainerTestHelper.newWriteChunkRequestBuilder;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
/**
* Test class for {@link OzoneBlockTokenSecretManager}.
*/
@@ -87,14 +62,15 @@ public class TestOzoneBlockTokenSecretManager {
private static final String BASEDIR = GenericTestUtils
.getTempPath(TestOzoneBlockTokenSecretManager.class.getSimpleName());
- private static final String ALGORITHM = "SHA256withRSA";
+ private static final String ALGORITHM = "HmacSHA256";
private OzoneBlockTokenSecretManager secretManager;
- private KeyPair keyPair;
- private String omCertSerialId;
- private CertificateClient client;
+ private UUID secretKeyId;
+ private SecretKeyVerifierClient secretKeyClient;
+ private SecretKeySignerClient secretKeySignerClient;
private TokenVerifier tokenVerifier;
private Pipeline pipeline;
+ private ManagedSecretKey secretKey;
@Before
public void setUp() throws Exception {
@@ -103,40 +79,19 @@ public class TestOzoneBlockTokenSecretManager {
OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
conf.setBoolean(HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED, true);
- // Create Ozone Master key pair.
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- // Create Ozone Master certificate (SCM CA issued cert) and key store.
SecurityConfig securityConfig = new SecurityConfig(conf);
- X509Certificate x509Certificate = KeyStoreTestUtil
- .generateCertificate("CN=OzoneMaster", keyPair, 30, ALGORITHM);
- omCertSerialId = x509Certificate.getSerialNumber().toString();
- secretManager = new OzoneBlockTokenSecretManager(securityConfig,
- TimeUnit.HOURS.toMillis(1));
- Logger log = mock(Logger.class);
- DefaultCertificateClient toStub =
- new DefaultCertificateClient(
- securityConfig, log, null, "test", null, null) {
- @Override
- protected String signAndStoreCertificate(
- PKCS10CertificationRequest request, Path certificatePath) {
- return null;
- }
- };
- client = mock(DefaultCertificateClient.class, delegatesTo(toStub));
- doReturn(x509Certificate).when(client).getCertificate();
- doReturn(x509Certificate).when(client).getCertificate(anyString());
- doReturn(keyPair.getPublic()).when(client).getPublicKey();
- doReturn(keyPair.getPrivate()).when(client).getPrivateKey();
- doReturn(null).when(client).signData(any(byte[].class));
-
- secretManager.start(client);
- tokenVerifier = new BlockTokenVerifier(securityConfig, client);
- }
- @After
- public void tearDown() throws IOException {
- secretManager = null;
- client.close();
+ secretKey = generateValidSecretKey();
+ secretKeyId = secretKey.getId();
+
+ secretKeyClient = Mockito.mock(SecretKeyVerifierClient.class);
+ secretKeySignerClient = Mockito.mock(SecretKeySignerClient.class);
+ when(secretKeySignerClient.getCurrentSecretKey()).thenReturn(secretKey);
+ when(secretKeyClient.getSecretKey(secretKeyId)).thenReturn(secretKey);
+
+ secretManager = new OzoneBlockTokenSecretManager(
+ TimeUnit.HOURS.toMillis(1), secretKeySignerClient);
+ tokenVerifier = new BlockTokenVerifier(securityConfig, secretKeyClient);
}
@Test
@@ -153,7 +108,7 @@ public class TestOzoneBlockTokenSecretManager {
identifier.getService());
Assert.assertEquals(EnumSet.allOf(AccessModeProto.class),
identifier.getAccessModes());
- Assert.assertEquals(omCertSerialId, identifier.getCertSerialId());
+ Assert.assertEquals(secretKeyId, identifier.getSecretKeyId());
validateHash(token.getPassword(), token.getIdentifier());
}
@@ -170,9 +125,8 @@ public class TestOzoneBlockTokenSecretManager {
btIdentifier.getService());
Assert.assertEquals(EnumSet.allOf(AccessModeProto.class),
btIdentifier.getAccessModes());
- Assert.assertEquals(omCertSerialId, btIdentifier.getCertSerialId());
-
byte[] hash = secretManager.createPassword(btIdentifier);
+ Assert.assertEquals(secretKeyId, btIdentifier.getSecretKeyId());
validateHash(hash, btIdentifier.getBytes());
}
@@ -224,57 +178,8 @@ public class TestOzoneBlockTokenSecretManager {
OzoneBlockTokenIdentifier.getTokenService(otherBlockID)));
}
- /**
- * Validate hash using public key of KeyPair.
- * */
private void validateHash(byte[] hash, byte[] identifier) throws Exception {
- Signature rsaSignature =
- Signature.getInstance(secretManager.getDefaultSignatureAlgorithm());
- rsaSignature.initVerify(client.getPublicKey());
- rsaSignature.update(identifier);
- assertTrue(rsaSignature.verify(hash));
- }
-
- @Test
- @SuppressWarnings("java:S2699")
- public void testCreateIdentifierFailure() throws Exception {
- LambdaTestUtils.intercept(SecurityException.class,
- "Ozone block token can't be created without owner and access mode "
- + "information.", () -> {
- secretManager.createIdentifier();
- });
- }
-
- @Test
- @SuppressWarnings("java:S2699")
- public void testRenewToken() throws Exception {
- LambdaTestUtils.intercept(UnsupportedOperationException.class,
- "Renew token operation is not supported for ozone block" +
- " tokens.", () -> {
- secretManager.renewToken(null, null);
- });
- }
-
- @Test
- @SuppressWarnings("java:S2699")
- public void testCancelToken() throws Exception {
- LambdaTestUtils.intercept(UnsupportedOperationException.class,
- "Cancel token operation is not supported for ozone block" +
- " tokens.", () -> {
- secretManager.cancelToken(null, null);
- });
- }
-
- @Test
- @SuppressWarnings("java:S2699")
- public void testVerifySignatureFailure() throws Exception {
- OzoneBlockTokenIdentifier id = new OzoneBlockTokenIdentifier(
- "testUser", "123", EnumSet.allOf(AccessModeProto.class),
- Time.now() + 60 * 60 * 24, "123444", 1024);
- LambdaTestUtils.intercept(UnsupportedOperationException.class, "operation" +
- " is not supported for block tokens",
- () -> secretManager.verifySignature(id,
- client.signData(id.getBytes())));
+ assertTrue(secretKey.isValidSignature(identifier, hash));
}
@Test
@@ -328,7 +233,7 @@ public class TestOzoneBlockTokenSecretManager {
}
@Test
- public void testExpiredCertificate() throws Exception {
+ public void testExpiredSecretKey() throws Exception {
String user = "testUser2";
BlockID blockID = new BlockID(102, 0);
Token<OzoneBlockTokenIdentifier> token =
@@ -342,79 +247,23 @@ public class TestOzoneBlockTokenSecretManager {
tokenVerifier.verify("testUser", token, writeChunkRequest);
// Mock client with an expired cert
- X509Certificate expiredCert = generateExpiredCert(
- "CN=OzoneMaster", keyPair, ALGORITHM);
- when(client.getCertificate(anyString())).thenReturn(expiredCert);
+ ManagedSecretKey expiredSecretKey = generateExpiredSecretKey();
+ when(secretKeyClient.getSecretKey(any())).thenReturn(expiredSecretKey);
BlockTokenException e = assertThrows(BlockTokenException.class,
() -> tokenVerifier.verify(user, token, writeChunkRequest));
String msg = e.getMessage();
assertTrue(msg, msg.contains("Token can't be verified due to" +
- " expired certificate"));
- }
-
- @Test
- public void testNetYetValidCertificate() throws Exception {
- String user = "testUser2";
- BlockID blockID = new BlockID(102, 0);
- Token<OzoneBlockTokenIdentifier> token =
- secretManager.generateToken(user, blockID,
- EnumSet.allOf(AccessModeProto.class), 100);
- ContainerCommandRequestProto writeChunkRequest =
- newWriteChunkRequestBuilder(pipeline, blockID, 100)
- .setEncodedToken(token.encodeToUrlString())
- .build();
-
- tokenVerifier.verify(user, token, writeChunkRequest);
-
- // Mock client with an expired cert
- X509Certificate netYetValidCert = generateNotValidYetCert(
- "CN=OzoneMaster", keyPair, ALGORITHM);
- when(client.getCertificate(anyString())).
- thenReturn(netYetValidCert);
-
- BlockTokenException e = assertThrows(BlockTokenException.class,
- () -> tokenVerifier.verify(user, token, writeChunkRequest));
- String msg = e.getMessage();
- assertTrue(msg, msg.contains("Token can't be verified due to not" +
- " yet valid certificate"));
- }
-
- private X509Certificate generateExpiredCert(String dn,
- KeyPair pair, String algorithm) throws CertificateException,
- IllegalStateException, IOException, OperatorCreationException {
- Date from = new Date();
- // Set end date same as start date to make sure the cert is expired.
- return generateTestCert(dn, pair, algorithm, from, from);
+ " expired secret key"));
}
- private X509Certificate generateNotValidYetCert(String dn,
- KeyPair pair, String algorithm) throws CertificateException,
- IllegalStateException, IOException, OperatorCreationException {
- Date from = new Date(Instant.now().toEpochMilli() + 100000L);
- Date to = new Date(from.getTime() + 200000L);
- return generateTestCert(dn, pair, algorithm, from, to);
+ private ManagedSecretKey generateValidSecretKey()
+ throws NoSuchAlgorithmException {
+ return SecretKeyTestUtil.generateKey(ALGORITHM, now(), ofDays(1));
}
- private X509Certificate generateTestCert(String dn,
- KeyPair pair, String algorithm, Date from, Date to)
- throws CertificateException, IllegalStateException,
- IOException, OperatorCreationException {
- BigInteger sn = new BigInteger(64, new SecureRandom());
- SubjectPublicKeyInfo subPubKeyInfo = SubjectPublicKeyInfo.getInstance(
- pair.getPublic().getEncoded());
- X500Name subjectDN = new X500Name(dn);
- X509v1CertificateBuilder builder = new X509v1CertificateBuilder(
- subjectDN, sn, from, to, subjectDN, subPubKeyInfo);
-
- AlgorithmIdentifier sigAlgId =
- new DefaultSignatureAlgorithmIdentifierFinder().find(algorithm);
- AlgorithmIdentifier digAlgId =
- new DefaultDigestAlgorithmIdentifierFinder().find(sigAlgId);
- ContentSigner signer =
- new BcRSAContentSignerBuilder(sigAlgId, digAlgId)
- .build(PrivateKeyFactory.createKey(pair.getPrivate().getEncoded()));
- X509CertificateHolder holder = builder.build(signer);
- return new JcaX509CertificateConverter().getCertificate(holder);
+ private ManagedSecretKey generateExpiredSecretKey() throws Exception {
+ return SecretKeyTestUtil.generateKey(ALGORITHM,
+ now().minus(ofDays(2)), ofDays(1));
}
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java
index 009b7a0105..1c7085b35b 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java
@@ -19,25 +19,30 @@ package org.apache.hadoop.hdds.security.token;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.crypto.SecretKey;
import java.io.IOException;
-import java.security.cert.CertificateException;
-import java.security.cert.CertificateExpiredException;
-import java.security.cert.CertificateNotYetValidException;
-import java.security.cert.X509Certificate;
+import java.time.Duration;
import java.time.Instant;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@@ -51,13 +56,13 @@ public abstract class TokenVerifierTests<T extends ShortLivedTokenIdentifier> {
private static final Logger LOG =
LoggerFactory.getLogger(TokenVerifierTests.class);
- protected static final String CERT_ID = "123";
+ protected static final UUID SECRET_KEY_ID = UUID.randomUUID();
/**
* Create the specific kind of TokenVerifier.
*/
protected abstract TokenVerifier newTestSubject(
- SecurityConfig secConf, CertificateClient caClient);
+ SecurityConfig secConf, SecretKeyVerifierClient secretKeyClient);
/**
* @return the config key to enable/disable the specific kind of tokens
@@ -82,98 +87,138 @@ public abstract class TokenVerifierTests<T extends ShortLivedTokenIdentifier> {
@Test
public void skipsVerificationIfDisabled() throws IOException {
// GIVEN
- CertificateClient caClient = mock(CertificateClient.class);
- TokenVerifier subject = newTestSubject(tokenDisabled(), caClient);
+ SecretKeyVerifierClient secretKeyClient = mock(
+ SecretKeyVerifierClient.class);
+ TokenVerifier subject = newTestSubject(tokenDisabled(), secretKeyClient);
// WHEN
subject.verify("anyUser", anyToken(), verifiedRequest(newTokenId()));
// THEN
- verify(caClient, never()).getCertificate(any());
+ verify(secretKeyClient, never()).getSecretKey(any());
}
@Test
public void skipsVerificationForMiscCommands() throws IOException {
// GIVEN
- CertificateClient caClient = mock(CertificateClient.class);
- TokenVerifier subject = newTestSubject(tokenEnabled(), caClient);
+ SecretKeyVerifierClient secretKeyClient = mock(
+ SecretKeyVerifierClient.class);
+ TokenVerifier subject = newTestSubject(tokenEnabled(), secretKeyClient);
// WHEN
subject.verify("anyUser", anyToken(), unverifiedRequest());
// THEN
- verify(caClient, never()).getCertificate(any());
+ verify(secretKeyClient, never()).getSecretKey(any());
}
@Test
- public void rejectsExpiredCertificate() throws Exception {
- rejectsInvalidCertificate(CertificateExpiredException.class);
- }
+ public void rejectsExpiredSecretKey() throws Exception {
+ // GIVEN
+ SecretKeyVerifierClient secretKeyClient =
+ mock(SecretKeyVerifierClient.class);
- @Test
- public void rejectsNotYetValidCertificate() throws Exception {
- rejectsInvalidCertificate(CertificateNotYetValidException.class);
+ Instant past = Instant.now().minus(Duration.ofHours(1));
+ ManagedSecretKey expiredSecretKey = new ManagedSecretKey(UUID.randomUUID(),
+ past, past, Mockito.mock(SecretKey.class));
+
+ when(secretKeyClient.getSecretKey(SECRET_KEY_ID))
+ .thenReturn(expiredSecretKey);
+ T tokenId = newTokenId();
+ ContainerCommandRequestProto cmd = verifiedRequest(tokenId);
+ TokenVerifier subject = newTestSubject(tokenEnabled(), secretKeyClient);
+
+ // WHEN+THEN
+ ShortLivedTokenSecretManager<T> secretManager = new MockTokenManager();
+ Token<T> token = secretManager.generateToken(tokenId);
+ BlockTokenException ex = assertThrows(BlockTokenException.class, () ->
+ subject.verify("anyUser", token, cmd));
+ assertThat(ex.getMessage(), containsString("expired secret key"));
}
- private void rejectsInvalidCertificate(
- Class<? extends CertificateException> problem) throws Exception {
+ @Test
+ public void rejectsTokenWithInvalidSecretId() throws Exception {
// GIVEN
- CertificateClient caClient = mock(CertificateClient.class);
- X509Certificate cert = invalidCertificate(problem);
- when(caClient.getCertificate(CERT_ID)).thenReturn(cert);
- ContainerCommandRequestProto cmd = verifiedRequest(newTokenId());
- TokenVerifier subject = newTestSubject(tokenEnabled(), caClient);
+ SecretKeyVerifierClient secretKeyClient =
+ mock(SecretKeyVerifierClient.class);
+
+ when(secretKeyClient.getSecretKey(SECRET_KEY_ID)).thenReturn(null);
+ T tokenId = newTokenId();
+ ContainerCommandRequestProto cmd = verifiedRequest(tokenId);
+ TokenVerifier subject = newTestSubject(tokenEnabled(), secretKeyClient);
// WHEN+THEN
- assertThrows(BlockTokenException.class, () ->
- subject.verify("anyUser", anyToken(), cmd));
+ ShortLivedTokenSecretManager<T> secretManager = new MockTokenManager();
+ Token<T> token = secretManager.generateToken(tokenId);
+ BlockTokenException ex = assertThrows(BlockTokenException.class, () ->
+ subject.verify("anyUser", token, cmd));
+ assertThat(ex.getMessage(),
+ containsString("Can't find the signer secret key"));
}
@Test
public void rejectsInvalidSignature() throws Exception {
// GIVEN
- CertificateClient caClient = mock(CertificateClient.class);
- when(caClient.getCertificate(CERT_ID)).thenReturn(validCertificate());
- Token<?> invalidToken = new Token<>();
- validSignature(caClient, false);
- ContainerCommandRequestProto cmd = verifiedRequest(newTokenId());
- TokenVerifier subject = newTestSubject(tokenEnabled(), caClient);
+ SecretKeyVerifierClient secretKeyClient =
+ mockSecretKeyClient(false);
+
+ ShortLivedTokenSecretManager<T> secretManager = new MockTokenManager();
+ T tokenId = newTokenId();
+ Token<?> invalidToken = secretManager.generateToken(tokenId);
+ ContainerCommandRequestProto cmd = verifiedRequest(tokenId);
+ TokenVerifier subject = newTestSubject(tokenEnabled(), secretKeyClient);
// WHEN+THEN
- assertThrows(BlockTokenException.class, () ->
- subject.verify("anyUser", invalidToken, cmd));
+ BlockTokenException ex =
+ assertThrows(BlockTokenException.class, () ->
+ subject.verify("anyUser", invalidToken, cmd));
+ assertThat(ex.getMessage(),
+ containsString("Invalid token for user"));
+ }
+
+ @NotNull
+ private SecretKeyVerifierClient mockSecretKeyClient(boolean validSignature)
+ throws IOException {
+ SecretKeyVerifierClient secretKeyClient =
+ mock(SecretKeyVerifierClient.class);
+ ManagedSecretKey validSecretKey = Mockito.mock(ManagedSecretKey.class);
+ when(secretKeyClient.getSecretKey(SECRET_KEY_ID))
+ .thenReturn(validSecretKey);
+ when(validSecretKey.isValidSignature((TokenIdentifier) any(), any()))
+ .thenReturn(validSignature);
+ return secretKeyClient;
}
@Test
public void rejectsExpiredToken() throws Exception {
// GIVEN
- SecurityConfig conf = tokenEnabled();
- CertificateClient caClient = mock(CertificateClient.class);
- when(caClient.getCertificate(CERT_ID)).thenReturn(validCertificate());
- validSignature(caClient, true);
- ShortLivedTokenSecretManager<T> secretManager = new MockTokenManager(conf);
+ SecretKeyVerifierClient secretKeyClient = mockSecretKeyClient(true);
+
+ ShortLivedTokenSecretManager<T> secretManager = new MockTokenManager();
T tokenId = expired(newTokenId());
ContainerCommandRequestProto cmd = verifiedRequest(tokenId);
Token<?> token = secretManager.generateToken(tokenId);
- TokenVerifier subject = newTestSubject(tokenEnabled(), caClient);
+ TokenVerifier subject = newTestSubject(tokenEnabled(), secretKeyClient);
// WHEN+THEN
- assertThrows(BlockTokenException.class, () ->
- subject.verify("anyUser", token, cmd));
+ BlockTokenException ex =
+ assertThrows(BlockTokenException.class, () ->
+ subject.verify("anyUser", token, cmd));
+ assertThat(ex.getMessage(),
+ containsString("Expired token for user"));
}
@Test
public void acceptsValidToken() throws Exception {
// GIVEN
SecurityConfig conf = tokenEnabled();
- CertificateClient caClient = mock(CertificateClient.class);
- when(caClient.getCertificate(CERT_ID)).thenReturn(validCertificate());
- validSignature(caClient, true);
- ShortLivedTokenSecretManager<T> secretManager = new MockTokenManager(conf);
+ SecretKeyVerifierClient secretKeyClient = mockSecretKeyClient(true);
+
+ ShortLivedTokenSecretManager<T> secretManager = new MockTokenManager();
T tokenId = valid(newTokenId());
ContainerCommandRequestProto cmd = verifiedRequest(tokenId);
Token<?> token = secretManager.generateToken(tokenId);
- TokenVerifier subject = newTestSubject(conf, caClient);
+ TokenVerifier subject = newTestSubject(conf, secretKeyClient);
// WHEN+THEN
subject.verify("anyUser", token, cmd);
@@ -189,24 +234,6 @@ public abstract class TokenVerifierTests<T extends ShortLivedTokenIdentifier> {
return tokenId;
}
- private void validSignature(CertificateClient caClient, boolean valid)
- throws Exception {
- when(caClient.verifySignature(any(byte[].class), any(), any()))
- .thenReturn(valid);
- }
-
- private static X509Certificate invalidCertificate(
- Class<? extends CertificateException> problem)
- throws CertificateExpiredException, CertificateNotYetValidException {
- X509Certificate cert = mock(X509Certificate.class);
- doThrow(problem).when(cert).checkValidity();
- return cert;
- }
-
- private static X509Certificate validCertificate() {
- return mock(X509Certificate.class);
- }
-
protected SecurityConfig tokenDisabled() {
return getSecurityConfig(false);
}
@@ -230,8 +257,9 @@ public abstract class TokenVerifierTests<T extends ShortLivedTokenIdentifier> {
*/
private class MockTokenManager extends ShortLivedTokenSecretManager<T> {
- MockTokenManager(SecurityConfig conf) {
- super(conf, TimeUnit.HOURS.toMillis(1), LOG);
+ MockTokenManager() {
+ super(TimeUnit.HOURS.toMillis(1),
+ Mockito.mock(SecretKeySignerClient.class));
}
@Override
diff --git a/hadoop-hdds/framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/hadoop-hdds/framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
new file mode 100644
index 0000000000..72652da02f
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+mock-maker-inline
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index b4ffd37a24..97e704df95 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -532,7 +532,8 @@ message ContainerTokenSecretProto {
required string ownerId = 1;
required ContainerID containerId = 2;
required uint64 expiryDate = 3;
- required string certSerialId = 4;
+ optional string certSerialId = 4 [deprecated=true];
+ optional UUID secretKeyId = 5;
}
message GetContainerTokenRequestProto {
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index a8a748aad7..ad086eb810 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -368,9 +368,10 @@ message BlockTokenSecretProto {
required string ownerId = 1;
required string blockId = 2;
required uint64 expiryDate = 3;
- required string omCertSerialId = 4;
+ optional string omCertSerialId = 4 [deprecated=true];
repeated AccessModeProto modes = 5;
required uint64 maxLength = 6;
+ optional UUID secretKeyId = 7;
}
message BlockID {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index 8d09b95296..b1c25ae7a6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -178,13 +178,13 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
@Override
public ManagedSecretKey getCurrentSecretKey() throws SCMSecurityException {
validateSecretKeyStatus();
- return secretKeyManager.getCurrentKey();
+ return secretKeyManager.getCurrentSecretKey();
}
@Override
public ManagedSecretKey getSecretKey(UUID id) throws SCMSecurityException {
validateSecretKeyStatus();
- return secretKeyManager.getKey(id);
+ return secretKeyManager.getSecretKey(id);
}
@Override
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index e79e7a5536..467a406f7e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -133,7 +133,6 @@ import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolic
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
-import org.apache.hadoop.hdds.security.OzoneSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer;
@@ -169,7 +168,6 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.Containe
import javax.management.ObjectName;
import java.io.IOException;
-import java.io.UncheckedIOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -992,8 +990,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmCertificateClient = new SCMCertificateClient(securityConfig,
certSerialNumber, SCM_ROOT_CA_COMPONENT_NAME);
}
- return new ContainerTokenSecretManager(securityConfig,
- expiryTime);
+ return new ContainerTokenSecretManager(expiryTime,
+ secretKeyManagerService.getSecretKeyManager());
}
/**
@@ -1495,7 +1493,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
scmHAManager.start();
- startSecretManagerIfNecessary();
ms = HddsServerUtil
.initializeMetrics(configuration, "StorageContainerManager");
@@ -1650,8 +1647,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
LOG.error("SCM block manager service stop failed.", ex);
}
- stopSecretManager();
-
if (metrics != null) {
metrics.unRegister();
}
@@ -1831,6 +1826,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return scmSafeModeManager;
}
+ @VisibleForTesting
+ public SecretKeyManager getSecretKeyManager() {
+ return secretKeyManagerService != null ?
+ secretKeyManagerService.getSecretKeyManager() : null;
+ }
+
@Override
public ReplicationManager getReplicationManager() {
return replicationManager;
@@ -2025,46 +2026,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return scmHAMetrics;
}
- private void startSecretManagerIfNecessary() {
- boolean shouldRun = securityConfig.isSecurityEnabled()
- && securityConfig.isContainerTokenEnabled()
- && containerTokenMgr != null;
- if (shouldRun) {
- boolean running = containerTokenMgr.isRunning();
- if (!running) {
- startSecretManager();
- }
- }
- }
-
- private void startSecretManager() {
- try {
- scmCertificateClient.assertValidKeysAndCertificate();
- } catch (OzoneSecurityException e) {
- LOG.error("Unable to read key pair.", e);
- throw new UncheckedIOException(e);
- }
- try {
- LOG.info("Starting token manager");
- containerTokenMgr.start(scmCertificateClient);
- } catch (IOException e) {
- // Unable to start secret manager.
- LOG.error("Error starting block token secret manager.", e);
- throw new UncheckedIOException(e);
- }
- }
-
- private void stopSecretManager() {
- if (containerTokenMgr != null) {
- LOG.info("Stopping block token manager.");
- try {
- containerTokenMgr.stop();
- } catch (IOException e) {
- LOG.error("Failed to stop block token manager", e);
- }
- }
- }
-
public ContainerTokenGenerator getContainerTokenGenerator() {
return containerTokenMgr != null
? containerTokenMgr
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index 7af924454b..b8eb7b0cb1 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -148,7 +148,7 @@ public class TestEndPoint {
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
OzoneContainer ozoneContainer = new OzoneContainer(
- datanodeDetails, conf, getContext(datanodeDetails), null);
+ datanodeDetails, conf, getContext(datanodeDetails));
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@@ -182,7 +182,7 @@ public class TestEndPoint {
serverAddress, 1000)) {
DatanodeDetails datanodeDetails = randomDatanodeDetails();
OzoneContainer ozoneContainer = new OzoneContainer(
- datanodeDetails, conf, getContext(datanodeDetails), null);
+ datanodeDetails, conf, getContext(datanodeDetails));
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
String clusterId = scmServerImpl.getClusterId();
@@ -261,7 +261,7 @@ public class TestEndPoint {
.captureLogs(VersionEndpointTask.LOG);
DatanodeDetails datanodeDetails = randomDatanodeDetails();
OzoneContainer ozoneContainer = new OzoneContainer(
- datanodeDetails, conf, getContext(datanodeDetails), null);
+ datanodeDetails, conf, getContext(datanodeDetails));
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@@ -310,7 +310,7 @@ public class TestEndPoint {
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
DatanodeDetails datanodeDetails = randomDatanodeDetails();
OzoneContainer ozoneContainer = new OzoneContainer(
- datanodeDetails, conf, getContext(datanodeDetails), null);
+ datanodeDetails, conf, getContext(datanodeDetails));
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
EndpointStateMachine.EndPointStates newState = versionTask.call();
@@ -338,7 +338,7 @@ public class TestEndPoint {
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
DatanodeDetails datanodeDetails = randomDatanodeDetails();
OzoneContainer ozoneContainer = new OzoneContainer(
- datanodeDetails, conf, getContext(datanodeDetails), null);
+ datanodeDetails, conf, getContext(datanodeDetails));
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@@ -576,7 +576,7 @@ public class TestEndPoint {
// Create a datanode state machine for stateConext used by endpoint task
try (DatanodeStateMachine stateMachine = new DatanodeStateMachine(
- randomDatanodeDetails(), conf, null, null, null);
+ randomDatanodeDetails(), conf);
EndpointStateMachine rpcEndPoint =
createEndpoint(conf, scmAddress, rpcTimeout)) {
HddsProtos.DatanodeDetailsProto datanodeDetailsProto =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
index 9f96448f3b..21a4435970 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
@@ -43,10 +43,10 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.WritableECContainerProvider.WritableECContainerProviderConfig;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -57,6 +57,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.client.io.InsufficientLocationsException;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
@@ -105,7 +106,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.READ;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.WRITE;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.newWriteChunkRequestBuilder;
/**
@@ -144,6 +144,7 @@ public class TestContainerCommandsEC {
private static Token<ContainerTokenIdentifier> containerToken;
private static ContainerTokenSecretManager containerTokenGenerator;
private static OzoneBlockTokenSecretManager blockTokenGenerator;
+ private static SecretKeyClient secretKeyClient;
private List<XceiverClientSpi> clients = null;
private static OzoneConfiguration config;
private static CertificateClient certClient;
@@ -401,7 +402,7 @@ public class TestContainerCommandsEC {
XceiverClientManager xceiverClientManager =
new XceiverClientManager(config);
ECReconstructionCoordinator coordinator =
- new ECReconstructionCoordinator(config, certClient,
+ new ECReconstructionCoordinator(config, certClient, secretKeyClient,
null, ECReconstructionMetrics.create())) {
ECReconstructionMetrics metrics =
@@ -597,7 +598,7 @@ public class TestContainerCommandsEC {
Assert.assertThrows(IOException.class, () -> {
try (ECReconstructionCoordinator coordinator =
- new ECReconstructionCoordinator(config, certClient,
+ new ECReconstructionCoordinator(config, certClient, secretKeyClient,
null, ECReconstructionMetrics.create())) {
coordinator.reconstructECContainerGroup(conID,
(ECReplicationConfig) containerPipeline.getReplicationConfig(),
@@ -659,10 +660,12 @@ public class TestContainerCommandsEC {
OzoneManager.setTestSecureOmFlag(true);
certClient = new CertificateClientTestImpl(conf);
+ secretKeyClient = new SecretKeyTestClient();
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(NUM_DN)
.setScmId(SCM_ID).setClusterId(CLUSTER_ID)
.setCertificateClient(certClient)
+ .setSecretKeyClient(secretKeyClient)
.build();
cluster.waitForClusterToBeReady();
cluster.getOzoneManager().startSecretManager();
@@ -706,16 +709,11 @@ public class TestContainerCommandsEC {
pipeline = pipelines.get(0);
datanodeDetails = pipeline.getNodes();
- OzoneConfiguration tweakedConfig = new OzoneConfiguration(config);
- tweakedConfig.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- SecurityConfig conf = new SecurityConfig(tweakedConfig);
long tokenLifetime = TimeUnit.DAYS.toMillis(1);
containerTokenGenerator = new ContainerTokenSecretManager(
- conf, tokenLifetime);
- containerTokenGenerator.start(certClient);
+ tokenLifetime, secretKeyClient);
blockTokenGenerator = new OzoneBlockTokenSecretManager(
- conf, tokenLifetime);
- blockTokenGenerator.start(certClient);
+ tokenLifetime, secretKeyClient);
containerToken = containerTokenGenerator
.generateToken(ANY_USER, new ContainerID(containerID));
}
@@ -732,14 +730,6 @@ public class TestContainerCommandsEC {
if (cluster != null) {
cluster.shutdown();
}
-
- if (blockTokenGenerator != null) {
- blockTokenGenerator.stop();
- }
-
- if (containerTokenGenerator != null) {
- containerTokenGenerator.stop();
- }
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 4d648ff3bd..8b4f2f146b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -337,6 +338,7 @@ public interface MiniOzoneCluster {
protected int numDataVolumes = 1;
protected boolean startDataNodes = true;
protected CertificateClient certClient;
+ protected SecretKeyClient secretKeyClient;
protected int pipelineNumLimit = DEFAULT_PIPELINE_LIMIT;
protected Builder(OzoneConfiguration conf) {
@@ -399,6 +401,11 @@ public interface MiniOzoneCluster {
return this;
}
+ public Builder setSecretKeyClient(SecretKeyClient client) {
+ this.secretKeyClient = client;
+ return this;
+ }
+
/**
* Sets the SCM id.
*
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 6e69ff91dd..3dc4125281 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.hdds.utils.db.CodecBuffer;
@@ -133,6 +134,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
private int waitForClusterToBeReadyTimeout = 120000; // 2 min
private CertificateClient caClient;
private final Set<AutoCloseable> clients = ConcurrentHashMap.newKeySet();
+ private SecretKeyClient secretKeyClient;
/**
* Creates a new MiniOzoneCluster with Recon.
@@ -500,6 +502,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
} catch (IOException e) {
LOG.error("Exception while setting certificate client to DataNode.", e);
}
+ datanode.setSecretKeyClient(secretKeyClient);
datanode.start();
});
}
@@ -534,6 +537,10 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
this.caClient = client;
}
+ private void setSecretKeyClient(SecretKeyClient client) {
+ this.secretKeyClient = client;
+ }
+
private static void stopDatanodes(
Collection<HddsDatanodeService> hddsDatanodes) {
if (!hddsDatanodes.isEmpty()) {
@@ -608,6 +615,9 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (certClient != null) {
om.setCertClient(certClient);
}
+ if (secretKeyClient != null) {
+ om.setSecretKeyClient(secretKeyClient);
+ }
om.start();
if (includeRecon) {
@@ -624,6 +634,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
hddsDatanodes, reconServer);
cluster.setCAClient(certClient);
+ cluster.setSecretKeyClient(secretKeyClient);
if (startDataNodes) {
cluster.startHddsDatanodes();
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index b6e43bd7e3..6937b2dd6d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -139,7 +139,7 @@ public class TestMiniOzoneCluster {
for (int i = 0; i < 3; i++) {
stateMachines.add(new DatanodeStateMachine(
- randomDatanodeDetails(), ozoneConf, null, null, null));
+ randomDatanodeDetails(), ozoneConf));
}
//we need to start all the servers to get the fix ports
@@ -184,11 +184,11 @@ public class TestMiniOzoneCluster {
ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
try (
DatanodeStateMachine sm1 = new DatanodeStateMachine(
- randomDatanodeDetails(), ozoneConf, null, null, null);
+ randomDatanodeDetails(), ozoneConf);
DatanodeStateMachine sm2 = new DatanodeStateMachine(
- randomDatanodeDetails(), ozoneConf, null, null, null);
+ randomDatanodeDetails(), ozoneConf);
DatanodeStateMachine sm3 = new DatanodeStateMachine(
- randomDatanodeDetails(), ozoneConf, null, null, null);
+ randomDatanodeDetails(), ozoneConf);
) {
HashSet<Integer> ports = new HashSet<Integer>();
assertTrue(ports.add(sm1.getContainer().getReadChannel().getIPCPort()));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
index 9ec8750d78..84423cabac 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
@@ -179,7 +179,6 @@ public final class TestSecretKeysApi {
ozoneKeytab.getAbsolutePath());
}
-
/**
* Test secret key apis in happy case.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 2281f5900b..ac54113271 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -31,16 +31,12 @@ import java.security.cert.X509Certificate;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
-import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
import java.util.Date;
-import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.Callable;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -52,30 +48,20 @@ import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.ha.HASecurityUtils;
import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails;
import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider;
import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.DefaultProfile;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer;
-import org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest;
import org.apache.hadoop.hdds.security.x509.certificate.utils.SelfSignedCertificate;
import org.apache.hadoop.hdds.security.x509.exception.CertificateException;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
@@ -88,8 +74,6 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
@@ -97,9 +81,6 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
-import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
-import org.apache.hadoop.ozone.om.protocolPB.GrpcOmTransport;
-import org.apache.hadoop.ozone.om.protocolPB.GrpcOmTransportFactory;
import org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.security.OMCertificateClient;
@@ -118,13 +99,8 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_ENABLED;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
@@ -143,7 +119,6 @@ import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigString
import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.net.ServerSocketUtil.getPort;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
@@ -152,7 +127,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_F
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_TRANSPORT_CLASS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import org.apache.ratis.protocol.ClientId;
@@ -1193,199 +1167,6 @@ public final class TestSecureOzoneCluster {
}
}
- /**
- * Tests container token renewal after a certificate renew.
- */
- @Test
- public void testContainerTokenRenewCrossCertificateRenew() throws Exception {
- // Setup secure SCM for start.
- final int certLifetime = 40 * 1000; // 40s
- conf.set(HDDS_X509_DEFAULT_DURATION,
- Duration.ofMillis(certLifetime).toString());
- conf.set(HDDS_X509_MAX_DURATION,
- Duration.ofMillis(certLifetime).toString());
- conf.set(HDDS_X509_RENEW_GRACE_DURATION,
- Duration.ofMillis(certLifetime - 15 * 1000).toString());
- conf.setBoolean(HDDS_CONTAINER_TOKEN_ENABLED, true);
- conf.setLong(HDDS_BLOCK_TOKEN_EXPIRY_TIME, certLifetime - 20 * 1000);
-
- initSCM();
- scm = HddsTestUtils.getScmSimple(conf);
- try {
- CertificateClientTestImpl certClient =
- new CertificateClientTestImpl(conf, true);
- X509Certificate scmCert = certClient.getCertificate();
- String scmCertId1 = scmCert.getSerialNumber().toString();
- // Start SCM
- scm.setScmCertificateClient(certClient);
- scm.start();
-
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- // Get SCM client which will authenticate via Kerberos
- SCMContainerLocationFailoverProxyProvider proxyProvider =
- new SCMContainerLocationFailoverProxyProvider(conf, ugi);
- StorageContainerLocationProtocolClientSideTranslatorPB scmClient =
- new StorageContainerLocationProtocolClientSideTranslatorPB(
- proxyProvider);
-
- // Since client is already connected get a delegation token
- ContainerID containerID = new ContainerID(1);
- Token<?> token1 = scmClient.getContainerToken(containerID);
-
- // Check if token is of right kind and renewer is running instance
- assertNotNull(token1);
- assertEquals(ContainerTokenIdentifier.KIND, token1.getKind());
- assertEquals(containerID.toString(), token1.getService().toString());
- ContainerTokenIdentifier temp = new ContainerTokenIdentifier();
- ByteArrayInputStream buf = new ByteArrayInputStream(
- token1.getIdentifier());
- DataInputStream in = new DataInputStream(buf);
- temp.readFields(in);
- assertEquals(scmCertId1, temp.getCertSerialId());
-
- // Wait for SCM certificate to renew
- GenericTestUtils.waitFor(() -> !scmCertId1.equals(
- certClient.getCertificate().getSerialNumber().toString()),
- 100, certLifetime);
- String scmCertId2 =
- certClient.getCertificate().getSerialNumber().toString();
- assertNotEquals(scmCertId1, scmCertId2);
-
- // Get a new container token
- containerID = new ContainerID(2);
- Token<?> token2 = scmClient.getContainerToken(containerID);
- buf = new ByteArrayInputStream(token2.getIdentifier());
- in = new DataInputStream(buf);
- temp.readFields(in);
- assertEquals(scmCertId2, temp.getCertSerialId());
- } finally {
- if (scm != null) {
- scm.stop();
- }
- }
- }
-
- /**
- * Test functionality to get SCM signed certificate for OM.
- */
- @Test
- public void testOMGrpcServerCertificateRenew() throws Exception {
- initSCM();
- try {
- scm = HddsTestUtils.getScmSimple(conf);
- scm.start();
-
- conf.set(OZONE_METADATA_DIRS, omMetaDirPath.toString());
- int certLifetime = 30; // second
- conf.set(HDDS_X509_DEFAULT_DURATION,
- Duration.ofSeconds(certLifetime).toString());
- conf.setInt(OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, 2);
-
- // initialize OmStorage, save om Cert and CA Certs to disk
- OMStorage omStore = new OMStorage(conf);
- omStore.setClusterId(clusterId);
- omStore.setOmId(omId);
-
- // Prepare the certificates for OM before OM start
- SecurityConfig securityConfig = new SecurityConfig(conf);
- CertificateClient scmCertClient = scm.getScmCertificateClient();
- CertificateCodec certCodec = new CertificateCodec(securityConfig, "om");
- X509Certificate scmCert = scmCertClient.getCertificate();
- X509Certificate rootCert = scmCertClient.getCACertificate();
- X509CertificateHolder certHolder = generateX509CertHolder(conf, keyPair,
- new KeyPair(scmCertClient.getPublicKey(),
- scmCertClient.getPrivateKey()), scmCert,
- Duration.ofSeconds(certLifetime),
- InetAddress.getLocalHost().getCanonicalHostName(), clusterId);
- String certId = certHolder.getSerialNumber().toString();
- certCodec.writeCertificate(certHolder);
- certCodec.writeCertificate(CertificateCodec.getCertificateHolder(scmCert),
- String.format(DefaultCertificateClient.CERT_FILE_NAME_FORMAT,
- CAType.SUBORDINATE.getFileNamePrefix() +
- scmCert.getSerialNumber().toString()));
- certCodec.writeCertificate(CertificateCodec.getCertificateHolder(
- scmCertClient.getCACertificate()),
- String.format(DefaultCertificateClient.CERT_FILE_NAME_FORMAT,
- CAType.ROOT.getFileNamePrefix() +
- rootCert.getSerialNumber().toString()));
- omStore.setOmCertSerialId(certId);
- omStore.initialize();
-
- conf.setBoolean(HDDS_GRPC_TLS_ENABLED, true);
- conf.setBoolean(OZONE_OM_S3_GPRC_SERVER_ENABLED, true);
- conf.setBoolean(HddsConfigKeys.HDDS_GRPC_TLS_TEST_CERT, true);
- OzoneManager.setTestSecureOmFlag(true);
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- // In this process, SCM has already login using Kerberos. So pass
- // specific UGI to DefaultCertificateClient and OzoneManager to avoid
- // conflict with SCM procedure.
- DefaultCertificateClient.setUgi(ugi);
- OzoneManager.setUgi(ugi);
- om = OzoneManager.createOm(conf);
- om.start();
-
- CertificateClient omCertClient = om.getCertificateClient();
- X509Certificate omCert = omCertClient.getCertificate();
- X509Certificate caCert = omCertClient.getCACertificate();
- X509Certificate rootCaCert = omCertClient.getRootCACertificate();
- List certList = new ArrayList<>();
- certList.add(caCert);
- certList.add(rootCaCert);
- // set certificates in GrpcOmTransport
- GrpcOmTransport.setCaCerts(certList);
-
- GenericTestUtils.waitFor(() -> om.isLeaderReady(), 500, 10000);
- String transportCls = GrpcOmTransportFactory.class.getName();
- conf.set(OZONE_OM_TRANSPORT_CLASS, transportCls);
- try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) {
-
- ServiceInfoEx serviceInfoEx = client.getObjectStore()
- .getClientProxy().getOzoneManagerClient().getServiceInfo();
- Assert.assertTrue(serviceInfoEx.getCaCertificate().equals(
- CertificateCodec.getPEMEncodedString(caCert)));
-
- // Wait for OM certificate to renewed
- GenericTestUtils.waitFor(() ->
- !omCert.getSerialNumber().toString().equals(
- omCertClient.getCertificate().getSerialNumber().toString()),
- 500, certLifetime * 1000);
-
- // rerun the command using old client, it should succeed
- serviceInfoEx = client.getObjectStore()
- .getClientProxy().getOzoneManagerClient().getServiceInfo();
- Assert.assertTrue(serviceInfoEx.getCaCertificate().equals(
- CertificateCodec.getPEMEncodedString(caCert)));
- }
-
- // get new client, it should succeed.
- try {
- OzoneClient client1 = OzoneClientFactory.getRpcClient(conf);
- client1.close();
- } catch (Exception e) {
- System.out.println("OzoneClientFactory.getRpcClient failed for " +
- e.getMessage());
- fail("Create client should succeed for certificate is renewed");
- }
-
- // Wait for old OM certificate to expire
- GenericTestUtils.waitFor(() -> omCert.getNotAfter().before(new Date()),
- 500, certLifetime * 1000);
- // get new client, it should succeed too.
- try {
- OzoneClient client1 = OzoneClientFactory.getRpcClient(conf);
- client1.close();
- } catch (Exception e) {
- System.out.println("OzoneClientFactory.getRpcClient failed for " +
- e.getMessage());
- fail("Create client should succeed for certificate is renewed");
- }
- } finally {
- DefaultCertificateClient.setUgi(null);
- OzoneManager.setUgi(null);
- GrpcOmTransport.setCaCerts(null);
- }
- }
-
public void validateCertificate(X509Certificate cert) throws Exception {
// Assert that we indeed have a self signed certificate.
@@ -1451,37 +1232,4 @@ public final class TestSecureOzoneCluster {
.setScmID("test")
.build();
}
-
- private static X509CertificateHolder generateX509CertHolder(
- OzoneConfiguration conf, KeyPair keyPair, KeyPair rootKeyPair,
- X509Certificate rootCert, Duration certLifetime, String subject,
- String clusterId) throws Exception {
- // Generate normal certificate, signed by RootCA certificate
- SecurityConfig secConfig = new SecurityConfig(conf);
- DefaultApprover approver = new DefaultApprover(new DefaultProfile(),
- secConfig);
-
- CertificateSignRequest.Builder csrBuilder =
- new CertificateSignRequest.Builder();
- // Get host name.
- csrBuilder.setKey(keyPair)
- .setConfiguration(conf)
- .setScmID("test")
- .setClusterID(clusterId)
- .setSubject(subject)
- .setDigitalSignature(true)
- .setDigitalEncryption(true);
-
- LocalDateTime start = LocalDateTime.now();
- String certDuration = conf.get(HDDS_X509_DEFAULT_DURATION,
- HDDS_X509_DEFAULT_DURATION_DEFAULT);
- X509CertificateHolder certificateHolder =
- approver.sign(secConfig, rootKeyPair.getPrivate(),
- new X509CertificateHolder(rootCert.getEncoded()),
- Date.from(start.atZone(ZoneId.systemDefault()).toInstant()),
- Date.from(start.plus(Duration.parse(certDuration))
- .atZone(ZoneId.systemDefault()).toInstant()),
- csrBuilder.build(), "test", clusterId);
- return certificateHolder;
- }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/SecretKeyTestClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/SecretKeyTestClient.java
new file mode 100644
index 0000000000..8742560e25
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/SecretKeyTestClient.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.client;
+
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Test implementation of {@link SecretKeyClient}.
+ */
+public class SecretKeyTestClient implements SecretKeyClient {
+ private final Map<UUID, ManagedSecretKey> keysMap = new HashMap<>();
+ private ManagedSecretKey current;
+
+ public SecretKeyTestClient() {
+ rotate();
+ }
+
+ public void rotate() {
+ this.current = generateKey();
+ keysMap.put(current.getId(), current);
+ }
+
+ @Override
+ public ManagedSecretKey getCurrentSecretKey() {
+ return current;
+ }
+
+ @Override
+ public ManagedSecretKey getSecretKey(UUID id) {
+ return keysMap.get(id);
+ }
+
+ private ManagedSecretKey generateKey() {
+ KeyGenerator keyGen = null;
+ try {
+ keyGen = KeyGenerator.getInstance("HmacSHA256");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("Should never happen", e);
+ }
+ SecretKey secretKey = keyGen.generateKey();
+ return new ManagedSecretKey(
+ UUID.randomUUID(),
+ Instant.now(),
+ Instant.now().plus(Duration.ofHours(1)),
+ secretKey
+ );
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
index e925a9cbd3..e87e3edc9c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.TestHelper;
@@ -114,6 +115,7 @@ public class TestContainerStateMachine {
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1)
.setHbInterval(200)
.setCertificateClient(new CertificateClientTestImpl(conf))
+ .setSecretKeyClient(new SecretKeyTestClient())
.build();
cluster.setWaitForClusterToBeReadyTimeout(300000);
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java
index 146fffa98a..cd76ee71d4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
@@ -117,6 +118,7 @@ public class TestContainerStateMachineFlushDelay {
.setStreamBufferSizeUnit(StorageUnit.BYTES)
.setHbInterval(200)
.setCertificateClient(new CertificateClientTestImpl(conf))
+ .setSecretKeyClient(new SecretKeyTestClient())
.build();
cluster.waitForClusterToBeReady();
cluster.getOzoneManager().startSecretManager();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
index 8d8667051f..23971fcf18 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.hdds.scm.storage.MultipartInputStream;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -153,6 +154,7 @@ public class TestOzoneAtRestEncryption {
.setChunkSize(CHUNK_SIZE)
.setStreamBufferSizeUnit(StorageUnit.BYTES)
.setCertificateClient(certificateClientTest)
+ .setSecretKeyClient(new SecretKeyTestClient())
.build();
cluster.getOzoneManager().startSecretManager();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
index b8fc491769..353023f186 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
@@ -43,6 +42,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -62,7 +62,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRespo
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.LambdaTestUtils;
@@ -97,7 +96,6 @@ public class TestSecureOzoneRpcClient extends TestOzoneRpcClient {
private static final String CLUSTER_ID = UUID.randomUUID().toString();
private static File testDir;
private static OzoneConfiguration conf;
- private static OzoneBlockTokenSecretManager secretManager;
/**
* Create a MiniOzoneCluster for testing.
@@ -131,10 +129,8 @@ public class TestSecureOzoneRpcClient extends TestOzoneRpcClient {
.setScmId(SCM_ID)
.setClusterId(CLUSTER_ID)
.setCertificateClient(certificateClientTest)
+ .setSecretKeyClient(new SecretKeyTestClient())
.build();
- secretManager = new OzoneBlockTokenSecretManager(new SecurityConfig(conf),
- 60 * 60);
- secretManager.start(certificateClientTest);
cluster.getOzoneManager().startSecretManager();
cluster.waitForClusterToBeReady();
ozClient = OzoneClientFactory.getRpcClient(conf);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 52aaa6a9c5..6bf927d629 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -80,7 +80,7 @@ public class TestOzoneContainer {
DatanodeStateMachine dsm = Mockito.mock(DatanodeStateMachine.class);
Mockito.when(dsm.getDatanodeDetails()).thenReturn(datanodeDetails);
Mockito.when(context.getParent()).thenReturn(dsm);
- container = new OzoneContainer(datanodeDetails, conf, context, null);
+ container = new OzoneContainer(datanodeDetails, conf, context);
//Set clusterId and manually start ozone container.
container.start(UUID.randomUUID().toString());
@@ -113,8 +113,7 @@ public class TestOzoneContainer {
DatanodeStateMachine dsm = Mockito.mock(DatanodeStateMachine.class);
Mockito.when(dsm.getDatanodeDetails()).thenReturn(datanodeDetails);
Mockito.when(context.getParent()).thenReturn(dsm);
- container = new OzoneContainer(datanodeDetails, conf,
- context, null);
+ container = new OzoneContainer(datanodeDetails, conf, context);
String clusterId = UUID.randomUUID().toString();
container.start(clusterId);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
index 22d55992cf..5ffe4037f7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
@@ -27,12 +27,13 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerC
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
@@ -99,6 +100,7 @@ public class TestOzoneContainerWithTLS {
private OzoneConfiguration conf;
private ContainerTokenSecretManager secretManager;
private CertificateClientTestImpl caClient;
+ private SecretKeyClient secretKeyClient;
private boolean containerTokenEnabled;
private int certLifetime = 15 * 1000; // 15s
@@ -144,8 +146,9 @@ public class TestOzoneContainerWithTLS {
TimeUnit.MILLISECONDS);
caClient = new CertificateClientTestImpl(conf);
- secretManager = new ContainerTokenSecretManager(new SecurityConfig(conf),
- expiryTime);
+ secretKeyClient = new SecretKeyTestClient();
+ secretManager = new ContainerTokenSecretManager(expiryTime,
+ secretKeyClient);
}
@Test(expected = CertificateExpiredException.class)
@@ -177,7 +180,8 @@ public class TestOzoneContainerWithTLS {
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
- container = new OzoneContainer(dn, conf, getContext(dn), caClient);
+ container = new OzoneContainer(dn, conf, getContext(dn), caClient,
+ secretKeyClient);
//Set scmId and manually start ozone container.
container.start(UUID.randomUUID().toString());
@@ -185,7 +189,6 @@ public class TestOzoneContainerWithTLS {
Collections.singletonList(caClient.getCACertificate()))) {
if (containerTokenEnabled) {
- secretManager.start(caClient);
client.connect();
createSecureContainer(client, containerId,
secretManager.generateToken(
@@ -217,15 +220,12 @@ public class TestOzoneContainerWithTLS {
OzoneContainer container = null;
try {
- container = new OzoneContainer(dn, conf, getContext(dn), caClient);
+ container = new OzoneContainer(dn, conf, getContext(dn), caClient,
+ secretKeyClient);
// Set scmId and manually start ozone container.
container.start(UUID.randomUUID().toString());
- if (containerTokenEnabled) {
- secretManager.start(caClient);
- }
-
// Create containers
long containerId = ContainerTestHelper.getTestContainerID();
List<Long> containerIdList = new ArrayList<>();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
index f51cf38de5..85149b69ad 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
@@ -19,23 +19,25 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.security.UserGroupInformation;
@@ -93,6 +95,7 @@ public class TestSecureOzoneContainer {
private final boolean hasToken;
private final boolean tokenExpired;
private CertificateClientTestImpl caClient;
+ private SecretKeyClient secretKeyClient;
private ContainerTokenSecretManager secretManager;
public TestSecureOzoneContainer(Boolean requireToken,
@@ -122,8 +125,9 @@ public class TestSecureOzoneContainer {
conf.set(OZONE_METADATA_DIRS, ozoneMetaPath);
secConfig = new SecurityConfig(conf);
caClient = new CertificateClientTestImpl(conf);
+ secretKeyClient = new SecretKeyTestClient();
secretManager = new ContainerTokenSecretManager(
- new SecurityConfig(conf), TimeUnit.DAYS.toMillis(1));
+ TimeUnit.DAYS.toMillis(1), secretKeyClient);
}
@Test
@@ -146,7 +150,8 @@ public class TestSecureOzoneContainer {
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
- container = new OzoneContainer(dn, conf, getContext(dn), caClient);
+ container = new OzoneContainer(dn, conf, getContext(dn), caClient,
+ secretKeyClient);
//Set scmId and manually start ozone container.
container.start(UUID.randomUUID().toString());
@@ -159,7 +164,6 @@ public class TestSecureOzoneContainer {
port = secConfig.getConfiguration().getInt(OzoneConfigKeys
.DFS_CONTAINER_IPC_PORT, DFS_CONTAINER_IPC_PORT_DEFAULT);
}
- secretManager.start(caClient);
ugi.doAs((PrivilegedAction<Void>) () -> {
try (XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf)) {
@@ -172,7 +176,7 @@ public class TestSecureOzoneContainer {
: Instant.now().plusSeconds(3600);
ContainerTokenIdentifier tokenIdentifier =
new ContainerTokenIdentifier(user, containerID,
- caClient.getCertificate().getSerialNumber().toString(),
+ secretKeyClient.getCurrentSecretKey().getId(),
expiryDate);
token = secretManager.generateToken(tokenIdentifier);
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
index 1890a95daa..3bd51c35db 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.RatisTestHelper;
+import org.apache.hadoop.ozone.client.SecretKeyTestClient;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
@@ -118,6 +120,7 @@ public class TestSecureContainerServer {
= GenericTestUtils.getTestDir("dfs").getAbsolutePath() + File.separator;
private static final OzoneConfiguration CONF = new OzoneConfiguration();
private static CertificateClientTestImpl caClient;
+ private static SecretKeyClient secretKeyClient;
private static OzoneBlockTokenSecretManager blockTokenSecretManager;
private static ContainerTokenSecretManager containerTokenSecretManager;
@@ -129,17 +132,15 @@ public class TestSecureContainerServer {
CONF.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
CONF.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);
caClient = new CertificateClientTestImpl(CONF);
+ secretKeyClient = new SecretKeyTestClient();
- SecurityConfig secConf = new SecurityConfig(CONF);
long tokenLifetime = TimeUnit.HOURS.toMillis(1);
- blockTokenSecretManager = new OzoneBlockTokenSecretManager(
- secConf, tokenLifetime);
- blockTokenSecretManager.start(caClient);
+ blockTokenSecretManager = new OzoneBlockTokenSecretManager(tokenLifetime,
+ secretKeyClient);
containerTokenSecretManager = new ContainerTokenSecretManager(
- secConf, tokenLifetime);
- containerTokenSecretManager.start(caClient);
+ tokenLifetime, secretKeyClient);
}
@AfterClass
@@ -149,8 +150,6 @@ public class TestSecureContainerServer {
@After
public void cleanUp() throws IOException {
- containerTokenSecretManager.stop();
- blockTokenSecretManager.stop();
FileUtils.deleteQuietly(new File(CONF.get(HDDS_DATANODE_DIR_KEY)));
}
@@ -194,7 +193,7 @@ public class TestSecureContainerServer {
}
HddsDispatcher hddsDispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, context, metrics,
- TokenVerifier.create(new SecurityConfig((conf)), caClient));
+ TokenVerifier.create(new SecurityConfig(conf), secretKeyClient));
hddsDispatcher.setClusterId(scmId.toString());
return hddsDispatcher;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 84a51bea69..bcdf751f4c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -102,6 +102,10 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
+import org.apache.hadoop.hdds.security.OzoneSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
+import org.apache.hadoop.hdds.security.symmetric.DefaultSecretKeySignerClient;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.ozone.security.OMCertificateClient;
@@ -169,7 +173,6 @@ import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolClientSideImpl;
import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolPB;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.common.ha.ratis.RatisSnapshotInfo;
-import org.apache.hadoop.hdds.security.OzoneSecurityException;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
@@ -189,7 +192,6 @@ import org.apache.hadoop.ozone.protocolPB.OMInterServiceProtocolServerSideImpl;
import org.apache.hadoop.ozone.protocolPB.OMAdminProtocolServerSideImpl;
import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo;
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
-import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager;
import org.apache.hadoop.ozone.security.OzoneDelegationTokenSecretManager;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -335,6 +337,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private OzoneDelegationTokenSecretManager delegationTokenMgr;
private OzoneBlockTokenSecretManager blockTokenMgr;
private CertificateClient certClient;
+ private SecretKeySignerClient secretKeyClient;
private String caCertPem = null;
private List<String> caCertPemList = new ArrayList<>();
private final Text omRpcAddressTxt;
@@ -625,9 +628,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
certClient = new OMCertificateClient(secConfig, omStorage,
scmInfo == null ? null : scmInfo.getScmId(), this::saveNewCertId,
this::terminateOM);
+ secretKeyClient = DefaultSecretKeySignerClient.create(conf);
}
if (secConfig.isBlockTokenEnabled()) {
- blockTokenMgr = createBlockTokenSecretManager(configuration);
+ blockTokenMgr = createBlockTokenSecretManager();
}
// Enable S3 multi-tenancy if config keys are set
@@ -1034,38 +1038,18 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
.build();
}
- private OzoneBlockTokenSecretManager createBlockTokenSecretManager(
- OzoneConfiguration conf) {
-
- long expiryTime = conf.getTimeDuration(
+ private OzoneBlockTokenSecretManager createBlockTokenSecretManager() {
+ long expiryTime = configuration.getTimeDuration(
HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME,
HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT,
TimeUnit.MILLISECONDS);
- long certificateGracePeriod = Duration.parse(
- conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION,
- HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis();
- if (expiryTime > certificateGracePeriod) {
- throw new IllegalArgumentException("Certificate grace period " +
- HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION +
- " should be greater than maximum block token lifetime " +
- HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME);
- }
- // TODO: Pass OM cert serial ID.
- if (testSecureOmFlag) {
- return new OzoneBlockTokenSecretManager(secConfig, expiryTime);
- }
- Objects.requireNonNull(certClient);
- return new OzoneBlockTokenSecretManager(secConfig, expiryTime);
+ return new OzoneBlockTokenSecretManager(expiryTime, secretKeyClient);
}
private void stopSecretManager() {
- if (blockTokenMgr != null) {
- LOG.info("Stopping OM block token manager.");
- try {
- blockTokenMgr.stop();
- } catch (IOException e) {
- LOG.error("Failed to stop block token manager", e);
- }
+ if (secretKeyClient != null) {
+ LOG.info("Stopping secret key client.");
+ secretKeyClient.stop();
}
if (delegationTokenMgr != null) {
@@ -1086,13 +1070,13 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
LOG.error("Unable to read key pair for OM.", e);
throw new UncheckedIOException(e);
}
+
if (secConfig.isBlockTokenEnabled() && blockTokenMgr != null) {
+ LOG.info("Starting secret key client.");
try {
- LOG.info("Starting OM block token secret manager");
- blockTokenMgr.start(certClient);
+ secretKeyClient.start(configuration);
} catch (IOException e) {
- // Unable to start secret manager.
- LOG.error("Error starting block token secret manager.", e);
+ LOG.error("Unable to initialize secret key.", e);
throw new UncheckedIOException(e);
}
}
@@ -1120,6 +1104,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
certClient = newClient;
}
+ /**
+ * For testing purpose only. This allows testing token in integration test
+ * without fully setting up a working secure cluster.
+ */
+ @VisibleForTesting
+ public void setSecretKeyClient(
+ SecretKeySignerClient secretKeyClient) {
+ this.secretKeyClient = secretKeyClient;
+ blockTokenMgr.setSecretKeyClient(secretKeyClient);
+ }
+
/**
* Login OM service user if security and Kerberos are enabled.
*
@@ -2283,8 +2278,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private void startSecretManagerIfNecessary() {
boolean shouldRun = isOzoneSecurityEnabled();
if (shouldRun) {
- boolean running = delegationTokenMgr.isRunning()
- && blockTokenMgr.isRunning();
+ boolean running = delegationTokenMgr.isRunning();
if (!running) {
startSecretManager();
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
deleted file mode 100644
index 3297e3d992..0000000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.security;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.ozone.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.crypto.KeyGenerator;
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import java.io.File;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.Signature;
-import java.security.SignatureException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test class for OzoneManagerDelegationToken.
- */
-public class TestOzoneManagerBlockToken {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TestOzoneManagerBlockToken.class);
- private static final String BASEDIR = GenericTestUtils
- .getTempPath(TestOzoneManagerBlockToken.class.getSimpleName());
- private static final String KEYSTORES_DIR =
- new File(BASEDIR).getAbsolutePath();
- private static long expiryTime;
- private static KeyPair keyPair;
- private static X509Certificate cert;
- private static final long MAX_LEN = 1000;
-
- @BeforeClass
- public static void setUp() throws Exception {
- File base = new File(BASEDIR);
- FileUtil.fullyDelete(base);
- base.mkdirs();
- expiryTime = Time.monotonicNow() + 60 * 60 * 24;
-
- // Create Ozone Master key pair.
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- // Create Ozone Master certificate (SCM CA issued cert) and key store.
- cert = KeyStoreTestUtil
- .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
- }
-
- @After
- public void cleanUp() {
- }
-
- @Test
- public void testSignToken() throws GeneralSecurityException, IOException {
- String keystore = new File(KEYSTORES_DIR, "keystore.jks")
- .getAbsolutePath();
- String truststore = new File(KEYSTORES_DIR, "truststore.jks")
- .getAbsolutePath();
- String trustPassword = "trustPass";
- String keyStorePassword = "keyStorePass";
- String keyPassword = "keyPass";
-
-
- KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
- "OzoneMaster", keyPair.getPrivate(), cert);
-
- // Create trust store and put the certificate in the trust store
- Map<String, X509Certificate> certs = Collections.singletonMap("server",
- cert);
- KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
-
- // Sign the OzoneMaster Token with Ozone Master private key
- PrivateKey privateKey = keyPair.getPrivate();
- OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
- "testUser", "84940",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
- byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
-
- // Verify a valid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
- LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
-
- // Verify an invalid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- tokenId = new OzoneBlockTokenIdentifier("", "",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
- LOG.info("Unsigned token {} is {}", tokenId,
- verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
-
- }
-
- public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- PrivateKey privateKey) throws NoSuchAlgorithmException,
- InvalidKeyException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initSign(privateKey);
- rsaSignature.update(tokenId.getBytes());
- byte[] signature = rsaSignature.sign();
- return signature;
- }
-
- public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- byte[] signature, Certificate certificate) throws InvalidKeyException,
- NoSuchAlgorithmException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initVerify(certificate);
- rsaSignature.update(tokenId.getBytes());
- boolean isValid = rsaSignature.verify(signature);
- return isValid;
- }
-
- private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
- Mac mac, SecretKey key) {
- try {
- mac.init(key);
- } catch (InvalidKeyException ike) {
- throw new IllegalArgumentException("Invalid key to HMAC computation",
- ike);
- }
- return mac.doFinal(identifier.getBytes());
- }
-
- OzoneBlockTokenIdentifier generateTestToken() {
- return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
- RandomStringUtils.randomAlphabetic(5),
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
- }
-
- @Test
- public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
- CertificateEncodingException, NoSuchProviderException,
- InvalidKeyException, SignatureException {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordAsym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
-
- // Create Ozone Master certificate (SCM CA issued cert) and key store
- X509Certificate omCert;
- omCert = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
- kp, 30, "SHA256withRSA");
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordAsym.add(
- signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
- duration / testTokenCount);
-
- startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i), omCert);
- }
- duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
- + "is {} ns", duration / testTokenCount);
- }
-
- @Test
- public void testSymmetricTokenPerf() {
- String hmacSHA1 = "HmacSHA1";
- String hmacSHA256 = "HmacSHA256";
-
- testSymmetricTokenPerfHelper(hmacSHA1, 64);
- testSymmetricTokenPerfHelper(hmacSHA256, 1024);
- }
-
- public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordSym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyGenerator keyGen;
- try {
- keyGen = KeyGenerator.getInstance(hmacAlgorithm);
- keyGen.init(keyLen);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- Mac mac;
- try {
- mac = Mac.getInstance(hmacAlgorithm);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- SecretKey secretKey = keyGen.generateKey();
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordSym.add(
- signTokenSymmetric(tokenIds.get(i), mac, secretKey));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
- hmacAlgorithm, keyLen, duration / testTokenCount);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/05: HDDS-7734. Implement symmetric SecretKeys lifescycle management in SCM (#4194)
Posted by du...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
duong pushed a commit to branch HDDS-7733-Symmetric-Tokens
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 43aaa9545a6814c99a2ea5bc6af31133ebefc34c
Author: Duong Nguyen <du...@gmail.com>
AuthorDate: Fri Mar 3 09:01:35 2023 -0800
HDDS-7734. Implement symmetric SecretKeys lifescycle management in SCM (#4194)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 22 +++
.../common/src/main/resources/ozone-default.xml | 48 +++++
.../security/symmetric/LocalSecretKeyStore.java | 199 +++++++++++++++++++
.../hdds/security/symmetric/ManagedSecretKey.java | 122 ++++++++++++
.../hdds/security/symmetric/SecretKeyConfig.java | 99 ++++++++++
.../hdds/security/symmetric/SecretKeyManager.java | 155 +++++++++++++++
.../hdds/security/symmetric/SecretKeyState.java | 52 +++++
.../security/symmetric/SecretKeyStateImpl.java | 108 +++++++++++
.../hdds/security/symmetric/SecretKeyStore.java | 35 ++++
.../hdds/security/symmetric/package-info.java | 63 ++++++
.../symmetric/LocalSecretKeyStoreTest.java | 188 ++++++++++++++++++
.../security/symmetric/SecretKeyManagerTest.java | 215 +++++++++++++++++++++
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../src/main/proto/ScmServerSecurityProtocol.proto | 9 +
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 6 +-
.../hdds/scm/ha/io/ManagedSecretKeyCodec.java | 44 +++++
.../scm/security/ScmSecretKeyStateBuilder.java | 60 ++++++
.../hdds/scm/security/SecretKeyManagerService.java | 159 +++++++++++++++
.../hadoop/hdds/scm/security/package-info.java | 22 +++
.../hdds/scm/server/StorageContainerManager.java | 9 +
.../src/main/compose/ozonesecure-ha/docker-config | 5 +
22 files changed, 1622 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 2c6b3ee62a..4a55d6709b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -216,6 +216,28 @@ public final class HddsConfigKeys {
public static final String HDDS_X509_ROOTCA_PRIVATE_KEY_FILE_DEFAULT =
"";
+ public static final String HDDS_SECRET_KEY_FILE =
+ "hdds.secret.key.file.name";
+ public static final String HDDS_SECRET_KEY_FILE_DEFAULT = "secret_keys.json";
+
+ public static final String HDDS_SECRET_KEY_EXPIRY_DURATION =
+ "hdds.secret.key.expiry.duration";
+ public static final String HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT = "7d";
+
+ public static final String HDDS_SECRET_KEY_ROTATE_DURATION =
+ "hdds.secret.key.rotate.duration";
+ public static final String HDDS_SECRET_KEY_ROTATE_DURATION_DEFAULT = "1d";
+
+ public static final String HDDS_SECRET_KEY_ALGORITHM =
+ "hdds.secret.key.algorithm";
+ public static final String HDDS_SECRET_KEY_ALGORITHM_DEFAULT =
+ "HmacSHA256";
+
+ public static final String HDDS_SECRET_KEY_ROTATE_CHECK_DURATION =
+ "hdds.secret.key.rotate.check.duration";
+ public static final String HDDS_SECRET_KEY_ROTATE_CHECK_DURATION_DEFAULT
+ = "10m";
+
/**
* Do not instantiate.
*/
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 56451ab83f..848b21e707 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -3799,4 +3799,52 @@
Max numbers of keys changed allowed for a snapshot diff job.
</description>
</property>
+
+ <property>
+ <name>hdds.secret.key.file.name</name>
+ <value>secret_keys.json</value>
+ <tag>SCM, SECURITY</tag>
+ <description>
+ Name of file which stores symmetric secret keys for token signatures.
+ </description>
+ </property>
+ <property>
+ <name>hdds.secret.key.expiry.duration</name>
+ <value>7d</value>
+ <tag>SCM, SECURITY</tag>
+ <description>
+ The duration for which symmetric secret keys issued by SCM are valid.
+ This default value, in combination with hdds.secret.key.rotate.duration=1d, results in 7 secret keys (for the
+ last 7 days) are kept valid at any point of time.
+ </description>
+ </property>
+ <property>
+ <name>hdds.secret.key.rotate.duration</name>
+ <value>1d</value>
+ <tag>SCM, SECURITY</tag>
+ <description>
+ The duration that SCM periodically generate a new symmetric secret keys.
+ </description>
+ </property>
+ <property>
+ <name>hdds.secret.key.rotate.check.duration</name>
+ <value>10m</value>
+ <tag>SCM, SECURITY</tag>
+ <description>
+ The duration that SCM periodically checks if it's time to generate new symmetric secret keys.
+ This config has an impact on the practical correctness of secret key expiry and rotation period. For example,
+ if hdds.secret.key.rotate.duration=1d and hdds.secret.key.rotate.check.duration=10m, the actual key rotation
+ will happen each 1d +/- 10m.
+ </description>
+ </property>
+ <property>
+ <name>hdds.secret.key.algorithm</name>
+ <value>HmacSHA256</value>
+ <tag>SCM, SECURITY</tag>
+ <description>
+ The algorithm that SCM uses to generate symmetric secret keys.
+ A valid algorithm is the one supported by KeyGenerator, as described at
+ https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#KeyGenerator.
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStore.java
new file mode 100644
index 0000000000..48cc633b67
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStore.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import com.fasterxml.jackson.databind.MappingIterator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.SequenceWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static java.nio.file.Files.createDirectories;
+import static java.nio.file.Files.createFile;
+import static java.nio.file.Files.exists;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * A {@link SecretKeyStore} that saves and loads SecretKeys from/to a
+ * JSON file on local file system.
+ */
+public class LocalSecretKeyStore implements SecretKeyStore {
+ private static final Set<PosixFilePermission> SECRET_KEYS_PERMISSIONS =
+ newHashSet(OWNER_READ, OWNER_WRITE);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalSecretKeyStore.class);
+
+ private final Path secretKeysFile;
+ private final ObjectMapper mapper;
+
+ public LocalSecretKeyStore(Path secretKeysFile) {
+ this.secretKeysFile = requireNonNull(secretKeysFile);
+ this.mapper = new ObjectMapper()
+ .registerModule(new JavaTimeModule())
+ .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+ }
+
+ @Override
+ public synchronized List<ManagedSecretKey> load() {
+ if (!secretKeysFile.toFile().exists()) {
+ return Collections.emptyList();
+ }
+
+ ObjectReader reader = mapper.readerFor(ManagedSecretKeyDto.class);
+ try (MappingIterator<ManagedSecretKeyDto> iterator =
+ reader.readValues(secretKeysFile.toFile())) {
+ List<ManagedSecretKeyDto> dtos = iterator.readAll();
+ List<ManagedSecretKey> result = dtos.stream()
+ .map(ManagedSecretKeyDto::toObject)
+ .collect(toList());
+ LOG.info("Loaded {} from {}", result, secretKeysFile);
+ return result;
+ } catch (IOException e) {
+ throw new IllegalStateException("Error reading SecretKeys from "
+ + secretKeysFile, e);
+ }
+ }
+
+ @Override
+ public synchronized void save(Collection<ManagedSecretKey> secretKeys) {
+ createSecretKeyFiles();
+
+ List<ManagedSecretKeyDto> dtos = secretKeys.stream()
+ .map(ManagedSecretKeyDto::new)
+ .collect(toList());
+
+ try (SequenceWriter writer =
+ mapper.writer().writeValues(secretKeysFile.toFile())) {
+ writer.init(true);
+ writer.writeAll(dtos);
+ } catch (IOException e) {
+ throw new IllegalStateException("Error saving SecretKeys to file "
+ + secretKeysFile, e);
+ }
+ LOG.info("Saved {} to file {}", secretKeys, secretKeysFile);
+ }
+
+ private void createSecretKeyFiles() {
+ try {
+ if (!exists(secretKeysFile)) {
+ Path parent = secretKeysFile.getParent();
+ if (parent != null && !exists(parent)) {
+ createDirectories(parent);
+ }
+ createFile(secretKeysFile);
+ }
+ Files.setPosixFilePermissions(secretKeysFile, SECRET_KEYS_PERMISSIONS);
+ } catch (IOException e) {
+ throw new IllegalStateException("Error setting secret keys file" +
+ " permission: " + secretKeysFile, e);
+ }
+ }
+
+ /**
+ * Just a simple DTO that allows serializing/deserializing the immutable
+ * {@link ManagedSecretKey} objects.
+ */
+ private static class ManagedSecretKeyDto {
+ private UUID id;
+ private Instant creationTime;
+ private Instant expiryTime;
+ private String algorithm;
+ private byte[] encoded;
+
+ /**
+ * Used by Jackson when deserializing.
+ */
+ ManagedSecretKeyDto() {
+ }
+
+ ManagedSecretKeyDto(ManagedSecretKey object) {
+ id = object.getId();
+ creationTime = object.getCreationTime();
+ expiryTime = object.getExpiryTime();
+ algorithm = object.getSecretKey().getAlgorithm();
+ encoded = object.getSecretKey().getEncoded();
+ }
+
+ public ManagedSecretKey toObject() {
+ SecretKey secretKey = new SecretKeySpec(this.encoded, this.algorithm);
+ return new ManagedSecretKey(id, creationTime,
+ expiryTime, secretKey);
+ }
+
+ public UUID getId() {
+ return id;
+ }
+
+ public void setId(UUID id) {
+ this.id = id;
+ }
+
+ public Instant getCreationTime() {
+ return creationTime;
+ }
+
+ public void setCreationTime(Instant creationTime) {
+ this.creationTime = creationTime;
+ }
+
+ public Instant getExpiryTime() {
+ return expiryTime;
+ }
+
+ public void setExpiryTime(Instant expiryTime) {
+ this.expiryTime = expiryTime;
+ }
+
+ public String getAlgorithm() {
+ return algorithm;
+ }
+
+ public void setAlgorithm(String algorithm) {
+ this.algorithm = algorithm;
+ }
+
+ public byte[] getEncoded() {
+ return encoded;
+ }
+
+ public void setEncoded(byte[] encoded) {
+ this.encoded = encoded;
+ }
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
new file mode 100644
index 0000000000..7e8aaacb48
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.UUID;
+
+/**
+ * Enclosed a symmetric {@link SecretKey} with additional data for life-cycle
+ * management.
+ */
+public final class ManagedSecretKey implements Serializable {
+ private final UUID id;
+ private final Instant creationTime;
+ private final Instant expiryTime;
+ private final SecretKey secretKey;
+
+ public ManagedSecretKey(UUID id,
+ Instant creationTime,
+ Instant expiryTime,
+ SecretKey secretKey) {
+ this.id = id;
+ this.creationTime = creationTime;
+ this.expiryTime = expiryTime;
+ this.secretKey = secretKey;
+ }
+
+ public boolean isExpired() {
+ return expiryTime.isBefore(Instant.now());
+ }
+
+ public UUID getId() {
+ return id;
+ }
+
+ public SecretKey getSecretKey() {
+ return secretKey;
+ }
+
+ public Instant getCreationTime() {
+ return creationTime;
+ }
+
+ public Instant getExpiryTime() {
+ return expiryTime;
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ManagedSecretKey)) {
+ return false;
+ }
+ ManagedSecretKey that = (ManagedSecretKey) obj;
+ return this.id.equals(that.id);
+ }
+
+ @Override
+ public String toString() {
+ return "SecretKey(id = " + id + ", creation at: "
+ + creationTime + ", expire at: " + expiryTime + ")";
+ }
+
+ /**
+ * @return the protobuf message to deserialize this object.
+ */
+ public SCMSecurityProtocolProtos.ManagedSecretKey toProtobuf() {
+ HddsProtos.UUID uuid = HddsProtos.UUID.newBuilder()
+ .setMostSigBits(this.id.getMostSignificantBits())
+ .setLeastSigBits(this.id.getLeastSignificantBits())
+ .build();
+
+ return SCMSecurityProtocolProtos.ManagedSecretKey.newBuilder()
+ .setId(uuid)
+ .setCreationTime(this.creationTime.toEpochMilli())
+ .setExpiryTime(this.expiryTime.toEpochMilli())
+ .setAlgorithm(this.secretKey.getAlgorithm())
+ .setEncoded(ByteString.copyFrom(this.secretKey.getEncoded()))
+ .build();
+ }
+
+ /**
+ * Create a {@link ManagedSecretKey} from a given protobuf message.
+ */
+ public static ManagedSecretKey fromProtobuf(
+ SCMSecurityProtocolProtos.ManagedSecretKey message) {
+ UUID id = new UUID(message.getId().getMostSigBits(),
+ message.getId().getLeastSigBits());
+ Instant creationTime = Instant.ofEpochMilli(message.getCreationTime());
+ Instant expiryTime = Instant.ofEpochMilli(message.getExpiryTime());
+ SecretKey secretKey = new SecretKeySpec(message.getEncoded().toByteArray(),
+ message.getAlgorithm());
+ return new ManagedSecretKey(id, creationTime, expiryTime, secretKey);
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java
new file mode 100644
index 0000000000..f2a9181051
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ALGORITHM_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_FILE;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_FILE_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_CHECK_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_CHECK_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * Configurations related to SecretKeys lifecycle management.
+ */
+public class SecretKeyConfig {
+ private final Path localSecretKeyFile;
+ private final Duration rotateDuration;
+ private final Duration expiryDuration;
+ private final String algorithm;
+ private final Duration rotationCheckDuration;
+
+ public SecretKeyConfig(ConfigurationSource conf, String component) {
+ String metadataDir = conf.get(HDDS_METADATA_DIR_NAME,
+ conf.get(OZONE_METADATA_DIRS));
+ String keyDir = conf.get(HDDS_KEY_DIR_NAME, HDDS_KEY_DIR_NAME_DEFAULT);
+ String fileName = conf.get(HDDS_SECRET_KEY_FILE,
+ HDDS_SECRET_KEY_FILE_DEFAULT);
+ localSecretKeyFile = Paths.get(metadataDir, component, keyDir, fileName);
+
+ long rotateDurationInMs = conf.getTimeDuration(
+ HDDS_SECRET_KEY_ROTATE_DURATION,
+ HDDS_SECRET_KEY_ROTATE_DURATION_DEFAULT, TimeUnit.MILLISECONDS);
+ this.rotateDuration = Duration.ofMillis(rotateDurationInMs);
+
+ long expiryDurationInMs = conf.getTimeDuration(
+ HDDS_SECRET_KEY_EXPIRY_DURATION,
+ HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT, TimeUnit.MILLISECONDS);
+ this.expiryDuration = Duration.ofMillis(expiryDurationInMs);
+
+ this.algorithm = conf.get(HDDS_SECRET_KEY_ALGORITHM,
+ HDDS_SECRET_KEY_ALGORITHM_DEFAULT);
+
+ long rotationCheckInMs = conf.getTimeDuration(
+ HDDS_SECRET_KEY_ROTATE_CHECK_DURATION,
+ HDDS_SECRET_KEY_ROTATE_CHECK_DURATION_DEFAULT, TimeUnit.MILLISECONDS);
+ this.rotationCheckDuration = Duration.ofMillis(rotationCheckInMs);
+ }
+
+ public Path getLocalSecretKeyFile() {
+ return localSecretKeyFile;
+ }
+
+ public Duration getRotateDuration() {
+ return rotateDuration;
+ }
+
+ public Duration getExpiryDuration() {
+ return expiryDuration;
+ }
+
+ public String getAlgorithm() {
+ return algorithm;
+ }
+
+ public Duration getRotationCheckDuration() {
+ return rotationCheckDuration;
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
new file mode 100644
index 0000000000..0dc5bf8902
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.KeyGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+import static java.time.Duration.between;
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * This component manages symmetric SecretKey life-cycle, including generation,
+ * rotation and destruction.
+ */
+public class SecretKeyManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SecretKeyManager.class);
+
+ private final SecretKeyState state;
+ private final Duration rotationDuration;
+ private final Duration validityDuration;
+ private final SecretKeyStore keyStore;
+
+ private final KeyGenerator keyGenerator;
+
+ public SecretKeyManager(SecretKeyState state,
+ SecretKeyStore keyStore,
+ Duration rotationDuration,
+ Duration validityDuration,
+ String algorithm) {
+ this.state = requireNonNull(state);
+ this.rotationDuration = requireNonNull(rotationDuration);
+ this.validityDuration = requireNonNull(validityDuration);
+ this.keyStore = requireNonNull(keyStore);
+ this.keyGenerator = createKeyGenerator(algorithm);
+ }
+
+ public SecretKeyManager(SecretKeyState state,
+ SecretKeyStore keyStore,
+ SecretKeyConfig config) {
+ this(state, keyStore, config.getRotateDuration(),
+ config.getExpiryDuration(), config.getAlgorithm());
+ }
+
+ /**
+ * If the SecretKey state is not initialized, initialize it from by loading
+ * SecretKeys from local file, or generate new keys if the file doesn't
+ * exist.
+ */
+ public synchronized void checkAndInitialize() throws TimeoutException {
+ if (isInitialized()) {
+ return;
+ }
+
+ LOG.info("Initializing SecretKeys.");
+
+ // Load and filter expired keys.
+ List<ManagedSecretKey> allKeys = keyStore.load()
+ .stream()
+ .filter(x -> !x.isExpired())
+ .collect(toList());
+
+ if (allKeys.isEmpty()) {
+ // if no valid key present , generate new key as the current key.
+ // This happens at first start or restart after being down for
+ // a significant time.
+ ManagedSecretKey newKey = generateSecretKey();
+ allKeys.add(newKey);
+ LOG.info("No valid key has been loaded. " +
+ "A new key is generated: {}", newKey);
+ } else {
+ LOG.info("Keys reloaded: {}", allKeys);
+ }
+
+ state.updateKeys(allKeys);
+ }
+
+ public boolean isInitialized() {
+ return state.getCurrentKey() != null;
+ }
+
+ /**
+ * Check and rotate the keys.
+ *
+ * @return true if rotation actually happens, false if it doesn't.
+ */
+ public synchronized boolean checkAndRotate() throws TimeoutException {
+ // Initialize the state if it's not initialized already.
+ checkAndInitialize();
+
+ ManagedSecretKey currentKey = state.getCurrentKey();
+ if (shouldRotate(currentKey)) {
+ ManagedSecretKey newCurrentKey = generateSecretKey();
+ List<ManagedSecretKey> updatedKeys = state.getSortedKeys()
+ .stream().filter(x -> !x.isExpired())
+ .collect(toList());
+ updatedKeys.add(newCurrentKey);
+
+ LOG.info("SecretKey rotation is happening, new key generated {}",
+ newCurrentKey);
+ state.updateKeys(updatedKeys);
+ return true;
+ }
+ return false;
+ }
+
+ private boolean shouldRotate(ManagedSecretKey currentKey) {
+ Duration established = between(currentKey.getCreationTime(), Instant.now());
+ return established.compareTo(rotationDuration) >= 0;
+ }
+
+ private ManagedSecretKey generateSecretKey() {
+ Instant now = Instant.now();
+ return new ManagedSecretKey(
+ UUID.randomUUID(),
+ now,
+ now.plus(validityDuration),
+ keyGenerator.generateKey()
+ );
+ }
+
+ private KeyGenerator createKeyGenerator(String algorithm) {
+ try {
+ return KeyGenerator.getInstance(algorithm);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalArgumentException("Error creating KeyGenerator for " +
+ "algorithm " + algorithm, e);
+ }
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
new file mode 100644
index 0000000000..7be70b4b02
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.apache.hadoop.hdds.scm.metadata.Replicate;
+
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * This component holds the state of managed SecretKeys, including the
+ * current key and all active keys.
+ */
+public interface SecretKeyState {
+ /**
+ * Get the current active key, which is used for signing tokens. This is
+ * also the latest key managed by this state.
+ *
+ * @return the current active key, or null if the state is not initialized.
+ */
+ ManagedSecretKey getCurrentKey();
+
+ /**
+ * Get the keys that managed by this manager.
+ * The returned keys are sorted by creation time, in the order of latest
+ * to oldest.
+ */
+ List<ManagedSecretKey> getSortedKeys();
+
+ /**
+ * Update the SecretKeys.
+ * This method replicates SecretKeys across all SCM instances.
+ */
+ @Replicate
+ void updateKeys(List<ManagedSecretKey> newKeys) throws TimeoutException;
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
new file mode 100644
index 0000000000..d5c886fd99
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static java.util.Comparator.comparing;
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Default implementation of {@link SecretKeyState}.
+ */
+public final class SecretKeyStateImpl implements SecretKeyState {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SecretKeyStateImpl.class);
+
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private List<ManagedSecretKey> sortedKeys;
+ private ManagedSecretKey currentKey;
+
+ private final SecretKeyStore keyStore;
+
+ /**
+ * Instantiate a state with no keys. This state object needs to be backed by
+ * a proper replication proxy so that the @Replication method works.
+ */
+ public SecretKeyStateImpl(SecretKeyStore keyStore) {
+ this.keyStore = requireNonNull(keyStore);
+ }
+
+ /**
+ * Get the current active key, which is used for signing tokens. This is
+ * also the latest key managed by this state.
+ */
+ @Override
+ public ManagedSecretKey getCurrentKey() {
+ lock.readLock().lock();
+ try {
+ return currentKey;
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Get the keys that managed by this manager.
+ * The returned keys are sorted by creation time, in the order of latest
+ * to oldest.
+ */
+ @Override
+ public List<ManagedSecretKey> getSortedKeys() {
+ lock.readLock().lock();
+ try {
+ return sortedKeys;
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Update the SecretKeys.
+ * This method replicates SecretKeys across all SCM instances.
+ */
+ @Override
+ public void updateKeys(List<ManagedSecretKey> newKeys) {
+ LOG.info("Updating keys with {}", newKeys);
+ lock.writeLock().lock();
+ try {
+ // Store sorted keys in order of latest to oldest and make it
+ // immutable so that can be used to answer queries directly.
+ sortedKeys = Collections.unmodifiableList(
+ newKeys.stream()
+ .sorted(comparing(ManagedSecretKey::getCreationTime).reversed())
+ .collect(toList())
+ );
+ currentKey = sortedKeys.get(0);
+ LOG.info("Current key updated {}", currentKey);
+ keyStore.save(sortedKeys);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStore.java
new file mode 100644
index 0000000000..c851c3683d
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStore.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Interface for SecretKey storage component, which is responsible for saving
+ * the SecretKeys states persistently to ensure they're not lost during
+ * restarts.
+ *
+ * This interface allows new persistent storage to be plugged in easily.
+ */
+public interface SecretKeyStore {
+ List<ManagedSecretKey> load();
+
+ void save(Collection<ManagedSecretKey> secretKeys);
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/package-info.java
new file mode 100644
index 0000000000..2997fe0a26
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/package-info.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * In secure mode, Ozone uses symmetric key algorithm to sign all its issued
+ * tokens, such as block or container tokens. These tokens are then verified
+ * by datanodes to ensure their authenticity and integrity.
+ * <p/>
+ *
+ * That process requires symmetric {@link javax.crypto.SecretKey} to be
+ * generated, managed, and distributed to different Ozone components.
+ * For example, the token signer (Ozone Manager and SCM) and the
+ * verifier (datanode) need to use the same SecretKey.
+ * <p/>
+ *
+ * This package encloses the logic to manage symmetric secret keys
+ * lifecycle. In details, it consists of the following components:
+ * <ul>
+ * <li>
+ * The definition of manage secret key which is shared between SCM,
+ * OM and datanodes, see
+ * {@link org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey}.
+ * </li>
+ *
+ * <li>
+ * The definition of secret key states, which is designed to get replicated
+ * across all SCM instances, see
+ * {@link org.apache.hadoop.hdds.security.symmetric.SecretKeyState}
+ * </li>
+ *
+ * <li>
+ * The definition and implementation of secret key persistent storage, to
+ * help retain SecretKey after restarts, see
+ * {@link org.apache.hadoop.hdds.security.symmetric.SecretKeyStore} and
+ * {@link org.apache.hadoop.hdds.security.symmetric.LocalSecretKeyStore}.
+ * </li>
+ *
+ * <li>
+ * The basic logic to manage secret key lifecycle, see
+ * {@link org.apache.hadoop.hdds.security.symmetric.SecretKeyManager}
+ * </li>
+ * </ul>
+ *
+ * <p/>
+ * The original overall design can be found at
+ * <a href=https://issues.apache.org/jira/browse/HDDS-7733>HDDS-7733</a>.
+ */
+package org.apache.hadoop.hdds.security.symmetric;
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStoreTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStoreTest.java
new file mode 100644
index 0000000000..c406ce2b08
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStoreTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.PosixFilePermission;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Stream;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Sets.newHashSet;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test cases for {@link LocalSecretKeyStore}.
+ */
+public class LocalSecretKeyStoreTest {
+ private SecretKeyStore secretKeyStore;
+ private Path testSecretFile;
+
+ @BeforeEach
+ private void setup() throws Exception {
+ testSecretFile = Files.createTempFile("key-strore-test", ".json");
+ secretKeyStore = new LocalSecretKeyStore(testSecretFile);
+ }
+
+ public static Stream<Arguments> saveAndLoadTestCases() throws Exception {
+ return Stream.of(
+ // empty
+ Arguments.of(ImmutableList.of()),
+ // single secret keys.
+ Arguments.of(newArrayList(
+ generateKey("HmacSHA256")
+ )),
+ // multiple secret keys.
+ Arguments.of(newArrayList(
+ generateKey("HmacSHA1"),
+ generateKey("HmacSHA256")
+ ))
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("saveAndLoadTestCases")
+ public void testSaveAndLoad(List<ManagedSecretKey> keys) throws IOException {
+ secretKeyStore.save(keys);
+
+ // Ensure the intended file exists and is readable and writeable to
+ // file owner only.
+ File file = testSecretFile.toFile();
+ assertTrue(file.exists());
+ Set<PosixFilePermission> permissions =
+ Files.getPosixFilePermissions(file.toPath());
+ assertEquals(newHashSet(OWNER_READ, OWNER_WRITE), permissions);
+
+ List<ManagedSecretKey> reloadedKeys = secretKeyStore.load();
+ assertEqualKeys(keys, reloadedKeys);
+ }
+
+ /**
+ * Verifies that secret keys are overwritten by subsequent writes.
+ */
+ @Test
+ public void testOverwrite() throws Exception {
+ List<ManagedSecretKey> initialKeys =
+ newArrayList(generateKey("HmacSHA256"));
+ secretKeyStore.save(initialKeys);
+
+ List<ManagedSecretKey> updatedKeys = newArrayList(
+ generateKey("HmacSHA1"),
+ generateKey("HmacSHA256")
+ );
+ secretKeyStore.save(updatedKeys);
+
+ assertEqualKeys(updatedKeys, secretKeyStore.load());
+ }
+
+ /**
+ * This scenario verifies if an existing secret keys file can be loaded.
+ * The intention of this is to ensure a saved file can be loaded after
+ * future changes to {@link ManagedSecretKey} schema.
+ *
+ * Please don't just change the content of test json if this
+ * test fails, instead, analyse the backward-compatibility of the change.
+ */
+ @Test
+ public void testLoadExistingFile() throws Exception {
+ // copy test file content to the backing file.
+ String testJson = "[\n" +
+ " {\n" +
+ " \"id\":\"78864cfb-793b-4157-8ad6-714c9f950a16\",\n" +
+ " \"creationTime\":\"2007-12-03T10:15:30Z\",\n" +
+ " \"expiryTime\":\"2007-12-03T11:15:30Z\",\n" +
+ " \"algorithm\":\"HmacSHA256\",\n" +
+ " \"encoded\":\"YSeCdJRB4RclxoeE69ENmTe2Cv8ybyKhHP3mq4M1r8o=\"\n" +
+ " }\n" +
+ "]";
+ Files.write(testSecretFile, Collections.singletonList(testJson),
+ StandardOpenOption.WRITE);
+
+ Instant date = Instant.parse("2007-12-03T10:15:30.00Z");
+ ManagedSecretKey secretKey = new ManagedSecretKey(
+ UUID.fromString("78864cfb-793b-4157-8ad6-714c9f950a16"),
+ date,
+ date.plus(Duration.ofHours(1)),
+ new SecretKeySpec(
+ Base64.getDecoder().decode(
+ "YSeCdJRB4RclxoeE69ENmTe2Cv8ybyKhHP3mq4M1r8o="),
+ "HmacSHA256"
+ ));
+
+ List<ManagedSecretKey> expectedKeys = newArrayList(secretKey);
+ assertEqualKeys(expectedKeys, secretKeyStore.load());
+ }
+
+ private void assertEqualKeys(List<ManagedSecretKey> expected,
+ List<ManagedSecretKey> actual) {
+ assertEquals(expected.size(), actual.size());
+ for (int i = 0; i < expected.size(); i++) {
+ ManagedSecretKey expectedKey = expected.get(i);
+ ManagedSecretKey actualKey = actual.get(i);
+
+ assertEquals(expectedKey.getId(), actualKey.getId());
+ assertEquals(expectedKey.getCreationTime().toEpochMilli(),
+ actualKey.getCreationTime().toEpochMilli());
+ assertEquals(expectedKey.getExpiryTime(),
+ actualKey.getExpiryTime());
+ assertEquals(expectedKey.getSecretKey(), actualKey.getSecretKey());
+ }
+ }
+
+ private static ManagedSecretKey generateKey(String algorithm)
+ throws Exception {
+ return generateKey(algorithm, Instant.now());
+ }
+
+ private static ManagedSecretKey generateKey(String algorithm,
+ Instant creationTime)
+ throws Exception {
+ KeyGenerator keyGen = KeyGenerator.getInstance(algorithm);
+ SecretKey secretKey = keyGen.generateKey();
+ return new ManagedSecretKey(
+ UUID.randomUUID(),
+ creationTime,
+ creationTime.plus(Duration.ofHours(1)),
+ secretKey
+ );
+ }
+}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java
new file mode 100644
index 0000000000..053148e28d
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.symmetric;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Stream;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static java.time.Instant.now;
+import static java.time.temporal.ChronoUnit.DAYS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.params.provider.Arguments.of;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests cases for {@link SecretKeyManager} implementation.
+ */
+public class SecretKeyManagerTest {
+ private static final Duration VALIDITY_DURATION = Duration.ofDays(3);
+ private static final Duration ROTATION_DURATION = Duration.ofDays(1);
+ private static final String ALGORITHM = "HmacSHA256";
+
+ private SecretKeyStore mockedKeyStore;
+
+ @BeforeEach
+ private void setup() {
+ mockedKeyStore = Mockito.mock(SecretKeyStore.class);
+ }
+
+ public static Stream<Arguments> loadSecretKeysTestCases() throws Exception {
+ ManagedSecretKey k0 = generateKey(now());
+ ManagedSecretKey k1 = generateKey(now().minus(1, DAYS));
+ ManagedSecretKey k2 = generateKey(now().minus(2, DAYS));
+ ManagedSecretKey k3 = generateKey(now().minus(3, DAYS));
+ ManagedSecretKey k4 = generateKey(now().minus(4, DAYS));
+ ManagedSecretKey k5 = generateKey(now().minus(5, DAYS));
+ return Stream.of(
+ // first start
+ of(ImmutableList.of(), null, null),
+
+ // restart => nothing is filtered
+ of(newArrayList(k0, k1, k2), k0, newArrayList(k0, k1, k2)),
+
+ // stop 1 day and start
+ of(newArrayList(k1, k2, k3), k1, newArrayList(k1, k2)),
+
+ // stop 2 day and start => expired keys are filtered
+ of(newArrayList(k2, k3, k4), k2, newArrayList(k2)),
+
+ // stop 3 day and start, all saved keys are filtered
+ of(newArrayList(k3, k4, k5), null, null)
+ );
+ }
+
+ /**
+ * Verify how SecretKeyManager initializes its keys under different scenarios,
+ * e.g. with or without the present of saved keys.
+ */
+ @ParameterizedTest
+ @MethodSource("loadSecretKeysTestCases")
+ public void testLoadSecretKeys(List<ManagedSecretKey> savedSecretKey,
+ ManagedSecretKey expectedCurrentKey,
+ List<ManagedSecretKey> expectedLoadedKeys)
+ throws Exception {
+ SecretKeyState state = new SecretKeyStateImpl(mockedKeyStore);
+ SecretKeyManager lifeCycleManager =
+ new SecretKeyManager(state, mockedKeyStore,
+ ROTATION_DURATION, VALIDITY_DURATION, ALGORITHM);
+
+ when(mockedKeyStore.load()).thenReturn(savedSecretKey);
+ lifeCycleManager.checkAndInitialize();
+
+ if (expectedCurrentKey != null) {
+ assertEquals(state.getCurrentKey(), expectedCurrentKey);
+ List<ManagedSecretKey> allKeys = state.getSortedKeys();
+ assertSameKeys(expectedLoadedKeys, allKeys);
+ } else {
+ // expect the current key is newly generated.
+ assertFalse(savedSecretKey.contains(state.getCurrentKey()));
+ assertEquals(1, state.getSortedKeys().size());
+ assertTrue(state.getSortedKeys().contains(
+ state.getCurrentKey()));
+ }
+ }
+
+ private static void assertSameKeys(Collection<ManagedSecretKey> expected,
+ Collection<ManagedSecretKey> actual) {
+ assertEquals(expected.size(), actual.size());
+ for (ManagedSecretKey expectedKey : expected) {
+ assertTrue(actual.contains(expectedKey));
+ }
+ }
+
+ public static Stream<Arguments> rotationTestCases() throws Exception {
+ ManagedSecretKey k0 = generateKey(now());
+ ManagedSecretKey k1 = generateKey(now().minus(1, DAYS));
+ ManagedSecretKey k2 = generateKey(now().minus(2, DAYS));
+ ManagedSecretKey k3 = generateKey(now().minus(3, DAYS));
+ ManagedSecretKey k4 = generateKey(now().minus(4, DAYS));
+ return Stream.of(
+
+ // Currentkey is new, not rotate.
+ of(newArrayList(k0, k1, k2), false, null),
+
+ // Current key just exceeds the rotation period.
+ of(newArrayList(k1, k2, k3), true, newArrayList(k1, k2)),
+
+ // Current key exceeds the rotation period for a significant time (2d).
+ of(newArrayList(k2, k3, k4), true, newArrayList(k2))
+ );
+ }
+
+ /**
+ * Verify rotation behavior under different scenarios.
+ */
+ @ParameterizedTest
+ @MethodSource("rotationTestCases")
+ public void testRotate(List<ManagedSecretKey> initialKeys,
+ boolean expectRotate,
+ List<ManagedSecretKey> expectedRetainedKeys)
+ throws TimeoutException {
+
+ SecretKeyState state = new SecretKeyStateImpl(mockedKeyStore);
+
+ SecretKeyManager lifeCycleManager =
+ new SecretKeyManager(state, mockedKeyStore,
+ ROTATION_DURATION, VALIDITY_DURATION, ALGORITHM);
+
+ // Set the initial state.
+ state.updateKeys(initialKeys);
+ ManagedSecretKey initialCurrentKey = state.getCurrentKey();
+ Mockito.reset(mockedKeyStore);
+
+ assertEquals(expectRotate, lifeCycleManager.checkAndRotate());
+
+ if (expectRotate) {
+ // Verify rotation behavior.
+
+ // 1. A new key is generated as current key.
+ ManagedSecretKey currentKey = state.getCurrentKey();
+ assertNotEquals(initialCurrentKey, currentKey);
+ assertFalse(initialKeys.contains(currentKey));
+
+ // 2. keys are correctly rotated, expired ones are excluded.
+ List<ManagedSecretKey> expectedAllKeys = expectedRetainedKeys;
+ expectedAllKeys.add(currentKey);
+ assertSameKeys(expectedAllKeys, state.getSortedKeys());
+
+ // 3. All keys are stored.
+ ArgumentCaptor<Collection<ManagedSecretKey>> storedKeyCaptor =
+ ArgumentCaptor.forClass(Collection.class);
+ verify(mockedKeyStore).save(storedKeyCaptor.capture());
+ assertSameKeys(expectedAllKeys, storedKeyCaptor.getValue());
+
+ // 4. The new generated key has correct data.
+ assertEquals(ALGORITHM, currentKey.getSecretKey().getAlgorithm());
+ assertEquals(0,
+ Duration.between(currentKey.getCreationTime(), now()).toMinutes());
+ Instant expectedExpiryTime = now().plus(VALIDITY_DURATION);
+ assertEquals(0,
+ Duration.between(currentKey.getExpiryTime(),
+ expectedExpiryTime).toMinutes());
+ } else {
+ assertEquals(initialCurrentKey, state.getCurrentKey());
+ assertSameKeys(initialKeys, state.getSortedKeys());
+ }
+ }
+
+ private static ManagedSecretKey generateKey(Instant creationTime)
+ throws Exception {
+ KeyGenerator keyGen = KeyGenerator.getInstance(ALGORITHM);
+ SecretKey secretKey = keyGen.generateKey();
+ return new ManagedSecretKey(
+ UUID.randomUUID(),
+ creationTime,
+ creationTime.plus(VALIDITY_DURATION),
+ secretKey
+ );
+ }
+}
diff --git a/hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto
index 41da6a5468..4fb0737b39 100644
--- a/hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto
@@ -29,6 +29,7 @@ enum RequestType {
MOVE = 6;
STATEFUL_SERVICE_CONFIG = 7;
FINALIZE = 8;
+ SECRET_KEY = 9;
}
message Method {
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
index dc6bcf986c..3621018fa8 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
@@ -249,3 +249,12 @@ message SCMRevokeCertificatesResponseProto {
service SCMSecurityProtocolService {
rpc submitRequest (SCMSecurityRequest) returns (SCMSecurityResponse);
}
+
+message ManagedSecretKey {
+ required UUID id = 1;
+ required uint64 creationTime = 2;
+ required uint64 expiryTime = 3;
+ required string algorithm = 4;
+ required bytes encoded = 5;
+}
+
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java
index 6c75593be1..af77051509 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java
@@ -23,6 +23,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.ProtocolMessageEnum;
import org.apache.commons.lang3.ClassUtils;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import java.math.BigInteger;
import java.security.cert.X509Certificate;
@@ -49,6 +50,7 @@ public final class CodecFactory {
codecs.put(BigInteger.class, new BigIntegerCodec());
codecs.put(X509Certificate.class, new X509CertificateCodec());
codecs.put(ByteString.class, new ByteStringCodec());
+ codecs.put(ManagedSecretKey.class, new ManagedSecretKeyCodec());
}
private CodecFactory() { }
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java
index 0667b8776f..67d8d55227 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.ListArgument;
import org.apache.hadoop.hdds.scm.ha.ReflectionUtil;
import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -51,8 +52,11 @@ public class ListCodec implements Codec {
public Object deserialize(Class<?> type, ByteString value)
throws InvalidProtocolBufferException {
try {
+ // If argument type is the generic interface, then determine a
+ // concrete implementation.
+ Class<?> concreteType = (type == List.class) ? ArrayList.class : type;
- List<Object> result = (List<Object>) type.newInstance();
+ List<Object> result = (List<Object>) concreteType.newInstance();
final ListArgument listArgs = (ListArgument) ReflectionUtil
.getMethod(ListArgument.class, "parseFrom", byte[].class)
.invoke(null, (Object) value.toByteArray());
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java
new file mode 100644
index 0000000000..384d818762
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.ha.io;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+
+/**
+ * A codec for {@link ManagedSecretKey} objects.
+ */
+public class ManagedSecretKeyCodec implements Codec {
+ @Override
+ public ByteString serialize(Object object)
+ throws InvalidProtocolBufferException {
+ ManagedSecretKey secretKey = (ManagedSecretKey) object;
+ return ByteString.copyFrom(secretKey.toProtobuf().toByteArray());
+ }
+
+ @Override
+ public Object deserialize(Class<?> type, ByteString value)
+ throws InvalidProtocolBufferException {
+ SCMSecurityProtocolProtos.ManagedSecretKey message =
+ SCMSecurityProtocolProtos.ManagedSecretKey.parseFrom(value);
+ return ManagedSecretKey.fromProtobuf(message);
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/ScmSecretKeyStateBuilder.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/ScmSecretKeyStateBuilder.java
new file mode 100644
index 0000000000..c689fd2db3
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/ScmSecretKeyStateBuilder.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm.security;
+
+import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol;
+import org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyState;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyStateImpl;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyStore;
+
+import java.lang.reflect.Proxy;
+
+/**
+ * Builder for {@link SecretKeyState} with a proper proxy to make @Replicate
+ * happen.
+ */
+public class ScmSecretKeyStateBuilder {
+ private SecretKeyStore secretKeyStore;
+ private SCMRatisServer scmRatisServer;
+
+ public ScmSecretKeyStateBuilder setSecretKeyStore(
+ SecretKeyStore secretKeyStore) {
+ this.secretKeyStore = secretKeyStore;
+ return this;
+ }
+
+ public ScmSecretKeyStateBuilder setRatisServer(
+ final SCMRatisServer ratisServer) {
+ scmRatisServer = ratisServer;
+ return this;
+ }
+
+ public SecretKeyState build() {
+ final SecretKeyState impl = new SecretKeyStateImpl(secretKeyStore);
+
+ final SCMHAInvocationHandler scmhaInvocationHandler =
+ new SCMHAInvocationHandler(SCMRatisProtocol.RequestType.SECRET_KEY,
+ impl, scmRatisServer);
+
+ return (SecretKeyState) Proxy.newProxyInstance(
+ SCMHAInvocationHandler.class.getClassLoader(),
+ new Class<?>[]{SecretKeyState.class}, scmhaInvocationHandler);
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java
new file mode 100644
index 0000000000..27ce30a8a1
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.security;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
+import org.apache.hadoop.hdds.scm.ha.SCMService;
+import org.apache.hadoop.hdds.security.symmetric.LocalSecretKeyStore;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyConfig;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyState;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyStore;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_CA_CERT_STORAGE_DIR;
+
+/**
+ * A background service running in SCM to maintain the SecretKeys lifecycle.
+ */
+public class SecretKeyManagerService implements SCMService, Runnable {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(SecretKeyManagerService.class);
+
+ private final SCMContext scmContext;
+ private final SecretKeyManager secretKeyManager;
+ private final SecretKeyConfig secretKeyConfig;
+
+
+ /**
+ * SCMService related variables.
+ */
+ private final Lock serviceLock = new ReentrantLock();
+ private ServiceStatus serviceStatus = ServiceStatus.PAUSING;
+
+ private final ScheduledExecutorService scheduler;
+
+ @SuppressWarnings("parameternumber")
+ public SecretKeyManagerService(SCMContext scmContext,
+ ConfigurationSource conf,
+ SCMRatisServer ratisServer) {
+ this.scmContext = scmContext;
+
+ secretKeyConfig = new SecretKeyConfig(conf,
+ SCM_CA_CERT_STORAGE_DIR);
+ SecretKeyStore secretKeyStore = new LocalSecretKeyStore(
+ secretKeyConfig.getLocalSecretKeyFile());
+ SecretKeyState secretKeyState = new ScmSecretKeyStateBuilder()
+ .setSecretKeyStore(secretKeyStore)
+ .setRatisServer(ratisServer)
+ .build();
+ secretKeyManager = new SecretKeyManager(secretKeyState,
+ secretKeyStore, secretKeyConfig);
+
+ scheduler = Executors.newScheduledThreadPool(1,
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat(getServiceName())
+ .build());
+
+ start();
+ }
+
+ @Override
+ public void notifyStatusChanged() {
+ serviceLock.lock();
+ try {
+ if (scmContext.isLeaderReady()) {
+ // Asynchronously initialize SecretKeys for first time leader.
+ if (!secretKeyManager.isInitialized()) {
+ scheduler.schedule(() -> {
+ try {
+ secretKeyManager.checkAndInitialize();
+ } catch (TimeoutException e) {
+ throw new RuntimeException(
+ "Timeout replicating initialized state.", e);
+ }
+ }, 0, TimeUnit.SECONDS);
+ }
+
+ serviceStatus = ServiceStatus.RUNNING;
+ } else {
+ serviceStatus = ServiceStatus.PAUSING;
+ }
+ } finally {
+ serviceLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean shouldRun() {
+ serviceLock.lock();
+ try {
+ return serviceStatus == ServiceStatus.RUNNING;
+ } finally {
+ serviceLock.unlock();
+ }
+ }
+
+ @Override
+ public void run() {
+ if (!shouldRun()) {
+ return;
+ }
+
+ try {
+ secretKeyManager.checkAndRotate();
+ } catch (TimeoutException e) {
+ LOG.error("Error occurred when updating SecretKeys.", e);
+ }
+ }
+
+ @Override
+ public String getServiceName() {
+ return SecretKeyManagerService.class.getSimpleName();
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Scheduling rotation checker with interval {}",
+ secretKeyConfig.getRotationCheckDuration());
+ scheduler.scheduleAtFixedRate(this, 0,
+ secretKeyConfig.getRotationCheckDuration().toMillis(),
+ TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void stop() {
+ scheduler.shutdownNow();
+ }
+
+ public static boolean isSecretKeyEnable(SecurityConfig conf) {
+ return conf.isSecurityEnabled() &&
+ (conf.isBlockTokenEnabled() || conf.isContainerTokenEnabled());
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/package-info.java
new file mode 100644
index 0000000000..296e7f0883
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Encapsulate classes dealing with security concern in SCM.
+ */
+package org.apache.hadoop.hdds.scm.security;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 7d0fd4bd78..1d1a6be253 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.node.NodeAddressUpdateHandler;
+import org.apache.hadoop.hdds.scm.security.SecretKeyManagerService;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManagerImpl;
import org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManager;
@@ -189,6 +190,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_EVENT_REPORT_EXEC_WAIT_THRESHOLD_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_EVENT_REPORT_QUEUE_WAIT_THRESHOLD_DEFAULT;
+import static org.apache.hadoop.hdds.scm.security.SecretKeyManagerService.isSecretKeyEnable;
import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
@@ -741,6 +743,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
serviceManager.register(expiredContainerReplicaOpScrubber);
+ if (isSecretKeyEnable(securityConfig)) {
+ SecretKeyManagerService secretKeyManagerService =
+ new SecretKeyManagerService(scmContext, conf,
+ scmHAManager.getRatisServer());
+ serviceManager.register(secretKeyManagerService);
+ }
+
if (configurator.getContainerManager() != null) {
containerManager = configurator.getContainerManager();
} else {
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
index 9be6e14a29..b57e5913e7 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
@@ -162,3 +162,8 @@ no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1
# Explicitly enable filesystem snapshot feature for this Docker compose cluster
OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
+
+
+OZONE-SITE.XML_hdds.secret.key.rotate.duration=5m
+OZONE-SITE.XML_hdds.secret.key.rotate.check.duration=1m
+OZONE-SITE.XML_hdds.secret.key.expiry.duration=1h
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 02/05: HDDS-7830. SCM API for OM and Datanode to get secret keys (#4345)
Posted by du...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
duong pushed a commit to branch HDDS-7733-Symmetric-Tokens
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 42e2cb302e074b41e7ace927498c719a52a65558
Author: Duong Nguyen <du...@gmail.com>
AuthorDate: Tue Mar 14 17:59:54 2023 -0700
HDDS-7830. SCM API for OM and Datanode to get secret keys (#4345)
---
.../security/exception/SCMSecurityException.java | 4 +-
.../java/org/apache/hadoop/util/ProtobufUtils.java | 41 +++
.../hadoop/hdds/utils/TestProtobufUtils.java | 48 +++
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 23 ++
.../SCMSecurityProtocolClientSideTranslatorPB.java | 38 +++
.../hdds/security/symmetric/ManagedSecretKey.java | 12 +-
.../hdds/security/symmetric/SecretKeyManager.java | 12 +
.../hdds/security/symmetric/SecretKeyState.java | 3 +
.../security/symmetric/SecretKeyStateImpl.java | 23 ++
.../src/main/proto/ScmServerSecurityProtocol.proto | 30 ++
.../SCMSecurityProtocolServerSideTranslatorPB.java | 50 ++++
.../hdds/scm/security/SecretKeyManagerService.java | 4 +
.../hdds/scm/server/SCMSecurityProtocolServer.java | 48 ++-
.../hdds/scm/server/StorageContainerManager.java | 22 +-
.../scm/server/TestSCMSecurityProtocolServer.java | 2 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 7 +-
.../org/apache/hadoop/ozone/TestSecretKeysApi.java | 329 +++++++++++++++++++++
17 files changed, 674 insertions(+), 22 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
index 7e008afc41..13b8395391 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
@@ -109,6 +109,8 @@ public class SCMSecurityException extends IOException {
MISSING_BLOCK_TOKEN,
BLOCK_TOKEN_VERIFICATION_FAILED,
GET_ROOT_CA_CERT_FAILED,
- NOT_A_PRIMARY_SCM
+ NOT_A_PRIMARY_SCM,
+ SECRET_KEY_NOT_ENABLED,
+ SECRET_KEY_NOT_INITIALIZED
}
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ProtobufUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ProtobufUtils.java
new file mode 100644
index 0000000000..428157981e
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ProtobufUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+
+import java.util.UUID;
+
+/**
+ * Contains utilities to ease common protobuf to java object conversions.
+ */
+public final class ProtobufUtils {
+ private ProtobufUtils() {
+ }
+
+ public static HddsProtos.UUID toProtobuf(UUID uuid) {
+ return HddsProtos.UUID.newBuilder()
+ .setMostSigBits(uuid.getMostSignificantBits())
+ .setLeastSigBits(uuid.getLeastSignificantBits())
+ .build();
+ }
+
+ public static UUID fromProtobuf(HddsProtos.UUID proto) {
+ return new UUID(proto.getMostSigBits(), proto.getLeastSigBits());
+ }
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestProtobufUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestProtobufUtils.java
new file mode 100644
index 0000000000..fe6a57846c
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestProtobufUtils.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.utils;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static org.apache.hadoop.util.ProtobufUtils.fromProtobuf;
+import static org.apache.hadoop.util.ProtobufUtils.toProtobuf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Test-cases for {@link org.apache.hadoop.util.ProtobufUtils}.
+ */
+public class TestProtobufUtils {
+ @Test
+ public void testUuidToProtobuf() {
+ UUID object = UUID.randomUUID();
+ HddsProtos.UUID protobuf = toProtobuf(object);
+ assertEquals(object.getLeastSignificantBits(), protobuf.getLeastSigBits());
+ assertEquals(object.getMostSignificantBits(), protobuf.getMostSigBits());
+ }
+
+ @Test
+ public void testUuidConversion() {
+ UUID original = UUID.randomUUID();
+ HddsProtos.UUID protobuf = toProtobuf(original);
+ UUID deserialized = fromProtobuf(protobuf);
+ assertEquals(original, deserialized);
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
index 26107d54ac..1cfe568d8a 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.protocol;
import java.io.IOException;
import java.util.List;
+import java.util.UUID;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -26,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmNodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeDetailsProto;
import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.security.KerberosInfo;
@@ -170,4 +172,25 @@ public interface SCMSecurityProtocol {
*/
String getCertificate(NodeDetailsProto nodeDetails,
String certSignReq) throws IOException;
+
+
+ /**
+ * Get the current SecretKey that is used for signing tokens.
+ * @return ManagedSecretKey
+ */
+ ManagedSecretKey getCurrentSecretKey() throws IOException;
+
+ /**
+ * Get a particular SecretKey by ID.
+ *
+ * @param id the id to get SecretKey.
+ * @return ManagedSecretKey.
+ */
+ ManagedSecretKey getSecretKey(UUID id) throws IOException;
+
+ /**
+ * Get all the non-expired SecretKey managed by SCM.
+ * @return list of ManagedSecretKey.
+ */
+ List<ManagedSecretKey> getAllSecretKeys() throws IOException;
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
index 19bae372b4..ab09061c44 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
@@ -22,7 +22,9 @@ import java.security.cert.CRLException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
@@ -40,6 +42,8 @@ import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCer
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCrlsRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetSecretKeyRequest;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetSecretKeyResponse;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCACertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetLatestCrlIdRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCertificateRequestProto;
@@ -51,6 +55,7 @@ import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecuri
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Type;
import org.apache.hadoop.hdds.scm.proxy.SCMSecurityProtocolFailoverProxyProvider;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.io.retry.RetryProxy;
@@ -60,6 +65,7 @@ import org.apache.hadoop.ipc.RPC;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
+
import static org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetOMCertRequestProto;
/**
@@ -189,6 +195,38 @@ public class SCMSecurityProtocolClientSideTranslatorPB implements
.getX509Certificate();
}
+ @Override
+ public ManagedSecretKey getCurrentSecretKey() throws IOException {
+ SCMSecurityProtocolProtos.ManagedSecretKey secretKeyProto =
+ submitRequest(Type.GetCurrentSecretKey, builder -> {
+ }).getCurrentSecretKeyResponseProto().getSecretKey();
+ return ManagedSecretKey.fromProtobuf(secretKeyProto);
+ }
+
+ @Override
+ public ManagedSecretKey getSecretKey(UUID id) throws IOException {
+ SCMGetSecretKeyRequest request = SCMGetSecretKeyRequest.newBuilder()
+ .setSecretKeyId(HddsProtos.UUID.newBuilder()
+ .setMostSigBits(id.getMostSignificantBits())
+ .setLeastSigBits(id.getLeastSignificantBits())).build();
+ SCMGetSecretKeyResponse response = submitRequest(Type.GetSecretKey,
+ builder -> builder.setGetSecretKeyRequest(request))
+ .getGetSecretKeyResponseProto();
+
+ return response.hasSecretKey() ?
+ ManagedSecretKey.fromProtobuf(response.getSecretKey()) : null;
+ }
+
+ @Override
+ public List<ManagedSecretKey> getAllSecretKeys() throws IOException {
+ List<SCMSecurityProtocolProtos.ManagedSecretKey> secretKeysList =
+ submitRequest(Type.GetAllSecretKeys, builder -> {
+ }).getSecretKeysListResponseProto().getSecretKeysList();
+ return secretKeysList.stream()
+ .map(ManagedSecretKey::fromProtobuf)
+ .collect(Collectors.toList());
+ }
+
/**
* Get signed certificate for SCM node.
*
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
index 7e8aaacb48..3128265e9a 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.hdds.security.symmetric;
import com.google.protobuf.ByteString;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.util.ProtobufUtils;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
@@ -92,13 +92,8 @@ public final class ManagedSecretKey implements Serializable {
* @return the protobuf message to deserialize this object.
*/
public SCMSecurityProtocolProtos.ManagedSecretKey toProtobuf() {
- HddsProtos.UUID uuid = HddsProtos.UUID.newBuilder()
- .setMostSigBits(this.id.getMostSignificantBits())
- .setLeastSigBits(this.id.getLeastSignificantBits())
- .build();
-
return SCMSecurityProtocolProtos.ManagedSecretKey.newBuilder()
- .setId(uuid)
+ .setId(ProtobufUtils.toProtobuf(id))
.setCreationTime(this.creationTime.toEpochMilli())
.setExpiryTime(this.expiryTime.toEpochMilli())
.setAlgorithm(this.secretKey.getAlgorithm())
@@ -111,8 +106,7 @@ public final class ManagedSecretKey implements Serializable {
*/
public static ManagedSecretKey fromProtobuf(
SCMSecurityProtocolProtos.ManagedSecretKey message) {
- UUID id = new UUID(message.getId().getMostSigBits(),
- message.getId().getLeastSigBits());
+ UUID id = ProtobufUtils.fromProtobuf(message.getId());
Instant creationTime = Instant.ofEpochMilli(message.getCreationTime());
Instant expiryTime = Instant.ofEpochMilli(message.getExpiryTime());
SecretKey secretKey = new SecretKeySpec(message.getEncoded().toByteArray(),
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
index 0dc5bf8902..cb529e10d1 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
@@ -129,6 +129,18 @@ public class SecretKeyManager {
return false;
}
+ public ManagedSecretKey getCurrentKey() {
+ return state.getCurrentKey();
+ }
+
+ public ManagedSecretKey getKey(UUID id) {
+ return state.getKey(id);
+ }
+
+ public List<ManagedSecretKey> getSortedKeys() {
+ return state.getSortedKeys();
+ }
+
private boolean shouldRotate(ManagedSecretKey currentKey) {
Duration established = between(currentKey.getCreationTime(), Instant.now());
return established.compareTo(rotationDuration) >= 0;
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
index 7be70b4b02..7b510a10b2 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdds.security.symmetric;
import org.apache.hadoop.hdds.scm.metadata.Replicate;
import java.util.List;
+import java.util.UUID;
import java.util.concurrent.TimeoutException;
/**
@@ -36,6 +37,8 @@ public interface SecretKeyState {
*/
ManagedSecretKey getCurrentKey();
+ ManagedSecretKey getKey(UUID id);
+
/**
* Get the keys that managed by this manager.
* The returned keys are sorted by creation time, in the order of latest
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
index d5c886fd99..b1d66e1186 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
@@ -23,12 +23,16 @@ import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
import static java.util.Comparator.comparing;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
/**
* Default implementation of {@link SecretKeyState}.
@@ -41,6 +45,7 @@ public final class SecretKeyStateImpl implements SecretKeyState {
private List<ManagedSecretKey> sortedKeys;
private ManagedSecretKey currentKey;
+ private Map<UUID, ManagedSecretKey> keyById;
private final SecretKeyStore keyStore;
@@ -66,6 +71,20 @@ public final class SecretKeyStateImpl implements SecretKeyState {
}
}
+ @Override
+ public ManagedSecretKey getKey(UUID id) {
+ lock.readLock().lock();
+ try {
+ // Return null if not initialized yet.
+ if (keyById == null) {
+ return null;
+ }
+ return keyById.get(id);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
/**
* Get the keys that managed by this manager.
* The returned keys are sorted by creation time, in the order of latest
@@ -98,6 +117,10 @@ public final class SecretKeyStateImpl implements SecretKeyState {
.collect(toList())
);
currentKey = sortedKeys.get(0);
+ keyById = newKeys.stream().collect(toMap(
+ ManagedSecretKey::getId,
+ Function.identity()
+ ));
LOG.info("Current key updated {}", currentKey);
keyStore.save(sortedKeys);
} finally {
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
index 3621018fa8..27d1e3c1c3 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
@@ -56,6 +56,7 @@ message SCMSecurityRequest {
optional SCMGetLatestCrlIdRequestProto getLatestCrlIdRequest = 11;
optional SCMRevokeCertificatesRequestProto revokeCertificatesRequest = 12;
optional SCMGetCertRequestProto getCertRequest = 13;
+ optional SCMGetSecretKeyRequest getSecretKeyRequest = 14;
}
message SCMSecurityResponse {
@@ -81,6 +82,12 @@ message SCMSecurityResponse {
optional SCMRevokeCertificatesResponseProto revokeCertificatesResponseProto = 10;
+ optional SCMGetCurrentSecretKeyResponse currentSecretKeyResponseProto = 11;
+
+ optional SCMGetSecretKeyResponse getSecretKeyResponseProto = 12;
+
+ optional SCMSecretKeysListResponse secretKeysListResponseProto = 13;
+
}
enum Type {
@@ -96,6 +103,9 @@ enum Type {
GetLatestCrlId = 10;
RevokeCertificates = 11;
GetCert = 12;
+ GetCurrentSecretKey = 13;
+ GetSecretKey = 14;
+ GetAllSecretKeys = 15;
}
enum Status {
@@ -116,6 +126,8 @@ enum Status {
GET_ROOT_CA_CERTIFICATE_FAILED = 15;
NOT_A_PRIMARY_SCM = 16;
REVOKE_CERTIFICATE_FAILED = 17;
+ SECRET_KEY_NOT_ENABLED = 18;
+ SECRET_KEY_NOT_INITIALIZED = 19;
}
/**
* This message is send by data node to prove its identity and get an SCM
@@ -258,3 +270,21 @@ message ManagedSecretKey {
required bytes encoded = 5;
}
+message SCMGetSecretKeyRequest {
+ required UUID secretKeyId = 1;
+}
+
+message SCMGetCurrentSecretKeyResponse {
+ required ManagedSecretKey secretKey = 1;
+}
+
+message SCMGetSecretKeyResponse {
+ optional ManagedSecretKey secretKey = 1;
+}
+
+message SCMSecretKeysListResponse {
+ repeated ManagedSecretKey secretKeys = 1;
+}
+
+
+
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
index 736aef15a0..9b7f71bd8a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.protocol;
import java.io.IOException;
import java.util.List;
+import java.util.UUID;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
@@ -26,16 +27,20 @@ import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCer
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCrlsRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCrlsResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCurrentSecretKeyResponse;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetLatestCrlIdRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetLatestCrlIdResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetOMCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetSCMCertRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetSecretKeyRequest;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetSecretKeyResponse;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCertificateResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRevokeCertificatesRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRevokeCertificatesResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecretKeysListResponse;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityResponse;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Status;
@@ -43,6 +48,7 @@ import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.scm.ha.RatisUtil;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher;
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
@@ -50,6 +56,7 @@ import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import com.google.protobuf.ProtocolMessageEnum;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
+import org.apache.hadoop.util.ProtobufUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -151,6 +158,21 @@ public class SCMSecurityProtocolServerSideTranslatorPB
getCertificate(request.getGetCertRequest()))
.build();
+ case GetCurrentSecretKey:
+ return scmSecurityResponse
+ .setCurrentSecretKeyResponseProto(getCurrentSecretKey())
+ .build();
+
+ case GetSecretKey:
+ return scmSecurityResponse.setGetSecretKeyResponseProto(
+ getSecretKey(request.getGetSecretKeyRequest()))
+ .build();
+
+ case GetAllSecretKeys:
+ return scmSecurityResponse
+ .setSecretKeysListResponseProto(getAllSecretKeys())
+ .build();
+
default:
throw new IllegalArgumentException(
"Unknown request type: " + request.getCmdType());
@@ -173,6 +195,34 @@ public class SCMSecurityProtocolServerSideTranslatorPB
}
}
+ private SCMSecretKeysListResponse getAllSecretKeys() throws IOException {
+ SCMSecretKeysListResponse.Builder builder =
+ SCMSecretKeysListResponse.newBuilder();
+ impl.getAllSecretKeys()
+ .stream().map(ManagedSecretKey::toProtobuf)
+ .forEach(builder::addSecretKeys);
+ return builder.build();
+ }
+
+ private SCMGetSecretKeyResponse getSecretKey(
+ SCMGetSecretKeyRequest getSecretKeyRequest) throws IOException {
+ SCMGetSecretKeyResponse.Builder builder =
+ SCMGetSecretKeyResponse.newBuilder();
+ UUID id = ProtobufUtils.fromProtobuf(getSecretKeyRequest.getSecretKeyId());
+ ManagedSecretKey secretKey = impl.getSecretKey(id);
+ if (secretKey != null) {
+ builder.setSecretKey(secretKey.toProtobuf());
+ }
+ return builder.build();
+ }
+
+ private SCMGetCurrentSecretKeyResponse getCurrentSecretKey()
+ throws IOException {
+ return SCMGetCurrentSecretKeyResponse.newBuilder()
+ .setSecretKey(impl.getCurrentSecretKey().toProtobuf())
+ .build();
+ }
+
/**
* Convert exception to corresponsing status.
* @param ex
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java
index 27ce30a8a1..1761f97992 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java
@@ -147,6 +147,10 @@ public class SecretKeyManagerService implements SCMService, Runnable {
TimeUnit.MILLISECONDS);
}
+ public SecretKeyManager getSecretKeyManager() {
+ return secretKeyManager;
+ }
+
@Override
public void stop() {
scheduler.shutdownNow();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index 2a99415573..8d09b95296 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -30,6 +30,7 @@ import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
+import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@@ -44,12 +45,16 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmNodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.update.server.SCMUpdateServiceGrpcServer;
import org.apache.hadoop.hdds.scm.update.client.UpdateServiceConfig;
import org.apache.hadoop.hdds.scm.update.server.SCMCRLStore;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfig;
@@ -68,6 +73,8 @@ import org.bouncycastle.asn1.x509.CRLReason;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+
import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.CERTIFICATE_NOT_FOUND;
import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CA_CERT_FAILED;
import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CERTIFICATE_FAILED;
@@ -92,15 +99,21 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
private final ProtocolMessageMetrics metrics;
private final StorageContainerManager storageContainerManager;
+ // SecretKey may not be enabled when neither block token nor container
+ // token is enabled.
+ private final SecretKeyManager secretKeyManager;
+
SCMSecurityProtocolServer(OzoneConfiguration conf,
CertificateServer rootCertificateServer,
CertificateServer scmCertificateServer,
- X509Certificate rootCACert, StorageContainerManager scm)
+ X509Certificate rootCACert, StorageContainerManager scm,
+ @Nullable SecretKeyManager secretKeyManager)
throws IOException {
this.storageContainerManager = scm;
this.rootCertificateServer = rootCertificateServer;
this.scmCertificateServer = scmCertificateServer;
this.rootCACertificate = rootCACert;
+ this.secretKeyManager = secretKeyManager;
final int handlerCount =
conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY,
ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT);
@@ -162,6 +175,37 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
return getEncodedCertToString(certSignReq, nodeDetails.getNodeType());
}
+ @Override
+ public ManagedSecretKey getCurrentSecretKey() throws SCMSecurityException {
+ validateSecretKeyStatus();
+ return secretKeyManager.getCurrentKey();
+ }
+
+ @Override
+ public ManagedSecretKey getSecretKey(UUID id) throws SCMSecurityException {
+ validateSecretKeyStatus();
+ return secretKeyManager.getKey(id);
+ }
+
+ @Override
+ public List<ManagedSecretKey> getAllSecretKeys() throws SCMSecurityException {
+ validateSecretKeyStatus();
+ return secretKeyManager.getSortedKeys();
+ }
+
+ private void validateSecretKeyStatus() throws SCMSecurityException {
+ if (secretKeyManager == null) {
+ throw new SCMSecurityException("Secret keys are not enabled.",
+ ErrorCode.SECRET_KEY_NOT_ENABLED);
+ }
+
+ if (!secretKeyManager.isInitialized()) {
+ throw new SCMSecurityException(
+ "Secret key initialization is not finished yet.",
+ ErrorCode.SECRET_KEY_NOT_INITIALIZED);
+ }
+ }
+
/**
* Get SCM signed certificate for OM.
*
@@ -368,7 +412,7 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
} catch (InterruptedException | ExecutionException e) {
Thread.currentThread().interrupt();
throw new SCMException("Fail to revoke certs",
- SCMException.ResultCodes.FAILED_TO_REVOKE_CERTIFICATES);
+ ResultCodes.FAILED_TO_REVOKE_CERTIFICATES);
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 1d1a6be253..e79e7a5536 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManagerImpl;
import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext;
import org.apache.hadoop.hdds.scm.server.upgrade.ScmHAUnfinalizedStateValidationAction;
import org.apache.hadoop.hdds.scm.pipeline.WritableContainerFactory;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager;
import org.apache.hadoop.hdds.security.token.ContainerTokenGenerator;
import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
@@ -303,6 +304,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
// container replicas.
private ContainerReplicaPendingOps containerReplicaPendingOps;
private final AtomicBoolean isStopped = new AtomicBoolean(false);
+ private final SecretKeyManagerService secretKeyManagerService;
/** A list of property that are reconfigurable at runtime. */
private final SortedSet<String> reconfigurableProperties =
@@ -388,6 +390,14 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
initializeSystemManagers(conf, configurator);
+ if (isSecretKeyEnable(securityConfig)) {
+ secretKeyManagerService = new SecretKeyManagerService(scmContext, conf,
+ scmHAManager.getRatisServer());
+ serviceManager.register(secretKeyManagerService);
+ } else {
+ secretKeyManagerService = null;
+ }
+
// Authenticate SCM if security is enabled, this initialization can only
// be done after the metadata store is initialized.
if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
@@ -743,13 +753,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
serviceManager.register(expiredContainerReplicaOpScrubber);
- if (isSecretKeyEnable(securityConfig)) {
- SecretKeyManagerService secretKeyManagerService =
- new SecretKeyManagerService(scmContext, conf,
- scmHAManager.getRatisServer());
- serviceManager.register(secretKeyManagerService);
- }
-
if (configurator.getContainerManager() != null) {
containerManager = configurator.getContainerManager();
} else {
@@ -879,6 +882,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
certificateStore, scmStorageConfig, new DefaultProfile());
}
+ SecretKeyManager secretKeyManager = secretKeyManagerService != null ?
+ secretKeyManagerService.getSecretKeyManager() : null;
+
// We need to pass getCACertificate as rootCA certificate,
// as for SCM CA is root-CA.
securityProtocolServer = new SCMSecurityProtocolServer(conf,
@@ -886,7 +892,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmCertificateClient == null ? null :
scmCertificateClient.getRootCACertificate() != null ?
scmCertificateClient.getRootCACertificate() :
- scmCertificateClient.getCACertificate(), this);
+ scmCertificateClient.getCACertificate(), this, secretKeyManager);
if (securityConfig.isContainerTokenEnabled()) {
containerTokenMgr = createContainerTokenSecretManager(configuration);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
index 5480caaf97..82f2cc182a 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
@@ -41,7 +41,7 @@ public class TestSCMSecurityProtocolServer {
config.set(OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT + ":0");
securityProtocolServer = new SCMSecurityProtocolServer(config, null,
- null, null, null);
+ null, null, null, null);
}
@AfterEach
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 4cb535d62f..26d31717e3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -383,7 +383,7 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
// StorageContainerManager constructors respectively). So we need to manage
// them separately, see initOMHAConfig() and initSCMHAConfig().
private final ReservedPorts omPorts = new ReservedPorts(3);
- private final ReservedPorts scmPorts = new ReservedPorts(3);
+ private final ReservedPorts scmPorts = new ReservedPorts(4);
/**
* Creates a new Builder.
@@ -673,12 +673,17 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
scmServiceId, scmNodeId);
String scmGrpcPortKey = ConfUtils.addKeySuffixes(
ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY, scmServiceId, scmNodeId);
+ String scmSecurityAddrKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY, scmServiceId,
+ scmNodeId);
PrimitiveIterator.OfInt nodePorts = scmPorts.assign(scmNodeId);
PrimitiveIterator.OfInt rpcPorts = scmRpcPorts.assign(scmNodeId);
conf.set(scmAddrKey, "127.0.0.1");
conf.set(scmHttpAddrKey, "127.0.0.1:" + nodePorts.nextInt());
conf.set(scmHttpsAddrKey, "127.0.0.1:" + nodePorts.nextInt());
+ conf.set(scmSecurityAddrKey, "127.0.0.1:" + nodePorts.nextInt());
+ conf.set("ozone.scm.update.service.port", "0");
int ratisPort = nodePorts.nextInt();
conf.setInt(scmRatisPortKey, ratisPort);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
new file mode 100644
index 0000000000..9ec8750d78
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ratis.util.ExitUtils;
+import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_CHECK_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION;
+import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.SECRET_KEY_NOT_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Integration test to verify symmetric SecretKeys APIs in a secure cluster.
+ */
+@InterfaceAudience.Private
+public final class TestSecretKeysApi {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestSecretKeysApi.class);
+
+ @Rule
+ public Timeout timeout = Timeout.seconds(1600);
+
+ private MiniKdc miniKdc;
+ private OzoneConfiguration conf;
+ private File workDir;
+ private File ozoneKeytab;
+ private File spnegoKeytab;
+ private File testUserKeytab;
+ private String testUserPrincipal;
+ private String host;
+ private String clusterId;
+ private String scmId;
+ private MiniOzoneHAClusterImpl cluster;
+
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
+
+ ExitUtils.disableSystemExit();
+
+ workDir = GenericTestUtils.getTestDir(getClass().getSimpleName());
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+
+ startMiniKdc();
+ setSecureConfig();
+ createCredentialsInKDC();
+ }
+
+ @After
+ public void stop() {
+ miniKdc.stop();
+ if (cluster != null) {
+ cluster.stop();
+ }
+ DefaultConfigManager.clearDefaultConfigs();
+ }
+
+ private void createCredentialsInKDC() throws Exception {
+ ScmConfig scmConfig = conf.getObject(ScmConfig.class);
+ SCMHTTPServerConfig httpServerConfig =
+ conf.getObject(SCMHTTPServerConfig.class);
+ createPrincipal(ozoneKeytab, scmConfig.getKerberosPrincipal());
+ createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal());
+ createPrincipal(testUserKeytab, testUserPrincipal);
+ }
+
+ private void createPrincipal(File keytab, String... principal)
+ throws Exception {
+ miniKdc.createPrincipal(keytab, principal);
+ }
+
+ private void startMiniKdc() throws Exception {
+ Properties securityProperties = MiniKdc.createConf();
+ miniKdc = new MiniKdc(securityProperties, workDir);
+ miniKdc.start();
+ }
+
+ private void setSecureConfig() throws IOException {
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ host = InetAddress.getLocalHost().getCanonicalHostName()
+ .toLowerCase();
+
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name());
+
+ String curUser = UserGroupInformation.getCurrentUser().getUserName();
+ conf.set(OZONE_ADMINISTRATORS, curUser);
+
+ String realm = miniKdc.getRealm();
+ String hostAndRealm = host + "@" + realm;
+ conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+ conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm);
+ conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+ conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm);
+ conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+
+ ozoneKeytab = new File(workDir, "scm.keytab");
+ spnegoKeytab = new File(workDir, "http.keytab");
+ testUserKeytab = new File(workDir, "testuser.keytab");
+ testUserPrincipal = "test@" + realm;
+
+ conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ ozoneKeytab.getAbsolutePath());
+ conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY,
+ spnegoKeytab.getAbsolutePath());
+ conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+ ozoneKeytab.getAbsolutePath());
+ conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE,
+ spnegoKeytab.getAbsolutePath());
+ conf.set(DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY,
+ ozoneKeytab.getAbsolutePath());
+ }
+
+
+ /**
+ * Test secret key apis in happy case.
+ */
+ @Test
+ public void testSecretKeyApiSuccess() throws Exception {
+ enableBlockToken();
+ // set a low rotation period, of 1s, expiry is 3s, expect 3 active keys
+ // at any moment.
+ conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "100ms");
+ conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1s");
+ conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "3000ms");
+
+ startCluster();
+ SCMSecurityProtocol securityProtocol = getScmSecurityProtocol();
+
+ // start the test when keys are full.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return securityProtocol.getAllSecretKeys().size() >= 3;
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }, 100, 4_000);
+
+ ManagedSecretKey initialKey = securityProtocol.getCurrentSecretKey();
+ assertNotNull(initialKey);
+ List<ManagedSecretKey> initialKeys = securityProtocol.getAllSecretKeys();
+ assertEquals(initialKey, initialKeys.get(0));
+ ManagedSecretKey lastKey = initialKeys.get(initialKeys.size() - 1);
+
+ LOG.info("Initial active key: {}", initialKey);
+ LOG.info("Initial keys: {}", initialKeys);
+
+ // wait for the next rotation.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ManagedSecretKey newCurrentKey = securityProtocol.getCurrentSecretKey();
+ return !newCurrentKey.equals(initialKey);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }, 100, 1500);
+ ManagedSecretKey updatedKey = securityProtocol.getCurrentSecretKey();
+ List<ManagedSecretKey> updatedKeys = securityProtocol.getAllSecretKeys();
+
+ LOG.info("Updated active key: {}", updatedKey);
+ LOG.info("Updated keys: {}", updatedKeys);
+
+ assertEquals(updatedKey, updatedKeys.get(0));
+ assertEquals(initialKey, updatedKeys.get(1));
+ // ensure the last key from the previous cycle no longer managed.
+ assertTrue(lastKey.isExpired());
+ assertFalse(updatedKeys.contains(lastKey));
+
+ // assert getSecretKey by ID.
+ ManagedSecretKey keyById = securityProtocol.getSecretKey(
+ updatedKey.getId());
+ assertNotNull(keyById);
+ ManagedSecretKey nonExisting = securityProtocol.getSecretKey(
+ UUID.randomUUID());
+ assertNull(nonExisting);
+ }
+
+ /**
+ * Verify API behavior when block token is not enable.
+ */
+ @Test
+ public void testSecretKeyApiNotEnabled() throws Exception {
+ startCluster();
+ SCMSecurityProtocol securityProtocol = getScmSecurityProtocol();
+
+ SCMSecurityException ex = assertThrows(SCMSecurityException.class,
+ securityProtocol::getCurrentSecretKey);
+ assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode());
+
+ ex = assertThrows(SCMSecurityException.class,
+ () -> securityProtocol.getSecretKey(UUID.randomUUID()));
+ assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode());
+
+ ex = assertThrows(SCMSecurityException.class,
+ securityProtocol::getAllSecretKeys);
+ assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode());
+ }
+
+ /**
+ * Verify API behavior when SCM leader fails.
+ */
+ @Test
+ public void testSecretKeyAfterSCMFailover() throws Exception {
+ enableBlockToken();
+ // set a long duration period, so that no rotation happens during SCM
+ // leader change.
+ conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "10m");
+ conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1d");
+ conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "7d");
+
+ startCluster();
+ SCMSecurityProtocol securityProtocol = getScmSecurityProtocol();
+ List<ManagedSecretKey> keysInitial = securityProtocol.getAllSecretKeys();
+ LOG.info("Keys before fail over: {}.", keysInitial);
+
+ // turn the current SCM leader off.
+ StorageContainerManager activeSCM = cluster.getActiveSCM();
+ cluster.shutdownStorageContainerManager(activeSCM);
+ // wait for
+ cluster.waitForSCMToBeReady();
+
+ List<ManagedSecretKey> keysAfter = securityProtocol.getAllSecretKeys();
+ LOG.info("Keys after fail over: {}.", keysAfter);
+
+ assertEquals(keysInitial.size(), keysAfter.size());
+ for (int i = 0; i < keysInitial.size(); i++) {
+ assertEquals(keysInitial.get(i), keysAfter.get(i));
+ }
+ }
+
+ private void startCluster()
+ throws IOException, TimeoutException, InterruptedException {
+ OzoneManager.setTestSecureOmFlag(true);
+ MiniOzoneCluster.Builder builder = MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setSCMServiceId("TestSecretKey")
+ .setScmId(scmId)
+ .setNumDatanodes(3)
+ .setNumOfStorageContainerManagers(3)
+ .setNumOfOzoneManagers(1);
+
+ cluster = (MiniOzoneHAClusterImpl) builder.build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ @NotNull
+ private SCMSecurityProtocol getScmSecurityProtocol() throws IOException {
+ UserGroupInformation ugi =
+ UserGroupInformation.loginUserFromKeytabAndReturnUGI(
+ testUserPrincipal, testUserKeytab.getCanonicalPath());
+ ugi.setAuthenticationMethod(KERBEROS);
+ SCMSecurityProtocol scmSecurityProtocolClient =
+ HddsServerUtil.getScmSecurityClient(conf, ugi);
+ assertNotNull(scmSecurityProtocolClient);
+ return scmSecurityProtocolClient;
+ }
+
+ private void enableBlockToken() {
+ conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 04/05: HDDS-8003. E2E integration test cases for block tokens (#4547)
Posted by du...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
duong pushed a commit to branch HDDS-7733-Symmetric-Tokens
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit b1880fffaeac9e7cf4791617c36f9f7144ad6c40
Author: Duong Nguyen <du...@gmail.com>
AuthorDate: Fri Apr 28 06:18:56 2023 -0700
HDDS-8003. E2E integration test cases for block tokens (#4547)
---
.../symmetric/DefaultSecretKeyVerifierClient.java | 30 +-
.../symmetric/SecretKeyVerifierClient.java | 2 +
.../security/token/ShortLivedTokenVerifier.java | 2 +-
.../hdds/security/token/TokenVerifierTests.java | 2 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 17 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 395 +++++++++++++++++++++
.../org/apache/hadoop/ozone/TestSecretKeysApi.java | 5 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 1 -
8 files changed, 434 insertions(+), 20 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java
index 56478793cb..c79ae6ef8b 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.time.Duration;
+import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -38,27 +39,38 @@ import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient;
/**
* Default implementation of {@link SecretKeyVerifierClient} that fetches
- * SecretKeys remotely via {@link SCMSecurityProtocol}.
+ * SecretKeys remotely via {@link SCMSecurityProtocol} and cache them locally.
*/
public class DefaultSecretKeyVerifierClient implements SecretKeyVerifierClient {
private static final Logger LOG =
LoggerFactory.getLogger(DefaultSecretKeyVerifierClient.class);
- private final LoadingCache<UUID, ManagedSecretKey> cache;
+ private final LoadingCache<UUID, Optional<ManagedSecretKey>> cache;
DefaultSecretKeyVerifierClient(SCMSecurityProtocol scmSecurityProtocol,
ConfigurationSource conf) {
Duration expiryDuration = parseExpiryDuration(conf);
Duration rotateDuration = parseRotateDuration(conf);
- long cacheSize = expiryDuration.toMillis() / rotateDuration.toMillis() + 1;
- CacheLoader<UUID, ManagedSecretKey> loader =
- new CacheLoader<UUID, ManagedSecretKey>() {
+ // if rotation is 1d, and each keys is valid for 7d before expiring,
+ // the expected number valid keys at any time is 7.
+ final long expectedValidKeys =
+ expiryDuration.toMillis() / rotateDuration.toMillis() + 1;
+ // However, we want to cache some expired keys as well, to avoid requesting
+ // SCM for recently expire secret keys. It makes sense to extend the
+ // secret keys cache by twice (e.g. 7 valid one and 7 recent expired).
+ final int secretKeyCacheMultiplier = 2;
+ long cacheSize = expectedValidKeys * secretKeyCacheMultiplier;
+ Duration cacheExpiry = expiryDuration.multipliedBy(
+ secretKeyCacheMultiplier);
+
+ CacheLoader<UUID, Optional<ManagedSecretKey>> loader =
+ new CacheLoader<UUID, Optional<ManagedSecretKey>>() {
@Override
- public ManagedSecretKey load(UUID id) throws Exception {
+ public Optional<ManagedSecretKey> load(UUID id) throws Exception {
ManagedSecretKey secretKey = scmSecurityProtocol.getSecretKey(id);
LOG.info("Secret key fetched from SCM: {}", secretKey);
- return secretKey;
+ return Optional.ofNullable(secretKey);
}
};
@@ -66,7 +78,7 @@ public class DefaultSecretKeyVerifierClient implements SecretKeyVerifierClient {
cacheSize, expiryDuration);
cache = CacheBuilder.newBuilder()
.maximumSize(cacheSize)
- .expireAfterWrite(expiryDuration.toMillis(), TimeUnit.MILLISECONDS)
+ .expireAfterWrite(cacheExpiry.toMillis(), TimeUnit.MILLISECONDS)
.recordStats()
.build(loader);
}
@@ -74,7 +86,7 @@ public class DefaultSecretKeyVerifierClient implements SecretKeyVerifierClient {
@Override
public ManagedSecretKey getSecretKey(UUID id) throws SCMSecurityException {
try {
- return cache.get(id);
+ return cache.get(id).orElse(null);
} catch (ExecutionException e) {
// handle cache load exception.
if (e.getCause() instanceof IOException) {
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java
index 59f49f72f1..08ed39d7f4 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.security.symmetric;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import javax.annotation.Nullable;
import java.util.UUID;
/**
@@ -26,5 +27,6 @@ import java.util.UUID;
* retrieve the relevant secret key to validate token authority.
*/
public interface SecretKeyVerifierClient {
+ @Nullable
ManagedSecretKey getSecretKey(UUID id) throws SCMSecurityException;
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java
index 4731f9149c..ae18305f9e 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenVerifier.java
@@ -111,7 +111,7 @@ public abstract class
ManagedSecretKey secretKey = secretKeyClient.getSecretKey(
tokenId.getSecretKeyId());
if (secretKey == null) {
- throw new BlockTokenException("Can't find the signer secret key " +
+ throw new BlockTokenException("Can't find the signing secret key " +
tokenId.getSecretKeyId() + " of the token for user: " +
tokenId.getUser());
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java
index 1c7085b35b..1ff9bee053 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java
@@ -153,7 +153,7 @@ public abstract class TokenVerifierTests<T extends ShortLivedTokenIdentifier> {
BlockTokenException ex = assertThrows(BlockTokenException.class, () ->
subject.verify("anyUser", token, cmd));
assertThat(ex.getMessage(),
- containsString("Can't find the signer secret key"));
+ containsString("Can't find the signing secret key"));
}
@Test
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
index 91d4b94404..4843c1c45e 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
@@ -74,6 +74,17 @@ public class KeyInputStream extends MultipartInputStream {
// BlockInputStream is only created here and not initialized. The
// BlockInputStream is initialized when a read operation is performed on
// the block for the first time.
+ Function<BlockID, BlockLocationInfo> retry;
+ if (retryFunction != null) {
+ retry = keyBlockID -> {
+ OmKeyInfo newKeyInfo = retryFunction.apply(keyInfo);
+ return getBlockLocationInfo(newKeyInfo,
+ omKeyLocationInfo.getBlockID());
+ };
+ } else {
+ retry = null;
+ }
+
BlockExtendedInputStream stream =
blockStreamFactory.create(
keyInfo.getReplicationConfig(),
@@ -82,11 +93,7 @@ public class KeyInputStream extends MultipartInputStream {
omKeyLocationInfo.getToken(),
verifyChecksum,
xceiverClientFactory,
- keyBlockID -> {
- OmKeyInfo newKeyInfo = retryFunction.apply(keyInfo);
- return getBlockLocationInfo(newKeyInfo,
- omKeyLocationInfo.getBlockID());
- });
+ retry);
partStreams.add(stream);
}
return partStreams;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestBlockTokens.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestBlockTokens.java
new file mode 100644
index 0000000000..a55e608fbf
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestBlockTokens.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.XceiverClientFactory;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
+import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory;
+import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl;
+import org.apache.hadoop.ozone.client.io.KeyInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.rpc.RpcClient;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ratis.util.ExitUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_CHECK_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION;
+import static org.apache.hadoop.hdds.StringUtils.string2Bytes;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.BLOCK_TOKEN_VERIFICATION_FAILED;
+import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
+import static org.apache.ozone.test.GenericTestUtils.assertExceptionContains;
+import static org.apache.ozone.test.GenericTestUtils.waitFor;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Integration test to verify block tokens in a secure cluster.
+ */
+@InterfaceAudience.Private
+public final class TestBlockTokens {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestBlockTokens.class);
+ private static final String TEST_VOLUME = "testvolume";
+ private static final String TEST_BUCKET = "testbucket";
+ private static final String TEST_FILE = "testfile";
+ private static final int ROTATE_DURATION_IN_MS = 3000;
+ private static final int EXPIRY_DURATION_IN_MS = 10000;
+ private static final int ROTATION_CHECK_DURATION_IN_MS = 100;
+
+ @Rule
+ public Timeout timeout = Timeout.seconds(180);
+
+ private static MiniKdc miniKdc;
+ private static OzoneConfiguration conf;
+ private static File workDir;
+ private static File ozoneKeytab;
+ private static File spnegoKeytab;
+ private static File testUserKeytab;
+ private static String testUserPrincipal;
+ private static String host;
+ private static String clusterId;
+ private static String scmId;
+ private static MiniOzoneHAClusterImpl cluster;
+ private static OzoneClient client;
+ private static BlockInputStreamFactory blockInputStreamFactory =
+ new BlockInputStreamFactoryImpl();
+
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
+
+ ExitUtils.disableSystemExit();
+
+ workDir =
+ GenericTestUtils.getTestDir(TestBlockTokens.class.getSimpleName());
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+
+ startMiniKdc();
+ setSecureConfig();
+ createCredentialsInKDC();
+ setSecretKeysConfig();
+ startCluster();
+ client = cluster.newClient();
+ createTestData();
+ }
+
+ private static void createTestData() throws IOException {
+ client.getProxy().createVolume(TEST_VOLUME);
+ client.getProxy().createBucket(TEST_VOLUME, TEST_BUCKET);
+ byte[] data = string2Bytes(RandomStringUtils.randomAlphanumeric(1024));
+ OzoneBucket bucket = client.getObjectStore().getVolume(TEST_VOLUME)
+ .getBucket(TEST_BUCKET);
+ try (OzoneOutputStream out = bucket.createKey(TEST_FILE, data.length)) {
+ org.apache.commons.io.IOUtils.write(data, out);
+ }
+ }
+
+ @AfterClass
+ public static void stop() {
+ miniKdc.stop();
+ IOUtils.close(LOG, client);
+ if (cluster != null) {
+ cluster.stop();
+ }
+ DefaultConfigManager.clearDefaultConfigs();
+ }
+
+ @Test
+ public void blockTokensHappyCase() throws Exception {
+ ManagedSecretKey currentScmKey =
+ getScmSecretKeyManager().getCurrentSecretKey();
+ OmKeyInfo keyInfo = getTestKeyInfo();
+
+ // assert block token points to the current SCM key.
+ assertEquals(currentScmKey.getId(), extractSecretKeyId(keyInfo));
+
+ // and the keyInfo can be used to read from datanodes.
+ readDataWithoutRetry(keyInfo);
+
+ // after the rotation passes, the old token is still usable.
+ waitFor(
+ () -> !Objects.equals(getScmSecretKeyManager().getCurrentSecretKey(),
+ currentScmKey),
+ ROTATION_CHECK_DURATION_IN_MS,
+ ROTATE_DURATION_IN_MS + ROTATION_CHECK_DURATION_IN_MS);
+ readDataWithoutRetry(keyInfo);
+ }
+
+ @Test
+ public void blockTokenFailsOnExpiredSecretKey() throws Exception {
+ OmKeyInfo keyInfo = getTestKeyInfo();
+ UUID secretKeyId = extractSecretKeyId(keyInfo);
+ readDataWithoutRetry(keyInfo);
+
+ // wait until the secret key expires.
+ ManagedSecretKey secretKey =
+ requireNonNull(getScmSecretKeyManager().getSecretKey(secretKeyId));
+ waitFor(secretKey::isExpired, ROTATION_CHECK_DURATION_IN_MS,
+ EXPIRY_DURATION_IN_MS);
+ assertTrue(secretKey.isExpired());
+ // verify that the read is denied because of the expired secret key.
+ StorageContainerException ex = assertThrows(StorageContainerException.class,
+ () -> readDataWithoutRetry(keyInfo));
+ assertEquals(BLOCK_TOKEN_VERIFICATION_FAILED, ex.getResult());
+ assertExceptionContains(
+ "Token can't be verified due to expired secret key", ex);
+ }
+
+ @Test
+ public void blockTokenOnExpiredSecretKeyRetrySuccessful() throws Exception {
+ OmKeyInfo keyInfo = getTestKeyInfo();
+ UUID secretKeyId = extractSecretKeyId(keyInfo);
+ readDataWithoutRetry(keyInfo);
+
+ // wait until the secret key expires.
+ ManagedSecretKey secretKey =
+ requireNonNull(getScmSecretKeyManager().getSecretKey(secretKeyId));
+ waitFor(secretKey::isExpired, ROTATION_CHECK_DURATION_IN_MS,
+ EXPIRY_DURATION_IN_MS);
+ assertTrue(secretKey.isExpired());
+ // verify that the read is denied because of the expired secret key.
+ readData(keyInfo, k -> {
+ try {
+ return getTestKeyInfo();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @Test
+ public void blockTokenFailsOnWrongSecretKeyId() throws Exception {
+ OmKeyInfo keyInfo = getTestKeyInfo();
+ // replace block token secret key id with wrong id.
+ for (OmKeyLocationInfoGroup v : keyInfo.getKeyLocationVersions()) {
+ for (OmKeyLocationInfo l : v.getLocationList()) {
+ Token<OzoneBlockTokenIdentifier> token = l.getToken();
+ OzoneBlockTokenIdentifier tokenId = token.decodeIdentifier();
+ tokenId.setSecretKeyId(UUID.randomUUID());
+ token.setID(tokenId.getBytes());
+ }
+ }
+
+ // verify that the read is denied because of the unknown secret key.
+ StorageContainerException ex =
+ assertThrows(StorageContainerException.class,
+ () -> readDataWithoutRetry(keyInfo));
+ assertEquals(BLOCK_TOKEN_VERIFICATION_FAILED, ex.getResult());
+ assertExceptionContains("Can't find the signing secret key", ex);
+ }
+
+ @Test
+ public void blockTokenFailsOnWrongPassword() throws Exception {
+ OmKeyInfo keyInfo = getTestKeyInfo();
+ // replace block token secret key id with wrong id.
+ for (OmKeyLocationInfoGroup v : keyInfo.getKeyLocationVersions()) {
+ for (OmKeyLocationInfo l : v.getLocationList()) {
+ Token<OzoneBlockTokenIdentifier> token = l.getToken();
+ token.setPassword(RandomUtils.nextBytes(100));
+ }
+ }
+
+ // verify that the read is denied because of the unknown secret key.
+ StorageContainerException ex =
+ assertThrows(StorageContainerException.class,
+ () -> readDataWithoutRetry(keyInfo));
+ assertEquals(BLOCK_TOKEN_VERIFICATION_FAILED, ex.getResult());
+ assertExceptionContains("Invalid token for user", ex);
+ }
+
+
+ private UUID extractSecretKeyId(OmKeyInfo keyInfo) throws IOException {
+ OmKeyLocationInfo locationInfo =
+ keyInfo.getKeyLocationVersions().get(0).getLocationList().get(0);
+ Token<OzoneBlockTokenIdentifier> token = locationInfo.getToken();
+ return token.decodeIdentifier().getSecretKeyId();
+ }
+
+ private OmKeyInfo getTestKeyInfo() throws IOException {
+ OmKeyArgs arg = new OmKeyArgs.Builder()
+ .setVolumeName(TEST_VOLUME)
+ .setBucketName(TEST_BUCKET)
+ .setKeyName(TEST_FILE)
+ .build();
+ return cluster.getOzoneManager()
+ .getKeyInfo(arg, false).getKeyInfo();
+ }
+
+ private void readDataWithoutRetry(OmKeyInfo keyInfo) throws IOException {
+ readData(keyInfo, null);
+ }
+
+ private void readData(OmKeyInfo keyInfo,
+ Function<OmKeyInfo, OmKeyInfo> retryFunc) throws IOException {
+ XceiverClientFactory xceiverClientManager =
+ ((RpcClient) client.getProxy()).getXceiverClientManager();
+ try (InputStream is = KeyInputStream.getFromOmKeyInfo(keyInfo,
+ xceiverClientManager,
+ false, retryFunc, blockInputStreamFactory)) {
+ byte[] buf = new byte[100];
+ int readBytes = is.read(buf, 0, 100);
+ assertEquals(100, readBytes);
+ }
+ }
+
+ private SecretKeyManager getScmSecretKeyManager() {
+ return cluster.getActiveSCM().getSecretKeyManager();
+ }
+
+ private static void setSecretKeysConfig() {
+ // Secret key lifecycle configs.
+ conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION,
+ ROTATION_CHECK_DURATION_IN_MS + "ms");
+ conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, ROTATE_DURATION_IN_MS + "ms");
+ conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, EXPIRY_DURATION_IN_MS + "ms");
+
+ // enable tokens
+ conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);
+ conf.setBoolean(HDDS_CONTAINER_TOKEN_ENABLED, true);
+ }
+
+ private static void createCredentialsInKDC() throws Exception {
+ ScmConfig scmConfig = conf.getObject(ScmConfig.class);
+ SCMHTTPServerConfig httpServerConfig =
+ conf.getObject(SCMHTTPServerConfig.class);
+ createPrincipal(ozoneKeytab, scmConfig.getKerberosPrincipal());
+ createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal());
+ createPrincipal(testUserKeytab, testUserPrincipal);
+ }
+
+ private static void createPrincipal(File keytab, String... principal)
+ throws Exception {
+ miniKdc.createPrincipal(keytab, principal);
+ }
+
+ private static void startMiniKdc() throws Exception {
+ Properties securityProperties = MiniKdc.createConf();
+ miniKdc = new MiniKdc(securityProperties, workDir);
+ miniKdc.start();
+ }
+
+ private static void setSecureConfig() throws IOException {
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ host = InetAddress.getLocalHost().getCanonicalHostName()
+ .toLowerCase();
+
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name());
+
+ String curUser = UserGroupInformation.getCurrentUser().getUserName();
+ conf.set(OZONE_ADMINISTRATORS, curUser);
+
+ String realm = miniKdc.getRealm();
+ String hostAndRealm = host + "@" + realm;
+ conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+ conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm);
+ conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+ conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm);
+ conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+
+ ozoneKeytab = new File(workDir, "scm.keytab");
+ spnegoKeytab = new File(workDir, "http.keytab");
+ testUserKeytab = new File(workDir, "testuser.keytab");
+ testUserPrincipal = "test@" + realm;
+
+ conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ ozoneKeytab.getAbsolutePath());
+ conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY,
+ spnegoKeytab.getAbsolutePath());
+ conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+ ozoneKeytab.getAbsolutePath());
+ conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE,
+ spnegoKeytab.getAbsolutePath());
+ conf.set(DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY,
+ ozoneKeytab.getAbsolutePath());
+ }
+
+ private static void startCluster()
+ throws IOException, TimeoutException, InterruptedException {
+ OzoneManager.setTestSecureOmFlag(true);
+ MiniOzoneCluster.Builder builder = MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setSCMServiceId("TestSecretKey")
+ .setScmId(scmId)
+ .setNumDatanodes(3)
+ .setNumOfStorageContainerManagers(3)
+ .setNumOfOzoneManagers(1);
+
+ cluster = (MiniOzoneHAClusterImpl) builder.build();
+ cluster.waitForClusterToBeReady();
+ }
+
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
index 84423cabac..217b08e728 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
@@ -85,7 +85,7 @@ public final class TestSecretKeysApi {
.getLogger(TestSecretKeysApi.class);
@Rule
- public Timeout timeout = Timeout.seconds(1600);
+ public Timeout timeout = Timeout.seconds(180);
private MiniKdc miniKdc;
private OzoneConfiguration conf;
@@ -94,7 +94,6 @@ public final class TestSecretKeysApi {
private File spnegoKeytab;
private File testUserKeytab;
private String testUserPrincipal;
- private String host;
private String clusterId;
private String scmId;
private MiniOzoneHAClusterImpl cluster;
@@ -146,7 +145,7 @@ public final class TestSecretKeysApi {
private void setSecureConfig() throws IOException {
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- host = InetAddress.getLocalHost().getCanonicalHostName()
+ String host = InetAddress.getLocalHost().getCanonicalHostName()
.toLowerCase();
conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index bcdf751f4c..f8d6d3addc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -42,7 +42,6 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.Timer;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org