You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ma...@apache.org on 2022/04/24 03:00:20 UTC
[ozone] branch HDDS-3630 updated (30d3b4d1a3 -> 4297479842)
This is an automated email from the ASF dual-hosted git repository.
markgui pushed a change to branch HDDS-3630
in repository https://gitbox.apache.org/repos/asf/ozone.git
discard 30d3b4d1a3 HDDS-6541. [Merge rocksdb in datanode] Per-disk DB location management. (#3292)
omit 49be0c06f3 HDDS-6540. Add a Cache for per-disk RocksDB Instance. (#3283)
omit 3e91e79c82 HDDS-6486. [Merge rocksdb in datanode] Add new container schema v3 definitions. (#3253)
omit b22dfd7e86 HDDS-6428. Add prefix iterator support to RDBTable. (#3176)
omit acdc4ca569 HDDS-6404. Format table key according to schema in KeyValueContainerData. (#3172)
omit 1146247779 HDDS-5923. Add DBHandle abstraction over ReferenceCountedDB. (#2794)
omit 3c38c3c4eb HDDS-5921. [Merge rocksdb in datanode]Decouple DatanodeStore impl classes from container. (#2791)
omit 31b415eb92 HDDS-5920. Allow to add ColumnFamilyOptions to tables. (#2790)
add 898ccbb46f HDDS-6376. Docs: Fix classpath for ofs and o3fs (#3138)
add 53d25b92ee HDDS-6307. Improve processing and memory efficiency of full container reports (#3085)
add 6e117f6488 HDDS-6395. Intermittent failure in TestReconScmHASnapshot.testScmHASnapshot (#3149)
add 2ed5994db1 HDDS-6268. Include audit log in acceptance test bundle (#3045)
add 5f029d0619 HDDS-6317: Export ReconTaskStatus as Prometheus metrics (#3088)
add ba31f8ce76 HDDS-6333. Add a metric to record sequence number lag between Recon and OM (#3100)
add e9722b759b HDDS-6401. Fix flaky TestFilePerBlockStrategy.testWriteAndReadChunkMultipleTimes (#3152)
add d0cde3a161 HDDS-5712. make it configurable to trigger refresh datanode usage info before start a new balance iteration (#2944)
add b83c1f9587 HDDS-6244. ContainerBalancer metrics don't show updated values in JMX (#3049)
add e716c1810e HDDS-6321. Avoid refresh pipeline for key lookup in checkAcls (#3103)
add fe6f060d89 HDDS-6370. Document connecting IDE to Ozone on Kubernetes (#3132)
add 94598d1cf4 HDDS-6420. Ratis and Standalone ReplicationConfig should return a static instance (#3165)
add 5217134f9a HDDS-6360. Create stack dump on acceptance test failure (#3154)
add b439802f5d HDDS-6417 - Fix console output message when container balancer does not start (#3164)
add bf38696d06 HDDS-6423. Fix typo in TestTools.md (#3170)
add 2a1b6ba234 HDDS-6379. Not deducting the STANDALONE pipelines when counting pipelines on each datanode to check the pipeline limit (#3158)
add 58207a20df HDDS-6153. Add simple integration test to the read-replicas debug tool. (#3065)
add 1c58d79c11 HDDS-6415 - add Over-Utilized and Under-Utilized DN details in debug log (#3171)
add 5587023a23 HDDS-6357. RenameKey request has memory leak (#3121)
add 2a776a3745 HDDS-6306 - Fix picocli warnings in ozone container balancer help (#3162)
add ea06d26afd HDDS-6419. Provide better error message for malformed auth header (#3167)
add 759ac57a66 HDDS-6407. Unwrap RemoteException in getClientProtocol (#3157)
add 933a19c90a HDDS-5656. Move old objects to delete table on overwriting multipart objects (#2813)
add 1c414bf10e HDDS-6299. Remove Log4J2 from Ozone FS shaded jars (#3073)
add a720211b6b HDDS-6267. Recon is accessing mock classes. (#3084)
add 27aad3c8b2 HDDS-6391. ClientVersions and DatanodeVersions class to define an enum with version and description (#3155)
add 7cfd846027 HDDS-6251. EC: Smoketest for ozone admin datanode expects exactly 3 nodes (#3182)
add b01138c27f HDDS-6392. Introduce OzoneManagerVersion enum (#3159)
add bb212de993 HDDS-6418. Datanode usage info uses wrong version number (#3173)
add f826d124aa HDDS-6431. Fix usedBytes for FSO bucket (#3178)
add 5a1babfd45 HDDS-6439. Minor perf optimization in the version enums (#3189)
add 4454e6b368 HDDS-6095. Skip slow integration tests, allow repeating known flaky tests 5x in CI (#3148)
add 5f3f9fae17 HDDS-4171. Typo in MiniOzoneCluster: DEFAULT_PIPELIME_LIMIT (#3191)
add c2a67aca1b HDDS-6450. [FSO] Fix correctedSpace for usedBytes in commitKey (#3193)
add 74d92c86be HDDS-6213. Pluggable OzoneManager request handling hooks (#3104)
add 909b0d23f7 HDDS-6079. Replace Freon RK with OCKG/OCKV in acceptance tests (#2916)
add be2021a369 HDDS-6470. Fix TestOzoneManagerHAWithData#testOMRestart() (#3213)
add ea296fe7f8 HDDS-6468. Set correct resource type for ACL checks in BucketAcl and KeyAcl requests (#3209)
add 416439d7a2 HDDS-5867. Update quota when deleting open keys (#3206)
add 97d8ed73bc HDDS-6432. Smoketest for read-replicas tool expects exactly 3 datanodes (#3187)
add 64081985ab HDDS-6464. Turn off cross-check fail-fast for CI (#3208)
add ca24bdb911 HDDS-5228 Make OM FailOverProxyProvider work across threads (#3160)
add d97687cdbe HDDS-6087. Compute composite CRC file checksum using chunk checksums from DataNodes. (#2937)
add 4da4cc174f HDDS-6477. Intermittent failure in TestOzoneManagerHAMetadataOnly#testListVolumes (#3219)
add 61a313fd3b HDDS-6228. Update config keys for open key cleanup service (#3022)
add 042aa8b330 HDDS-3370. Cleanup old write-path of bucket in OM (#3183)
add 92995ec8ae HDDS-6261. OM crashes when trying to overwrite a key during upgrade downgrade testing (#3174)
add a8ed41ebd3 HDDS-6490. Replace deprecated --no-ansi option in acceptance tests (#3224)
add a5602a8b01 HDDS-6480. Add missing CleanupTableInfo to OM responses (#3220)
add 1baa6521d1 HDDS-6488. Intermittent failure in ozone debug read-replicas with one datanode STALE (#3223)
add 4ff630082b HDDS-6398. Add audit logs for DelegationToken (#3163)
add 2f2f52591c HDDS-6472. Errors in TestOzoneContainer integration test (#3216)
add cfb8bae20b HDDS-6393. Change the OzoneManager ServiceInfo message to carry the int version from OzoneManagerVersions. (#3211)
add 9f716b8aa7 HDDS-6482. Refactor Ratis Rpc and TLS config code. (#3222)
add 75f5501602 HDDS-6481. Add metrics for S3 gateway (#3221)
add b21a613146 HDDS-6473. Empty data in response for V0 GetSmallFile request (#3230)
add d35da4091d HDDS-6098. Add a metric to track amount of data committed. (#2913)
add e07c66ca27 HDDS-6425. OmMetadataManagerImpl#isBucketEmpty does not work on FSO buckets. (#3227)
add 605746617b HDDS-6497. S3 Gateway throws exception when checking jmx or prom pages from browser (#3228)
add ff9f523899 HDDS-6509. Checkstyle: Enable setterCanReturnItsClass in HiddenField (#3237)
add a8ef728055 HDDS-6499. Cleanup OMRequest class constructors. (#3248)
add 682be322fa HDDS-6367. ContainerBalancer shows incorrect iteration result sometimes (#3129)
add d1e39315f0 HDDS-6502. Blocks of old key versions are not deleted on key deletion (#3232)
add 0c4a01766b HDDS-6516. Shade Kotlin for Ozone Filesystem (#3246)
add 0ec6005ce7 HDDS-6429. getContainerReplicas should respect client version (#3231)
add 0bf2090078 HDDS-6505. TestContainerStateManagerIntegration consistently fails (#3234)
add adecbd1f92 HDDS-6362. Add aggregate metrics to ContainerBalancerMetrics (#3239)
add 0262c8ac3a HDDS-6466. Drop S3 Gateway bucket browser feature (#3241)
add 2eabcc5bd5 HDDS-5963. Implement ListBlock command (#3212)
add d93fbdfa64 HDDS-6397. Implement ContainerBalancer as an SCMService (#3153)
add 1d40e55e91 HDDS-6538. Update Spring to 5.2.20 to fix CVE-2022-22965 aka Spring4shell. (#3261)
add a6b09e777e HDDS-6456 Update RocksDB Version to 7.0.4 (#3199)
add c23aae0d93 HDDS-6435. Add read lock waiting and held time metrics (#3180)
add 68270e46bf HDDS-6414. Ozone doesn’t reclaim space after deletion of volume via ofs (#3175)
add 15a09aa127 HDDS-6523. Refine OMClientRequest Classes to handle bucket layout better. (#3265)
add 0f04fa7b17 HDDS-6537. Deprecate the renameKeys API (#3260)
add aa68b9abed HDDS-5119. Recon file count by size page has incorrect data when keys are deleted (#3269)
add 805b5e1d3b HDDS-6545. OzoneManagerUtils#getBucketLayout should not suppress bucket key lookup errors. (#3277)
add 76f809f3ec HDDS-6436. Add write lock waiting and held time metrics (#3271)
add 473627b8f5 HDDS-6554. Have the datanode heartbeat include queued command counts (#3275)
add e81ab0a3f7 HDDS-6564. DirectoryDeletionService should use getFileTable to handle bucket delete (#3284)
add 55d5ecbda2 HDDS-6552. Bump Jackson Databind (#3273)
add 64d67668d2 HDDS-5862. Datanode to Recon heartbeat interval independently configurable (#3245)
add de0b1f9b81 HDDS-6528. Adding Unit-Test cases for S3-Gateway Bucket-Endpoint Metrics. (#3263)
add 4c1ba2b998 HDDS-6535. Ozone TokenRenewer not found (#3259)
add 141ced1196 HDDS-6579. Add commandline argument for Container Info command. (#3306)
add 22cdd95a9e HDDS-6141. terminate om if statemachine is shut down by ratis (#2953)
add c583b3e7c6 HDDS-6162. Limit OM DoubleBuffer pending requests to avoid taking too much memory (#3024)
add d8a21a2bdb HDDS-6593. Bump node to v16.14.2 for Recon (#3316)
add 0b4af02bfb HDDS-6591. Javadoc - update StorageContinerLocationProtocol for change of qualified class name (#3314)
add 2851ea53f0 HDDS-5339. Document OM bootstrap and decommission process (#3299)
add 8f98af911c HDDS-6547. om snapshot failed file not deleted. (#3268)
add fe49f69a33 HDDS-6346. OmFailoverProxyProvider cleanup in loadConfigs for legacy service ids code (#3313)
add d5eb702c68 HDDS-6587. SCM Web UI is not displaying HEALTHY_READONLY header in node status (#3311)
add a0684e26c8 HDDS-6574. Set owner of buckets created via S3 Gateway to actual user rather than `s3g`; print LinkBucket owner field on the client (#3298)
add 3fd7dc6c77 HDDS-6525. Add audit log for S3Gateway (#3252)
add 0b6f46714f HDDS-6562. Exclude specific operations in Audit log (#3289)
add 172947815b HDDS-6532. Enable READ audit log by default (#3255)
add 204584b395 HDDS-6515. Intermittent failure in TestRootedOzoneFileSystem (#3322)
add cec910568c HDDS-5003. Introduce EC replication type (#2064)
add a51efc541b HDDS-4954. Add replicaIndex to the RPC protocols (#2055)
add 23fe4f5e56 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add a915dbfdd5 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add c050b8f67e HDDS-5011. Introduce ECReplicationConfig and Java based ReplicationConfig implementation (#2068)
add aeedef8c7b Merge branch 'master' into the EC branch
add 1d5c593319 HDDS-5145. Extend Pipline/ReplicationConfig refactor with ECReplicationConfig (#2206)
add 132911c504 HDDS-5222. ReplicaIndex in Pipeline should be deserialized in the protobuf message (#2241)
add 9c4008d540 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add ad790b660e HDDS-4883. Persist replicationIndex on datanode side (#2069)
add c960ec7c5a Merge remote-tracking branch 'origin/HDDS-3816-ec' into HDDS-3816-ec
add f904539c0f Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add fbea53d4f5 Merge branch 'master' into EC branch
add b6f960daaa HDDS-5313. ContainerInfo should use ReplicationConfig (#2320)
add 9e0a2fc4e2 HDDS-5247. EC: Create ECReplicationConfig on client side based on input string (#2331)
add 5e42c31933 Merge branch 'master' into EC branch.
add 27f57b7d3c Fixed conflict issue as part of merge branch 'master' into EC branch.
add f53a93e6a9 HDDS-5343 : EC: Add ECReplicationConfig into KeyInfo proto. (#2340). Contributed by Uma Maheswara Rao G.
add 206c9358a5 HDDS-4892. EC: Implement basic EC pipeline provider (#2353)
add 3a94742765 HDDS-5375. EC: Extend PipelineManager.createPipeline API to support excluded nodes (#2378)
add 4115827ff9 HDDS-4940 : EC: Implement the ECKeyOutputStream which should handle the EC mode writes. (#2335)
add cd781b9785 HDDS-5174. EC: Allow EC blocks to be requested from OM (#2414)
add a0f6881a28 Merge branch 'master' into ec-merge-master
add 0c5c859b99 Fix issues caused by merging master into branch
add 2ef364f47b HDDS-5509. EC: Add missing break in switch statement when requesting EC blocks (#2468)
add a64945bd7c HDDS-5325. EC: Add configuration to set an EC container placement policy (#2483)
add 942a23f14e Merge remote-tracking branch 'origin/master' into merge-master-0803
add f3ce6c7801 HDDS-5510: allocateContainer should handle ec replication config (#2490)
add 6eba2ff17d HDDS-5543: OMKeyRequest#createFileInfo should handle ECReplicationConfig (#2501)
add 20851dc199 HDDS-5561. EC: ECReplicationConfig should be immutable (#2516)
add d2dca9b079 HDDS-5562. EC: ContainerPlacementPolicyFactory#getPolicyInternal() should not be public. (#2517)
add 94814e5dba HDDS-5425: Client side pipeline cache key should use host and port combination (#2510)
add 0e6e00e97e Merge branch 'master' into ec-branch
add db35f8c003 HDDS-4942. EC: Implement ECBlockInputStream to read a single EC BlockGroup (#2507)
add d6dc278195 EC: Make ECReplicationConfig stored as bucket level attributes. (#2401)
add 823b505ab5 HDDS-5326. EC: Create a new as many racks as possible placement policy for EC (#2497)
add 90c4d9286d HDDS-5470 : EC: Add padding and generate parity if the last stripe is not full (#2455)
add 2df6c9caf5 Merge branch 'master' into ec branch
add e08d5009ab Removed SCMPipelineManager.java which should have been removed via HDDS-5554, but was somehow missing on an earlier master to ec branch merge.
add 6b17c7570d HDDS-5745. EC: Pipeline builder should copy replica indexes from original pipeline (#2642)
add 1d3423de76 HDDS-5477. EC: commit key should consolidate and create one keyLocationInfo per blockGrp (#2648)
add ebdd1f5e3d HDDS-5682: EC: Provide replication config option from CLI when creating bucket. (#2639)
add b55107fe76 HDDS-5550. EC: Adapt KeyInputStream to read EC Blocks (#2653)
add 0a0e62e002 EC: Add Codec and Stripe Size to ECReplicationConfig (#2674)
add fc17518932 HDDS-5784: EC: In BasicRootedOzoneClientAdapterImpl, Inherit bucket default replication config only in the case of EC. (#2698)
add 5cf251654e HDDS-5741. EC: Remove hard coded chunksize and get from from ReplicationConfig (#2687)
add 831b3faa27 HDDS-5833: EC: Fix TestRootedOzoneFileSystem.testBucketDefaultsShouldBeInheritedToFileForEC failure in branch (#2718)
add 7a67f63429 HDDS-5822 : Writing a large buffer to an EC file duplicates first chunk in block 1 and 2 (#2714)
add a16ee9d2fb Merge branch 'master' into HDDS-3816-ec
add aee577c171 HDDS-5825: EC: ECKeyOutputStream#close fails if we write the partial chunk (#2720)
add f478b96360 HDDS-5832. EC: ECKeyOutputStream persists blocks in random order (#2717)
add d37fe6901b HDDS-5552. EC: Implement seek on ECBlockInputStream (#2723)
add 38e4cca273 HDDS-5755 EC: Refactor ECBlockOutputStreamEntry to accommodate all block group related ECBlockOuputStreams. (#2702)
add 914ce67af3 HDDS-5364: EC: Adopt EC related utility from Hadoop source repository (#2733)
add f5dbe00309 Hdds 5491: EC: Write should handle node failures. (#2767)
add c9fa9bdf0f HDDS-5874 EC: Integrate the Codec changes into EC Streams. (#2777)
add 818118a7ad HDDS-5948: EC: Fix the compile issue in TestOzoneECClient (Due to concurrent commits) (#2812)
add e82682fe8b HDDS-5551. EC: Implement an Input Stream to reconstruct EC blocks on demand (#2797)
add f85ac376e6 HDDS-5950. EC: Implement seek on ECBlockReconstructedStripeInputStream (#2818)
add 438799ba75 HDDS-5953: EC: Review the current flush API and clean up (#2828)
add 52f05fc349 HDDS-5951. EC: ECBlockReconstructedStripeInputStream should handle block read failures and continue reading (#2831)
add 67043109c0 Merging master into EC branch
add d5ac6738b7 HDDS-6012: EC: Fix TestOzoneShellHA failures post master merge with EC branch
add 9a7025a39b HDDS-6035: EC: Fix TestOmMetrics in merge branch
add a0349bd9a9 HDDS-6029: EC: Fix the replication config handling in OMDirectoryCreateRequest#dirKeyInfoBuilderNoACL (#1)
add 685442c6b8 HDDS-6001. EC: Create ECBlockReconstructedInputStream to wrap ECBlockReconstructedStripeInputStream (#2848)
add 93fadc9789 HDDS-5979: EC: Change CLI bucket default replication option name to -type (#2873)
add 6e7d0dc206 HDDS-6009. EC: Optimize ECBlockReconstructedStripeInputStream where there are no missing data indexes (#2872)
add 1b9724d60a HDDS-5955: EC: Track the failed servers to add into the excludeList when invoking allocateBlock (#2849)
add 460aa9b89b HDDS-6020: EC: Provide set replicationConfig option to bucket (#2880)
add a1d7292bcf HDDS-6010. EC: Create ECBlockInputStreamProxy to choose between reconstruction and normal reads (#2889)
add 73bddf36d0 Updates to address compile issues after merge
add 3007b5c68d HDDS-6081. EC: Fix ReplicationConfig related test failures that happened due to merging HDDS-5997 to the EC branch
add c6604833d1 HDDS-5952. EC: ECBlockReconstructedStripeInputStream should read from blocks in parallel (#2899)
add 86f735418a HDDS-6036: EC: handleStripeFailure should retry (#2910)
add f8fd2ff66b HDDS-6065: EC: Provide CLI option to reset the bucket replication config (#2927)
add ba192b1de1 HDDS-6132: EC: HandleStripeFailure should not release the cachebuffers. (#2940)
add 3eb55db014 HDDS-5949. EC: Create reusable buffer pool shared by all EC input and output streams (#2929)
add 5856411168 HDDS-6179. EC: Provide toString in ECReplicationConfig (#2986)
add 07b2338525 HDDS-6180. EC: Pipeline creator should ignore creating pipelines for ZERO factor (#2985)
add aaf6f17045 HDDS-6021: EC: Client side exclude nodes list should expire after certain time period or based on the list size. (#2973)
add d0467bac1d HDDS-6220. EC: Introduce a gRPC client implementation for EC with really async WriteChunk and PutBlock. (#3016)
add b40ab3f97b HDDS-6172: EC: Document the Ozone EC (#3006)
add 78ccdd540c HDDS-6131: EC: Replication config from bucket should be refreshed in o3fs (#3008)
add 6b5c01c843 HDDS-6184. EC: put command should create EC key if bucket is EC (#2990)
add 4d660c4d79 HDDS-6233. EC: Bucket does not display correct EC replication details (#3025)
add 2e60a4f8cb Merge master to EC Branch
add 58f8e30538 HDDS-6239. ozonesecure-mr failing with No URLs in mirrorlist (#3029)
add 78ca097eac HDDS-6083: EC: Fix flakyness of tests around nodefailures (#3021)
add dfa8141937 HDDS-6237: EC: putBlock should pass close flag true on end of block group/close file. (#3026)
add ee630e2995 HDDS-6240. EC: Container Info command with json switch fails for EC containers (#3033)
add 8e52561eab HDDS-6251. EC: Smoketest for ozone admin datanode expects exactly 3 nodes (#3037)
add ddcfc521af HDDS-6252. EC: Datanode Chunk Validator fails on encountering EC pipeline (#3035)
add ca6e4c3599 HDDS-6255. EC: Replication Manager should skip EC Containers (#3038)
add 5d2fa94949 HDDS-6256. EC: Parity blocks are incorrectly padded with zeros to the chunk size (#3043)
add 9a0f812b3b HDDS-6231. EC: Create EC acceptance test environment and some basic tests (#3030)
add 722c444aea HDDS-6250. EC: Add replica index to the output in the container info command (#3041)
add 29c69b109b HDDS-6272: EC: Recon UI failing because of EC replication factor (#3067)
add 22e34820cf HDDS-6258. EC: Read with stopped but not dead nodes gives IllegalStateException rather than InsufficientNodesException (#3048)
add c4aec7bb97 HDDS-6259. EC: Pipelines for closed containers should contain correct replica indexes (#3062)
add dafdb23255 HDDS-6295. EC: Fix unaligned stripe write failure due to length overflow. (#3076)
add 2d63cc4ec9 HDDS-6194. EC: Freon ockg support EC write (#2996)
add 79f60c4d5c HDDS-6271. EC: Apply fix for HDFS-16422 to the Ozone EC libraries (#3087)
add aebaff55a2 Merge master to EC branch HDDS-3816-ec
add b2a33b759e HDDS-6320: EC: Fix new checkstyle rule warnings in EC branch
add 4b9d38763a HDDS-6294. EC: Make cluster-wide EC configuration take effect (#3089)
add 550eadc970 HDDS-6308. EC: Handle Replication Factor to consider EC Config in Recon UI (#3078)
add 6f64eb77d5 HDDS-6342. EC: Fix large write with multiple stripes upon stripe failure. (#3108)
add 60a41bc85e HDDS-6249. EC: Fix todo items in TestECKeyOutputStream (#3102)
add 1d23619164 HDDS-6188: EC: Adapt java side native coder classes from hadoop. (#3046)
add 387fb1a2c1 HDDS-6232. EC: Update help strings for replication config (#3113)
add 0a83e7018e HDDS-6341: EC: Fix the race condition in TestECBlockReconstructedStripeInputStream. (#3111)
add aac07ef790 HDDS-6347. EC: Freon randomKeys EC key support (#3112)
add 6b8d935f73 HDDS-6319. EC: Fix read big file failure with EC policy 10+4. (#3094)
add b9adbcd89a HDDS-6348: EC: PartialStripe failure handling logic is writing padding bytes also to DNs (#3124)
add b90f00192d HDDS-6372. EC: Do not throw NotImplementedException in flush() (#3133)
add 78ca327e77 HDDS-6358. EC: Refactor ECKeyOutputStream#write() (#3120)
add 9e3cf98a30 HDDS-6279. EC: Calculate EC replication correctly when updating bucket usage (#3136)
add e675e3ba3d HDDS-6380. EC: Key Info command should not display legacy replication fields as they duplicate ReplicationConfig (#3141)
add 9ee6d72f31 HDDS-6373. EC: Exclude pipeline upon container close instead of exclude DNs. (#3137)
add 38409ea5a0 HDDS-6364. EC: Discard pre-allocated blocks to eliminate worthless retries. (#3127)
add 9680edf850 HDDS-6384. EC: Ensure EC container usage is updated correctly when handling reports (#3147)
add 58eac935cd HDDS-6185: EC: EC keys can't be created via S3 interfaces (#3150)
add 8cff9fd2ad HDDS-6368. EC: Fix broken future chain and cleanup unnecessary validation function. (#3128)
add 8776430812 HDDS-6382. EC: ListPipelines command should consider EC Containers (#3161)
add 28482bf2b9 HDDS-6422. EC: Fix too many idle threads during reconstruct read. (#3168)
add 074203fc0a HDDS-6437. EC: Avoid allocating buffers in EC Reconstruction Streams until first read (#3181)
add 9675863705 HDDS-6413. EC: OzoneManagerRequestHandler needs to handle ECReplicationConfig (#3198)
add 06f1eb5ad6 HDDS-6248. EC: Container list command should allow filtering of EC containers (#3179)
add 03ff3f9592 HDDS-6445. EC: Fix allocateBlock failure due to inaccurate excludedNodes check. (#3190)
add af48c537a9 HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys (#3194)
add 217c1911b2 EC: Check isFullCell inside handleDataWrite (#3202)
add 9cda6a1a06 HDDS-6460. EC: Adjust requested size by EC DataNum in WritableECContainerProvider (#3203)
add 04c62c53f2 HDDS-6411. EC: OmMultipartKeyInfo needs to handle ECReplicationConfig (#3210)
add fc6ac0c48c HDDS-6448. EC: Reconstructed Input Streams should free resources after reading to end of block (#3197)
add f386662a37 HDDS-6381. EC: Improve exception message in ByteBufferEncodingState (#3143)
add 9ec90f5820 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec-merge-master
add c55f30edf0 HDDS-6412: EC: Handle ECReplicationConfig in initiateMultipartUpload API (#3184)
add 16c3e99eae HDDS-6478. EC: Fix ISA-l load hadoop native lib UnsatisfiedLinkError (#3218)
add 143315c89b HDDS-6409. EC: OzoneMultipartUpload needs to handle ECReplicationConfig (#3279)
add d9529b32ea HDDS-6434: EC: Overwriting an EC key with a Ratis key fails (#3274)
add df96b86c8d HDDS-6557. EC: Execute S3 acceptance tests with EC (#3281)
add e9b6215210 Merge master to EC Branch HDDS-3816-ec
add e38607b0cd HDDS-6565. EC: Unify replication-related CLI params (#3293)
add 6c6b740a47 HDDS-6209. EC: [Forward compatibility issue] New client to older server could fail due to the unavailability for client default replication config (#3267)
add 70b547be23 HDDS-5909. EC: Onboard EC into upgrade framework (#3262)
add cb0a84b27c Merge master to EC branch HDDS-3816-ec
add 96669856ad Merge EC branch HDDS-3816-ec to master
add 681d0a8f5e HDDS-6471. Only repeat flaky tests (#3247)
add dc9da7d405 HDDS-6606. Rat check failure ignored for ozone-annotation-processing (#3321)
add be5888b907 HDDS-6580. Introduce KEY_PATH_LOCK under OzoneManagerLock.Resource (#3308)
add 4345fb1c58 HDDS-6189. Intermittent failure in TestDirectoryDeletingServiceWithFSO (#3328)
add 03b0a61958 HDDS-6583. Introduce lock strategy pattern implementations based on configuration flags (#3310)
add 0bc3321950 HDDS-5920. Allow to add ColumnFamilyOptions to tables. (#2790)
add fcf74fcd96 HDDS-5921. [Merge rocksdb in datanode]Decouple DatanodeStore impl classes from container. (#2791)
add 83bb7034b4 HDDS-5923. Add DBHandle abstraction over ReferenceCountedDB. (#2794)
add db4734237b HDDS-6404. Format table key according to schema in KeyValueContainerData. (#3172)
add 482886d77b HDDS-6428. Add prefix iterator support to RDBTable. (#3176)
add d693014b02 HDDS-6486. [Merge rocksdb in datanode] Add new container schema v3 definitions. (#3253)
add 4e7a0c395b HDDS-6540. Add a Cache for per-disk RocksDB Instance. (#3283)
add 7c8d51dca5 HDDS-6541. [Merge rocksdb in datanode] Per-disk DB location management. (#3292)
add 2646a8db21 Update layout version to 4.
add 4297479842 Update another layout version to 4.
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (30d3b4d1a3)
\
N -- N -- N refs/heads/HDDS-3630 (4297479842)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/post-commit.yml | 13 +-
dev-support/annotations/pom.xml | 114 +++
.../RequestFeatureValidatorProcessor.java | 289 ++++++
.../apache/ozone/annotations}/package-info.java | 8 +-
.../services/javax.annotation.processing.Processor | 19 +-
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../scm/storage/BlockExtendedInputStream.java} | 21 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 58 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 177 ++++
.../hdds/scm/storage/ExtendedInputStream.java | 91 ++
.../storage/DummyBlockInputStreamWithRetry.java | 2 +-
hadoop-hdds/common/pom.xml | 13 +-
...DatanodeVersions.java => ComponentVersion.java} | 24 +-
.../org/apache/hadoop/hdds/DatanodeVersion.java | 71 ++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 6 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 1 +
.../hdds/client/DefaultReplicationConfig.java | 134 +++
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++++
.../hadoop/hdds/client/RatisReplicationConfig.java | 41 +-
.../hadoop/hdds/client/ReplicationConfig.java | 57 +-
.../hadoop/hdds/client/ReplicationFactor.java | 16 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 41 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 66 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 11 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 63 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 44 +-
.../ContainerCommandResponseBuilders.java | 13 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 20 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 78 ++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 41 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 5 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 74 ++
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 55 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 3 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 30 +-
.../audit/{AuditLoggerType.java => S3GAction.java} | 41 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 35 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 49 +-
.../common/src/main/resources/ozone-default.xml | 145 ++-
.../hdds/TestComponentVersionInvariants.java | 98 ++
.../hdds/client/TestECReplicationConfig.java | 91 ++
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +++-
.../client/TestReplicationConfigValidator.java | 16 +-
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 11 +-
.../hdds/scm/container/TestContainerInfo.java | 86 ++
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/common/helpers/TestExcludeList.java | 67 ++
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 34 +-
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 72 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 54 +-
.../ozone/container/ContainerTestHelper.java | 28 +
.../common/src/test/resources/auditlog.properties | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../common/statemachine/DatanodeStateMachine.java | 31 +
.../common/statemachine/StateContext.java | 26 +
.../commandhandler/CommandDispatcher.java | 16 +
.../commandhandler/CommandHandler.java | 10 +
.../CreatePipelineCommandHandler.java | 13 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 11 +
.../DeleteContainerCommandHandler.java | 5 +
.../ReplicateContainerCommandHandler.java | 5 +
.../states/datanode/RunningDatanodeState.java | 8 +-
.../states/endpoint/HeartbeatEndpointTask.java | 34 +-
.../states/endpoint/RegisterEndpointTask.java | 6 +-
.../transport/server/ratis/XceiverServerRatis.java | 18 +-
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 42 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 10 +-
.../replication/ReplicationSupervisor.java | 2 +-
.../commands/RefreshVolumeUsageCommand.java | 57 ++
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../container/common/impl/TestHddsDispatcher.java | 20 +
.../common/statemachine/TestStateContext.java | 36 +
.../TestCreatePipelineCommandHandler.java | 36 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 26 +-
.../container/keyvalue/TestKeyValueHandler.java | 7 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 1 -
.../upgrade/TestDatanodeUpgradeToScmHA.java | 39 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 +
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++++
hadoop-hdds/docs/content/feature/OM-HA.md | 37 +
hadoop-hdds/docs/content/interface/O3fs.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Ofs.md | 2 +-
hadoop-hdds/docs/content/interface/S3.md | 7 -
hadoop-hdds/docs/content/interface/S3.zh.md | 7 -
.../docs/content/start/StartFromDockerHub.md | 6 -
.../docs/content/start/StartFromDockerHub.zh.md | 5 -
hadoop-hdds/docs/content/tools/TestTools.md | 4 +-
hadoop-hdds/docs/content/tools/TestTools.zh.md | 4 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 ++
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 ++
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +++
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 ++
.../rawcoder/AbstractNativeRawEncoder.java | 99 ++
.../rawcoder/ByteArrayDecodingState.java | 132 +++
.../rawcoder/ByteArrayEncodingState.java | 105 ++
.../rawcoder/ByteBufferDecodingState.java | 143 +++
.../rawcoder/ByteBufferEncodingState.java | 110 ++
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 ++
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 ++
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 ++
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 ++
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 ++
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 ++
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 ++
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 ++
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 ++
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 ++
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 ++++++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++++++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 ++
.../ozone/erasurecode/TestCodecRegistry.java | 159 +++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++++++++
.../apache/ozone/erasurecode}/package-info.java | 12 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 ++
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 ++
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +++
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +++
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++++++
.../rawcoder/TestRawErasureCoderBenchmark.java | 37 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 ++
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
hadoop-hdds/framework/pom.xml | 8 +
...lockLocationProtocolClientSideTranslatorPB.java | 12 +-
...inerLocationProtocolClientSideTranslatorPB.java | 60 +-
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 14 +
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 +
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 75 --
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 50 +-
.../hdds/utils/db/cache/PartialTableCache.java | 56 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 4 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 77 +-
.../src/main/proto/ScmAdminProtocol.proto | 3 +
hadoop-hdds/interface-client/pom.xml | 5 +
.../src/main/proto/DatanodeClientProtocol.proto | 3 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 16 +
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 1 +
.../container/AbstractContainerReportHandler.java | 130 ++-
.../hdds/scm/container/ContainerManagerImpl.java | 20 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ContainerReportHandler.java | 141 ++-
.../IncrementalContainerReportHandler.java | 18 +-
.../hdds/scm/container/ReplicationManager.java | 11 +-
.../scm/container/balancer/ContainerBalancer.java | 526 +++++++---
.../balancer/ContainerBalancerConfiguration.java | 18 +
.../balancer/ContainerBalancerMetrics.java | 163 +--
.../IllegalContainerBalancerStateException.java | 36 +-
...lidContainerBalancerConfigurationException.java | 37 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 ++++++
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 37 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 4 +-
...ffer.java => SCMHADBTransactionBufferStub.java} | 8 +-
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +-
...MockSCMHAManager.java => SCMHAManagerStub.java} | 42 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 14 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 15 +-
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 6 +-
.../hadoop/hdds/scm/metadata/PipelineCodec.java | 6 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 15 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 18 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 18 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 34 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 2 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 4 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 9 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +++
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 36 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 54 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 10 +
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 30 +
.../hdds/scm/pipeline/RatisPipelineProvider.java | 32 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 3 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 +++++
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 281 +++++-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 2 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 2 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +
.../hdds/scm/server/SCMClientProtocolServer.java | 148 ++-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 9 +
.../hdds/scm/server/StorageContainerManager.java | 25 +-
.../main/resources/webapps/scm/scm-overview.html | 1 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 24 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 7 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 21 +
.../hdds/scm/container/SimpleMockNodeManager.java | 14 +-
.../container/TestCloseContainerEventHandler.java | 8 +-
.../scm/container/TestContainerManagerImpl.java | 22 +-
.../scm/container/TestContainerReportHandler.java | 187 +++-
.../scm/container/TestContainerStateManager.java | 8 +-
.../TestIncrementalContainerReportHandler.java | 4 +-
.../hdds/scm/container/TestReplicationManager.java | 6 +-
.../scm/container/TestUnknownContainerReport.java | 4 +-
.../container/balancer/TestContainerBalancer.java | 233 +++--
.../algorithms/TestContainerPlacementFactory.java | 8 +
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +++---
.../hdds/scm/ha/TestReplicationAnnotation.java | 5 +
.../hdds/scm/ha/TestSequenceIDGenerator.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 6 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 24 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 3 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 58 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 +++
.../TestPipelineDatanodesIntersection.java | 13 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 168 ++-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 158 ++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 62 +-
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 ++
.../scm/pipeline/TestRatisPipelineProvider.java | 111 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 18 +-
.../pipeline/TestWritableECContainerProvider.java | 446 ++++++++
.../safemode/TestHealthyPipelineSafeModeRule.java | 20 +-
.../TestOneReplicaPipelineSafeModeRule.java | 10 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 22 +-
.../scm/server/TestSCMBlockProtocolServer.java | 10 +-
.../TestSCMHAUnfinalizedStateValidationAction.java | 8 +
.../testutils/ReplicationNodeManagerMock.java | 14 +
hadoop-hdds/test-utils/pom.xml | 5 +
.../java/org/apache/ozone/test/tag/Flaky.java} | 33 +-
.../main/java/org/apache/ozone/test/tag/Slow.java} | 32 +-
.../org/apache/ozone/test/tag}/package-info.java | 7 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 18 +-
.../hdds/scm/cli/ContainerOperationClient.java | 23 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 37 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 79 +-
.../datanode/TestContainerBalancerSubCommand.java | 22 +-
.../cli/pipeline/TestListPipelinesSubCommand.java | 192 ++++
hadoop-ozone/client/pom.xml | 4 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 48 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 5 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../client/checksum/BaseFileChecksumHelper.java | 54 +-
.../checksum/ReplicatedFileChecksumHelper.java | 44 +-
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 ++
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 420 ++++++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 ++
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 ++
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 +++++
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 +++++++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 +++++
.../io/ECBlockReconstructedStripeInputStream.java | 631 ++++++++++++
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 706 +++++++++++++
.../client/io/InsufficientLocationsException.java | 26 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 12 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 276 ++++-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 ++-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 +++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1066 ++++++++++++++++++++
.../checksum/TestReplicatedFileChecksumHelper.java | 25 +-
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +++
.../hadoop/ozone/client/rpc/RpcClientTest.java | 279 ++---
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 39 +
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 180 ++--
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 39 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 3 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 ++
.../hadoop/ozone/om/helpers/ServiceInfo.java | 25 +-
.../apache/hadoop/ozone/om/lock/LockUsageInfo.java | 63 ++
.../apache/hadoop/ozone/om/lock/OMLockMetrics.java | 207 ++++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 282 +++++-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 3 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 69 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 74 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 ++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 552 ++++++++++
hadoop-ozone/csi/pom.xml | 4 +
hadoop-ozone/datanode/pom.xml | 1 +
.../dev-support/checks/_mvn_unit_report.sh | 24 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 3 +-
hadoop-ozone/dev-support/checks/rat.sh | 13 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 +
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone-topology/test.sh | 6 -
.../dist/src/main/compose/ozone/docker-config | 4 +
.../dist/src/main/compose/ozone/prometheus.yml | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 10 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 98 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 8 +-
.../dist/src/main/compose/upgrade/testlib.sh | 16 +-
.../non-rolling-upgrade/1.1.0-1.2.0/callback.sh | 8 +-
.../{1.1.0-1.2.0 => 1.2.1-1.3.0}/callback.sh | 20 +-
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 5 +-
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 41 +
hadoop-ozone/dist/src/main/k8s/README.md | 68 ++
hadoop-ozone/dist/src/main/license/jar-report.txt | 2 +
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../src/main/smoketest/admincli/datanode.robot | 6 +-
.../main/smoketest/auditparser/auditparser.robot | 22 +-
.../dist/src/main/smoketest/basic/basic.robot | 9 +-
.../debug/ozone-debug-corrupt-block.robot | 49 +
.../ozone-debug-dead-datanode.robot} | 35 +-
.../debug/ozone-debug-stale-datanode.robot | 48 +
.../main/smoketest/debug/ozone-debug-tests.robot | 51 +
.../src/main/smoketest/debug/ozone-debug.robot | 93 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 ++
.../dist/src/main/smoketest/ec/basic.robot | 64 ++
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/freon/generate.robot | 19 +-
.../dist/src/main/smoketest/freon/remove.robot | 21 +-
.../dist/src/main/smoketest/freon/validate.robot | 13 +-
.../dist/src/main/smoketest/omha/om-prepare.robot | 3 +-
.../dist/src/main/smoketest/ozone-lib/freon.robot | 65 ++
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 6 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 15 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/spnego/web.robot | 4 -
.../dist/src/main/smoketest/upgrade/generate.robot | 5 +-
.../dist/src/main/smoketest/upgrade/prepare.robot | 2 +-
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
...g4j2.properties => s3g-audit-log4j2.properties} | 8 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 25 +
hadoop-ozone/integration-test/pom.xml | 10 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 62 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 72 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 3 +
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 241 +++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 100 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 48 +
.../rooted/ITestRootedOzoneContractRootDir.java | 1 -
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 23 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 7 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 19 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 5 +-
.../TestContainerStateManagerIntegration.java | 101 +-
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 28 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 29 +-
.../TestRatisPipelineCreateAndDestroy.java | 31 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 7 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 33 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 6 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 33 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 2 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 5 +-
.../hadoop/ozone/TestContainerOperations.java | 35 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 8 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 36 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 6 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 12 +-
.../ozone/client/TestOzoneClientFactory.java | 75 ++
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 3 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 23 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 23 +-
.../rpc/TestCloseContainerHandlingByClient.java | 14 +-
.../TestContainerStateMachineFailureOnRead.java | 21 +-
.../rpc/TestContainerStateMachineFailures.java | 13 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 10 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++++++
.../client/rpc/TestFailureHandlingByClient.java | 30 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 3 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 6 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 19 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 468 ++++++---
.../client/rpc/TestOzoneRpcClientWithRatis.java | 10 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 24 +-
.../ozone/client/rpc/TestWatchForCommit.java | 12 +-
.../ozone/client/rpc/read/ECStreamTestUtil.java | 408 ++++++++
.../rpc/read/TestBlockInputStreamFactoryImpl.java | 111 ++
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../client/rpc/read/TestECBlockInputStream.java | 522 ++++++++++
.../rpc/read/TestECBlockInputStreamProxy.java | 399 ++++++++
.../read/TestECBlockReconstructedInputStream.java | 379 +++++++
.../TestECBlockReconstructedStripeInputStream.java | 724 +++++++++++++
.../ozone/client/rpc/read/TestInputStreamBase.java | 23 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 61 +-
.../client/rpc/read/TestKeyInputStreamEC.java | 118 +++
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestBlockDeletion.java | 19 +-
.../TestCloseContainerByPipeline.java | 24 +-
.../commandhandler/TestCloseContainerHandler.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
...ler.java => TestRefreshVolumeUsageHandler.java} | 121 +--
.../container/ozoneimpl/TestOzoneContainer.java | 127 +--
.../container/server/TestContainerServer.java | 23 +-
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 182 ++--
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 19 +-
.../ozone/om/TestContainerReportWithKeys.java | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 39 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 -
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 81 ++
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 77 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 11 +-
.../org/apache/hadoop/ozone/om/TestOmInit.java | 7 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 5 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 25 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 2 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 40 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 62 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 4 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 2 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 14 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 2 -
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 6 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 35 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 8 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 7 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 5 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 18 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 4 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 4 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 21 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 140 ++-
hadoop-ozone/interface-client/pom.xml | 4 +
.../src/main/proto/OmClientProtocol.proto | 23 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 6 +-
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 6 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 4 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 3 +-
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 3 +-
hadoop-ozone/ozone-manager/pom.xml | 5 +
.../org/apache/hadoop/ozone/om/BucketManager.java | 22 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 255 -----
.../hadoop/ozone/om/DirectoryDeletingService.java | 7 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 9 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 176 ++--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 11 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 173 +++-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 76 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 94 +-
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 2 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 +
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 ++
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 ++
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 41 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 17 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 18 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 77 +-
.../BucketLayoutAwareOMKeyRequestFactory.java | 314 ++++++
.../ozone/om/request/OMClientRequestUtils.java | 50 +
.../ozone/om/request/OMKeyRequestFactory.java | 139 ---
.../om/request/bucket/OMBucketCreateRequest.java | 34 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/bucket/acl/OMBucketAclRequest.java | 4 +-
.../om/request/file/OMDirectoryCreateRequest.java | 109 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 74 +-
.../request/file/OMFileCreateRequestWithFSO.java | 15 +-
.../ozone/om/request/file/OMFileRequest.java | 60 +-
.../om/request/key/OMAllocateBlockRequest.java | 45 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 92 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 29 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 70 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 15 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 15 -
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 19 -
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 90 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 137 ++-
.../ozone/om/request/key/OMKeysRenameRequest.java | 5 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 48 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 141 +++
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 2 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 2 +-
.../S3InitiateMultipartUploadRequest.java | 54 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 43 +-
.../S3MultipartUploadCommitPartRequest.java | 39 +-
.../S3MultipartUploadCompleteRequest.java | 126 ++-
.../S3MultipartUploadCompleteRequestWithFSO.java | 15 +-
.../security/OMCancelDelegationTokenRequest.java | 42 +-
.../security/OMGetDelegationTokenRequest.java | 54 +-
.../security/OMRenewDelegationTokenRequest.java | 51 +-
.../validation/RequestFeatureValidator.java | 99 ++
.../request/validation/RequestProcessingPhase.java | 29 +-
.../om/request/validation/RequestValidations.java | 114 +++
.../om/request/validation/ValidationCondition.java | 55 +
.../om/request/validation/ValidationContext.java | 52 +
.../om/request/validation/ValidatorRegistry.java | 201 ++++
.../ozone/om/request/validation/package-info.java | 62 ++
.../hadoop/ozone/om/response/OMClientResponse.java | 16 -
.../om/response/bucket/OMBucketCreateResponse.java | 3 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 3 +-
.../response/file/OMFileCreateResponseWithFSO.java | 4 +-
.../om/response/key/OMAllocateBlockResponse.java | 3 +-
.../key/OMAllocateBlockResponseWithFSO.java | 3 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 4 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 3 +-
.../response/key/OMKeyCreateResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 3 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 3 +-
.../om/response/key/OMKeysDeleteResponse.java | 12 +-
...thFSO.java => OMKeysDeleteResponseWithFSO.java} | 76 +-
.../om/response/key/OMOpenKeysDeleteResponse.java | 18 +-
.../multipart/S3MultipartUploadAbortResponse.java | 3 +-
.../S3MultipartUploadCommitPartResponse.java | 3 +-
.../S3MultipartUploadCompleteResponse.java | 36 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 9 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 9 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 5 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 32 +-
.../protocolPB/OzoneManagerRequestHandler.java | 233 ++++-
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 23 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 14 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 242 ++---
.../hadoop/ozone/om/TestKeyDeletingService.java | 142 ++-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 23 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 50 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 ++
.../apache/hadoop/ozone/om/TestTrashService.java | 6 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 108 ++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 1 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 7 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 8 +
.../om/ratis/TestOzoneManagerStateMachine.java | 1 +
.../ozone/om/request/OMRequestTestUtils.java | 16 +
.../request/TestBucketLayoutAwareOMKeyFactory.java | 166 +++
.../request/file/TestOMDirectoryCreateRequest.java | 37 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 16 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../file/TestOMFileCreateRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 88 +-
.../om/request/key/TestOMKeyRenameRequest.java | 14 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 33 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 35 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 109 ++
.../om/request/key/TestOMKeysRenameRequest.java | 4 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 178 ++--
.../TestS3InitiateMultipartUploadRequest.java | 2 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../TestS3MultipartUploadAbortRequest.java | 4 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 5 +
.../TestS3MultipartUploadCommitPartRequest.java | 6 +-
.../TestS3MultipartUploadCompleteRequest.java | 40 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 7 +-
.../security/TestOMGetDelegationTokenRequest.java | 10 +-
.../upgrade/TestOMCancelPrepareRequest.java | 2 +-
.../TestRequestFeatureValidatorProcessor.java | 524 ++++++++++
.../request/validation/TestRequestValidations.java | 348 +++++++
.../request/validation/TestValidatorRegistry.java | 215 ++++
.../GeneralValidatorsForTesting.java | 190 ++++
.../ValidatorsForOnlyOldClientValidations.java | 43 +
.../ozone/om/response/TestCleanupTableInfo.java | 9 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 3 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 61 +-
.../s3/multipart/TestS3MultipartResponse.java | 19 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 17 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 4 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 34 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 133 ++-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 126 ++-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +++-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 1 +
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 3 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 +
.../org.apache.hadoop.security.token.DtFetcher | 0
...rg.apache.hadoop.security.token.TokenIdentifier | 0
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 11 +
.../org.apache.hadoop.security.token.TokenRenewer | 1 +
hadoop-ozone/pom.xml | 7 +-
hadoop-ozone/recon-codegen/pom.xml | 4 +
hadoop-ozone/recon/pom.xml | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 8 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 12 +
.../recon/metrics/ReconTaskStatusMetrics.java | 83 ++
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 20 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../scm/ReconStorageContainerManagerFacade.java | 11 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 14 +-
.../impl/StorageContainerServiceProviderImpl.java | 4 +-
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 37 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestOpenContainerCount.java | 8 +-
.../recon/fsck/TestContainerHealthStatus.java | 4 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 6 +-
.../TestContainerHealthTaskRecordGenerator.java | 3 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 17 +-
.../ozone/recon/scm/TestReconContainerManager.java | 2 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 12 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 4 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 4 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 256 +++--
hadoop-ozone/s3gateway/pom.xml | 8 +
.../org/apache/hadoop/ozone/s3/ClientIpFilter.java | 69 ++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 9 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 22 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 19 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 231 +++--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 63 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 250 +++--
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 46 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 368 +++++++
.../hadoop/ozone/s3/metrics}/package-info.java | 7 +-
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 94 ++
.../s3gateway/src/main/resources/browser.html | 617 -----------
.../hadoop/ozone/client/OzoneBucketStub.java | 68 +-
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 156 +++
.../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 36 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 9 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 282 ++++++
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 +++
.../src/test/resources/auditlog.properties | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 3 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../ozone/freon/FreonReplicationOptions.java | 73 ++
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 109 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 2 +-
.../freon/containergenerator/GeneratorOm.java | 5 +-
.../freon/containergenerator/GeneratorScm.java | 4 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 5 +-
.../hadoop/ozone/shell/ReplicationOptions.java | 87 ++
.../ozone/shell/ShellReplicationOptions.java | 26 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 8 +
.../ozone/shell/bucket/InfoBucketHandler.java | 6 +
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 94 +-
765 files changed, 37634 insertions(+), 6447 deletions(-)
create mode 100644 dev-support/annotations/pom.xml
create mode 100644 dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container => dev-support/annotations/src/main/java/org/apache/ozone/annotations}/package-info.java (81%)
rename .github/workflows/cancel-ci.yaml => dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor (61%)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ECXceiverClientGrpc.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java => client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockExtendedInputStream.java} (67%)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ExtendedInputStream.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/{DatanodeVersions.java => ComponentVersion.java} (67%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersion.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/{AuditLoggerType.java => S3GAction.java} (60%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestECReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerInfo.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/common/helpers/TestExcludeList.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java
create mode 100644 hadoop-hdds/docs/content/feature/EC-Chunk-Layout.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reads-With-No-Failures.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Reconstructional-Read.png
create mode 100644 hadoop-hdds/docs/content/feature/EC-Write-Block-Allocation-in-Containers.png
create mode 100644 hadoop-hdds/docs/content/feature/ErasureCoding.md
copy {hadoop-ozone/client => hadoop-hdds/erasurecode}/pom.xml (57%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/HadoopNativeECAccessorUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder}/package-info.java (73%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/BufferAllocator.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/CodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/ECChunk.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode}/package-info.java (69%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferDecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferEncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/CoderUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DecodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/EncodingState.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ErasureCodeNative.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderFactory.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawDecoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawEncoder.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawErasureCoderFactory.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/package-info.java} (50%)
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/CodecUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GF256.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GaloisField.java
create mode 100644 hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/RSUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util}/package-info.java (74%)
create mode 100644 hadoop-hdds/erasurecode/src/main/resources/META-INF/services/org.apache.ozone.erasurecode.rawcoder.RawErasureCoderFactory
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/DumpUtil.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCodecRegistry.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCoderBase.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/test/java/org/apache/ozone/erasurecode}/package-info.java (83%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderBenchmark.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestCodecRawCoderMapping.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestDummyRawCoder.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeRSRawCoder.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeXORRawCoder.java} (57%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoder.java} (69%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoderBase.java
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawCoderBase.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java (51%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoder.java} (75%)
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoderBase.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder}/package-info.java (83%)
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/EpochEntry.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/IllegalContainerBalancerStateException.java (52%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/InvalidContainerBalancerConfigurationException.java (50%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/{MockSCMHADBTransactionBuffer.java => SCMHADBTransactionBufferStub.java} (92%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/{MockSCMHAManager.java => SCMHAManagerStub.java} (89%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
copy hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/{TestSCMContainerPlacementRackAware.java => TestSCMContainerPlacementRackScatter.java} (70%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateMap.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java => test-utils/src/main/java/org/apache/ozone/test/tag/Flaky.java} (56%)
rename hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java => test-utils/src/main/java/org/apache/ozone/test/tag/Slow.java} (59%)
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container => test-utils/src/main/java/org/apache/ozone/test/tag}/package-info.java (89%)
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/pipeline/TestListPipelinesSubCommand.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BadDataLocationException.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactory.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactory.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntryPool.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedInputStream.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedStripeInputStream.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
copy hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java => hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/InsufficientLocationsException.java (58%)
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MultiNodePipelineBlockAllocator.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/BlockStreamAccessor.java
create mode 100644 hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/QuotaUtil.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/LockUsageInfo.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OMLockMetrics.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestQuotaUtil.java
copy hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/{1.1.0-1.2.0 => 1.2.1-1.3.0}/callback.sh (79%)
create mode 100644 hadoop-ozone/dist/src/main/k8s/README.md
create mode 100644 hadoop-ozone/dist/src/main/org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory
create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
copy hadoop-ozone/dist/src/main/smoketest/{freon/validate.robot => debug/ozone-debug-dead-datanode.robot} (50%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot
copy hadoop-ozone/dist/src/shell/conf/{scm-audit-log4j2.properties => s3g-audit-log4j2.properties} (94%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedDDSWithFSO.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClientFactory.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/ECStreamTestUtil.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestBlockInputStreamFactoryImpl.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStreamProxy.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockReconstructedInputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockReconstructedStripeInputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStreamEC.java
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{TestCloseContainerHandler.java => TestRefreshVolumeUsageHandler.java} (55%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OBSKeyPathLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockProvider.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockStrategy.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/RegularBucketLockStrategy.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock}/package-info.java (86%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/BucketLayoutAwareOMKeyRequestFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequestUtils.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMKeyRequestFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java (69%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/{OMKeyDeleteResponseWithFSO.java => OMKeysDeleteResponseWithFSO.java} (55%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOzoneLockProvider.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestBucketLayoutAwareOMKeyFactory.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequestWithFSO.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier (100%)
copy hadoop-ozone/{ozonefs => ozonefs-hadoop3}/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (92%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconTaskStatusMetrics.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ClientIpFilter.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container => hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics}/package-info.java (89%)
delete mode 100644 hadoop-ozone/s3gateway/src/main/resources/browser.html
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayAuditLog.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
copy {hadoop-hdds/common => hadoop-ozone/s3gateway}/src/test/resources/auditlog.properties (97%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersions.java => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java (61%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org