You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2021/11/24 21:57:18 UTC
[ozone] branch HDDS-3816-ec updated (52f05fc -> 9a7025a)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a change to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 52f05fc HDDS-5951. EC: ECBlockReconstructedStripeInputStream should handle block read failures and continue reading (#2831)
add 8eaa9c5 HDDS-4440. Diagrams for old and new approach are mixed up (#2555)
add c5a92a2 HDDS-5824. `ozone sh volume/bucket/key list` should print valid JSON array (#2713)
add db97a32 HDDS-5733. Incorrect calculation of iteration related metrics in ContainerBalancer (#2631)
add efb5263 HDDS-5806. remove BackgroundPipelineCreator v1 code (#2700)
add 57729b4 HDDS-5536. Add metrics for ReplicationSupervisor (#2493)
add 1ffcfe9 HDDS-5349. Avoid usage of locks in listStatus. (#2369)
add 3e05e71 HDDS-5373. [FSO] Define default bucket layout. (#2707)
add f902b13 HDDS-5785. containerCreationLock is a severe bottleneck when there are many containers to create. (#2679)
add a62c61f HDDS-5823. Change the key permission to 600. (#2722)
add 2f49993 HDDS-5845. Add metrics for stateMachineDataCache cache hit and miss. (#2726)
add 730a2e8 HDDS-5842. Parse and dump SCM ratis segment file to printable text (#2731)
add e19fa38 HDDS-5814. Handle InterruptedException in Ozone (#2706)
add ae812c06 HDDS-5831. Remove empty TaskQueue in ContainerStateMachine. (#2716)
add ed069bd HDDS-5854. Bump json-smart to 2.3.1 (#2736)
add 7376cfe HDDS-5855. Bump commons-compress to 1.21 (#2737)
add 0e5ad31 HDDS-5856. Bump Spring to 5.2.17 (#2738)
add d341c05 HDDS-5857. Bump aws-java-sdk to latest 1.11 (#2740)
add 24ddf0f HDDS-5659. Use bucket layout specific DB table in OmMetadataManager.getKeyTable() - part1 (#2724)
add bf6da11 HDDS-5858. Fix resource leak due to Files.list (#2741)
add f2f6aa5 HDDS-5671. ContainerBalancer#stop should prevent the current balancing thread from interrupting itself. (#2578)
add bc81dcf HDDS-5734. add a command to trigger datanode executing du immediately (#2685)
add a1701e5 HDDS-5850. Update default value of 'ozone.scm.ha.ratis.segment.size' and 'preallocated.size' to improve SCM write perf. (#2732)
add 7d79d73 HDDS-5812. Increase Ratis version to 2.2.0 (#2753)
add fbb8cf0 HDDS-5794. The misleading "No available thread in pool for past * second" log message in DN StateContext (#2693)
add 996518a HDDS-5860. Shade Jackson for Ozone Filesystem (#2744)
add 42d3de3 HDDS-5725. Catch possible AlreadyExistsException for create pipeline command. (#2623)
add 2a4fcde HDDS-5888. Handle InterruptedException in OzoneClientProducer and ContainerBalancer (#2761)
add ba038e4 HDDS-5875. Do not cache tokens as part of Key Information. (#2754)
add 2a4bc83 HDDS-5461. Move old objects to delete table on overwrite (#2433)
add 6e5db67 HDDS-5871. Fix ContainerBalancerConfiguration annotation string (#2762)
add d8d42b5 HDDS-5868. Change getBucketInfo annotation (#2749)
add 0828afc HDDS-5843. SCM terminates when adding container to a pipeline during startup (#2745)
add c5aaad8 HDDS-3371. Cleanup of old write-path of key in OM (#2703)
add fc666fa HDDS-5893. Potential data loss on multipart upload commit (#2763)
add a51822f HDDS-5882 Changes to protobuf messages for HDDS-5881 (#2774)
add f133303 HDDS-5707. List container supports replication factor filter. (#2606)
add 990db75 HDDS-5835. Follow up task to use bucket layout specific DB table in OmMetadataManager.getKeyTable() - part2 (#2751)
add 0906337 HDDS-5902 Ambiguous OM kerberos keytab settings (#2769)
add a5d6707 HDDS-5901. Delete on volume/bucket throws fatal error on ofs (#2776)
add 26824ca HDDS-5884. OM Validate S3 Auth for write requests. (#2778)
add d018d00 HDDS-5805. remove containerStateManager V1 code (#2727)
add f153236 HDDS-5863. Error message having null fields on volume creation (#2752)
add 5c76856 HDDS-5885. OM Auth Validate for read requests and handle non ratis enabled code. (#2779)
add 5f0a588 HDDS-5908. MPU getKey can fail, if completeMPU result is still in cache. (#2787)
add e427b71 HDDS-5883 Change S3G client to set S3 Auth per req (#2775)
add 73a081a HDDS-5864. Retry when DN connection issue during getBlock/ReadChunk call during Ozone key Read (#2746)
add 83f8b04 HDDS-5736. Update navbar from Hadoop to Ozone on doc page (#2792)
add 47cb26a HDDS-3983 Ozone RocksDB Iterator wrapper should not expose key() and value() API. (#2402)
add 565972c Support more detailed error log when handleFlush in Ozone client (#2795)
add 1cf43ac HDDS-5931. Broken link in dist/README (#2799)
add 9cea16d HDDS-5933. Ozone Filesystem shaded jars include unnecessary dependencies (#2798)
add eb7136d HDDS-3369. Cleanup old write-path of volume in OM. (#2780)
add c088cd1 HDDS-5925. Fix SecuringOzoneHTTP doc (#2796)
add ac86851 HDDS-5943. Shutdown ResultHandlerExecutorService for StorageVolumeChecker. (#2806)
add aa7e083 HDDS-5944. Fix typo in OzoneManagerServiceProviderImpl.java (#2807)
add 6ebb316 HDDS-5945. bump rocksdb version to 6.25.3 (#2809)
add 4c2181f HDDS-5935. Bump Spring to 5.2.18 (#2801)
add f3b482a HDDS-5922. ignore deletetransaction when container is not found (#2793)
add c63479c HDDS-5839. Make sure buckets created from OFS are in FILE_SYSTEM_OPTIMIZED layout (#2730)
add 96e2d7f HDDS-5947. Remove unused mina-core and sshd-core dependencies (#2811)
add 252fee4 HDDS-5910 Add additional verification for S3 Auth. (#2817)
add 9d9e780 HDDS-5956. Speed up TestOzoneRpcClientAbstract#testZReadKeyWithUnhealthyContainerReplica (#2820)
add 0eedab9 HDDS-5929. Make FSO and OBS bucket layouts independent of normalization config flag (#2819)
add ba3ac30 HDDS-5960. Change option name to bucketlayout instead of type (#2825)
add 4558e40 HDDS-5970. Remove OMKeyRequest#getBucketLayout overridden method in subclasses (#2830)
add 9384bd7 HDDS-5820. Intermittent failure in TestPipelinePlacementPolicy#testPickLowestLoadAnchor (#2757)
add d0e9140 HDDS-5958. Refine container scrub log message. (#2823)
add 45c1899 HDDS-5937. Inaccurate bucket info returned from bucket list command in Shell (#2832)
add c2db377 HDDS-5872. Do not failover on some RpcExceptions (#2772)
add 71f47d4 HDDS-5981. Refactor usage of bucket type. (#2836)
add 026d878 HDDS-5982. Remove isBucketFSOptimized flag. (#2838)
add 346b55d HDDS-5906. Fix DBScanner to support Datanode DB (#2781)
add 1b7072c HDDS-5983. Prefix Parser tool should only work for FSO buckets. (#2839)
add 5ef59b4 HDDS-5957. Bump proto-backwards-compatibility from 1.0.5 to 1.0.7 (#2821)
add b4a785c HDDS-5891. OFS mkdir -p does not work as expected for bucket creation when volume exists due to volume create ACL check (#2814)
add 6ea9916 HDDS-5964. [S3-TDE] Range Reads not working on encrypted buckets (#2827)
add 83e5aa3 HDDS-5848. Introduce more replication metrics (#2758)
add 4348d1d HDDS-5980. Get on a key which is created/overwrite through MPU on a TDE bucket, the data does not match with uploaded data. (#2844)
add 78e725c HDDS-5969. TestRootedOzoneFileSystem parameterized test is not initialized properly (#2843)
add 6704310 Merging master into EC branch
add d5ac673 HDDS-6012: EC: Fix TestOzoneShellHA failures post master merge with EC branch
add 9a7025a HDDS-6035: EC: Fix TestOmMetrics in merge branch
No new revisions were added by this update.
Summary of changes:
.../hadoop/hdds/scm/storage/BlockInputStream.java | 52 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 14 +-
.../hdds/scm/storage/TestBlockInputStream.java | 32 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 51 ++
.../java/org/apache/hadoop/hdds/NodeDetails.java | 0
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 23 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 4 +
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 31 +-
.../protocol/StorageContainerLocationProtocol.java | 20 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 5 -
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +
.../common/src/main/resources/ozone-default.xml | 50 +-
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../common/statemachine/StateContext.java | 14 +-
.../CreatePipelineCommandHandler.java | 10 +-
....java => RefreshVolumeUsageCommandHandler.java} | 67 +-
.../common/transport/server/ratis/CSMMetrics.java | 10 +
.../server/ratis/ContainerStateMachine.java | 9 +-
.../transport/server/ratis/XceiverServerRatis.java | 9 +-
.../container/common/volume/MutableVolumeSet.java | 4 +
.../container/common/volume/StorageVolume.java | 4 +
.../common/volume/StorageVolumeChecker.java | 1 +
.../ozone/container/common/volume/VolumeInfo.java | 8 +-
.../ozone/container/common/volume/VolumeUsage.java | 3 +
.../container/keyvalue/KeyValueContainerCheck.java | 2 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 29 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +-
.../replication/DownloadAndImportReplicator.java | 5 +-
.../container/replication/MeasuredReplicator.java | 34 +-
.../replication/ReplicationSupervisor.java | 9 +-
.../replication/ReplicationSupervisorMetrics.java | 71 ++
.../replication/SimpleContainerDownloader.java | 7 +-
.../common/volume/TestStorageVolumeChecker.java | 4 +
.../replication/TestMeasuredReplicator.java | 34 +-
.../replication/TestReplicationSupervisor.java | 59 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 16 +
hadoop-hdds/docs/content/feature/Topology.md | 2 +-
hadoop-hdds/docs/content/interface/CSI.md | 4 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Cli.md | 7 +-
.../docs/content/security/SecuringOzoneHTTP.md | 20 +-
hadoop-hdds/docs/content/start/FromSource.md | 2 +-
.../docs/static/design/s3-performance-new.png | Bin 33688 -> 34927 bytes
.../docs/static/design/s3-performance-old.png | Bin 34927 -> 33688 bytes
.../themes/ozonedoc/layouts/partials/navbar.html | 6 +-
.../themes/ozonedoc/layouts/partials/sidebar.html | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 12 +-
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 31 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 16 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 19 +-
.../apache/hadoop/hdds/utils/db/TableIterator.java | 12 -
.../apache/hadoop/hdds/utils/db/TypedTable.java | 18 -
.../hdds/security/x509/keys/TestKeyCodec.java | 5 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 19 +-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../scm/block/DeletedBlockLogStateManagerImpl.java | 10 -
.../hdds/scm/container/ContainerManager.java | 2 +-
.../hdds/scm/container/ContainerManagerImpl.java | 4 +-
.../hdds/scm/container/ContainerStateManager.java | 570 +++-----------
.../scm/container/ContainerStateManagerImpl.java | 19 +-
.../scm/container/ContainerStateManagerV2.java | 189 -----
.../hdds/scm/container/ReplicationManager.java | 33 +-
.../scm/container/balancer/ContainerBalancer.java | 221 +++---
.../balancer/ContainerBalancerConfiguration.java | 2 +-
.../balancer/ContainerBalancerMetrics.java | 115 +--
.../replication/ReplicationManagerMetrics.java | 41 +
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 24 +
.../scm/pipeline/BackgroundPipelineCreator.java | 264 +++++--
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 332 --------
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 10 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 17 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 24 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 12 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 27 +
...inerLocationProtocolServerSideTranslatorPB.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 53 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 18 +-
.../scm/container/TestContainerReportHandler.java | 267 ++++---
.../scm/container/TestContainerStateManager.java | 91 ++-
.../TestIncrementalContainerReportHandler.java | 126 +--
.../hdds/scm/container/TestReplicationManager.java | 380 +++++----
.../scm/container/TestUnknownContainerReport.java | 47 +-
.../container/balancer/TestContainerBalancer.java | 27 +
.../hdds/scm/container/balancer}/package-info.java | 6 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 8 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 6 +
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 26 +
.../hdds/scm/cli/ContainerOperationClient.java | 5 +-
.../hdds/scm/cli/container/ListSubcommand.java | 6 +-
.../org/apache/hadoop/ozone/client/BucketArgs.java | 8 +-
.../hadoop/ozone/client/io/OzoneInputStream.java | 33 +-
.../ozone/client/protocol/ClientProtocol.java | 19 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 50 +-
hadoop-ozone/common/pom.xml | 4 -
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 3 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 27 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 12 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 11 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 25 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 25 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 7 -
.../ozone/om/protocol/OzoneManagerProtocol.java | 168 +++-
.../apache/hadoop/ozone/om/protocol/S3Auth.java} | 33 +-
...nsport.java => OzoneManagerClientProtocol.java} | 33 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 59 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 41 +-
.../org/apache/hadoop/ozone/csi/NodeService.java | 10 +-
hadoop-ozone/dist/README.md | 6 +-
.../dist/src/main/compose/ozone-mr/common-config | 1 +
.../src/main/compose/ozonesecure-mr/docker-config | 3 +
.../src/main/compose/ozonesecure/docker-config | 1 +
.../src/main/smoketest/admincli/container.robot | 8 +
.../dist/src/main/smoketest/basic/links.robot | 8 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 10 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +-
.../main/smoketest/security/ozone-secure-fs.robot | 2 +-
.../dev-support/findbugsExcludeFile.xml | 4 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 19 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 3 +
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 40 +-
.../fs/ozone/TestOzoneFileInterfacesWithFSO.java | 25 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 29 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 3 +
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 2 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 151 +++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 12 +-
.../TestContainerStateManagerIntegration.java | 2 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 11 +
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 5 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 111 ++-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 100 ++-
.../commandhandler/TestBlockDeletion.java | 10 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 43 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 10 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 103 +++
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 16 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 27 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 277 ++++---
.../TestOzoneHARatisLogParser.java} | 52 +-
.../ozone/recon/TestReconWithOzoneManager.java | 19 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 141 +++-
.../src/main/proto/OmClientProtocol.proto | 11 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 3 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 21 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 667 ++++++++++------
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 14 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 11 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 871 +++------------------
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 5 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 1 +
.../org/apache/hadoop/ozone/om/VolumeManager.java | 40 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 300 -------
.../ozone/om/ratis/OzoneManagerRatisServer.java | 12 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 158 ++--
.../hadoop/ozone/om/request/OMClientRequest.java | 22 +-
.../om/request/bucket/OMBucketCreateRequest.java | 11 +-
.../om/request/file/OMDirectoryCreateRequest.java | 10 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 7 +-
.../request/file/OMFileCreateRequestWithFSO.java | 9 +-
.../ozone/om/request/file/OMFileRequest.java | 122 ++-
.../om/request/key/OMAllocateBlockRequest.java | 15 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 90 ++-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 39 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 23 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 10 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 18 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 12 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 14 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 13 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 33 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 5 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 9 +-
.../om/request/key/OMPathsPurgeRequestWithFSO.java | 10 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 9 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 5 +-
.../S3InitiateMultipartUploadRequest.java | 15 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 10 +-
.../multipart/S3MultipartUploadAbortRequest.java | 18 +-
.../S3MultipartUploadAbortRequestWithFSO.java | 10 +-
.../S3MultipartUploadCommitPartRequest.java | 19 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 10 +-
.../S3MultipartUploadCompleteRequest.java | 41 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 10 +-
.../ozone/om/request/upgrade/OMPrepareRequest.java | 9 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 21 +
.../response/file/OMDirectoryCreateResponse.java | 6 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 22 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 11 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 2 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 3 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 3 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 12 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 8 +-
.../om/response/key/OMKeysDeleteResponse.java | 3 +-
.../om/response/key/OMKeysRenameResponse.java | 11 +-
.../response/key/OMPathsPurgeResponseWithFSO.java | 4 +-
.../om/response/key/OMTrashRecoverResponse.java | 2 +-
.../om/response/key/acl/OMKeyAclResponse.java | 4 +-
.../response/key/acl/OMKeyAclResponseWithFSO.java | 4 +-
.../S3MultipartUploadCompleteResponse.java | 27 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 49 +-
.../protocolPB/OzoneManagerRequestHandler.java | 25 -
.../hadoop/ozone/security/S3SecurityUtil.java | 82 ++
.../hadoop/ozone/om/TestBucketManagerImpl.java | 63 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 7 +-
.../ozone/om/request/TestOMRequestUtils.java | 89 ++-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 5 +
.../request/file/TestOMDirectoryCreateRequest.java | 53 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 91 ++-
.../om/request/file/TestOMFileCreateRequest.java | 8 +-
.../file/TestOMFileCreateRequestWithFSO.java | 21 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 7 +-
.../key/TestOMAllocateBlockRequestWithFSO.java | 7 +-
.../ozone/om/request/key/TestOMKeyAclRequest.java | 30 +-
.../om/request/key/TestOMKeyAclRequestWithFSO.java | 13 +-
.../om/request/key/TestOMKeyCommitRequest.java | 129 ++-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 19 +-
.../om/request/key/TestOMKeyCreateRequest.java | 48 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 11 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 31 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 20 +-
.../om/request/key/TestOMKeyRenameRequest.java | 8 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 2 +
.../om/request/key/TestOMKeysDeleteRequest.java | 4 +-
.../om/request/key/TestOMKeysRenameRequest.java | 20 +-
.../TestS3InitiateMultipartUploadRequest.java | 27 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 6 +-
.../s3/multipart/TestS3MultipartRequest.java | 16 +-
.../TestS3MultipartUploadAbortRequest.java | 3 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 7 +-
.../TestS3MultipartUploadCommitPartRequest.java | 18 +-
...tS3MultipartUploadCommitPartRequestWithFSO.java | 13 +-
.../TestS3MultipartUploadCompleteRequest.java | 20 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 28 +-
.../file/TestOMDirectoryCreateResponse.java | 7 +-
.../file/TestOMFileCreateResponseWithFSO.java | 13 -
.../key/TestOMAllocateBlockResponseWithFSO.java | 13 -
.../om/response/key/TestOMKeyCommitResponse.java | 34 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 18 +-
.../key/TestOMKeyCreateResponseWithFSO.java | 13 -
.../om/response/key/TestOMKeyDeleteResponse.java | 18 +-
.../key/TestOMKeyDeleteResponseWithFSO.java | 14 -
.../om/response/key/TestOMKeyRenameResponse.java | 30 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 3 +
.../om/response/key/TestOMKeysDeleteResponse.java | 9 +-
.../om/response/key/TestOMKeysRenameResponse.java | 16 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 79 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 21 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 25 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 2 +
hadoop-ozone/pom.xml | 2 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 9 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 3 +
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 3 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 4 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 8 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 25 +-
.../ozone/recon/api/TestContainerEndpoint.java | 8 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 3 -
.../recovery/TestReconOmMetadataManagerImpl.java | 25 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 17 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 9 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 9 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 16 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 30 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 77 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 29 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 11 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 5 +
.../hadoop/ozone/s3/io/S3WrapperInputStream.java | 84 --
.../apache/hadoop/ozone/s3/io/package-info.java | 23 -
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 5 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 5 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 17 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 6 +
.../apache/hadoop/ozone/debug/PrefixParser.java | 16 +-
.../ozone/debug/container/ContainerCommands.java | 16 +-
.../ozone/freon/ClosedContainerReplicator.java | 10 +-
.../containergenerator/GeneratorDatanode.java | 23 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 7 +-
.../ozone/genesis/BenchMarkOMKeyAllocation.java | 137 ----
.../org/apache/hadoop/ozone/genesis/Genesis.java | 2 +-
.../hadoop/ozone/segmentparser/RatisLogParser.java | 4 +-
...MRatisLogParser.java => SCMRatisLogParser.java} | 15 +-
.../org/apache/hadoop/ozone/shell/Handler.java | 17 +
.../ozone/shell/bucket/CreateBucketHandler.java | 10 +-
.../ozone/shell/bucket/ListBucketHandler.java | 10 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 8 +-
.../ozone/shell/volume/ListVolumeHandler.java | 9 +-
.../ozone/debug/TestDBDefinitionFactory.java | 7 +-
pom.xml | 22 +-
310 files changed, 5808 insertions(+), 4946 deletions(-)
rename hadoop-hdds/{framework => common}/src/main/java/org/apache/hadoop/hdds/NodeDetails.java (100%)
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{ReplicateContainerCommandHandler.java => RefreshVolumeUsageCommandHandler.java} (51%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorMetrics.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerV2.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/security => server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer}/package-info.java (84%)
copy hadoop-ozone/{client/src/main/java/org/apache/hadoop/ozone/client/io/InsufficientLocationsException.java => common/src/main/java/org/apache/hadoop/ozone/om/protocol/S3Auth.java} (56%)
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/{OmTransport.java => OzoneManagerClientProtocol.java} (56%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{om/parser/TestOMRatisLogParser.java => parser/TestOzoneHARatisLogParser.java} (68%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
delete mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
delete mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMKeyAllocation.java
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/segmentparser/{OMRatisLogParser.java => SCMRatisLogParser.java} (75%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org