You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2024/03/27 23:25:49 UTC
(ozone) branch HDDS-7593 updated (509c970054 -> 44b12428c2)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit 509c970054 HDDS-10442. [hsync] Add a Freon tool to measure client to DataNode round-trip latency (#6297)
add 9dab4d2173 HDDS-10487. Intermittent crash in TestSnapshotDiffManager (#6346)
add d94aadc4b8 HDDS-10480. Avoid proto2 ByteString.toByteArray() calls. (#6342)
add 48bc30f514 HDDS-10470. Populate Maven dependency cache in separate workflow (#6340)
add 1322add415 HDDS-10493. Fix parameter number warning in DatanodeDetails and ContainerInfo (#6349)
add 6222aed33e HDDS-10495. Removed JMockit dependency (#6352)
add f8a1e3cc3a HDDS-10090. Clean up TestStorageContainerManager (#6347)
add bf91862796 HDDS-10473. Extract constants in HddsDispatcher (#6337)
add c439b7341a HDDS-10499. Bump org.kohsuke.metainf-services to 1.11 (#6354)
add 4ee5fd917b HDDS-10500. Bump Jackson to 2.16.1 (#6356)
add cb255044e2 HDDS-10501. Bump maven-deploy-plugin to 3.1.1 (#6353)
add 5c57bcb37d HDDS-10188. Add unit tests around Distcp across buckets (#6132)
add 131eec0587 HDDS-10485. Enable container scanner by default. (#6345)
add 07639859ec HDDS-10504. Remove unused VolumeInfo#configuredCapacity (#6363)
add d68ea97e02 HDDS-10503. Bump jgrapht to 1.4.0 (#6364)
add 6c95fc90ee HDDS-10502. Improve generic type of IOUtils.close (#6361)
add a8dd1c2cc2 HDDS-10475. Refine audit logging for bucket creation (#6366)
add 91af26ab39 HDDS-10414. Some acceptance tests fail with Docker Compose V2 (#6269)
add 6cd0b6f751 HDDS-10454. Make OzoneAcl immutable (#6319)
add e42b46d69c HDDS-10497. [hsync] Refresh block token immediately if block token expires. (#6371)
add 3ce3a284bf HDDS-10512. Reorder initialization of ScmTopologyClient in OM (#6374)
add 9222193084 HDDS-10492. [Doc] Update zh translation to Recon Architecture (#6348)
add 359bbad1f1 HDDS-10050. Clean up TestStorageContainerManagerHelper (#6373)
add 71e4ff373a HDDS-10498. Improved configuration for license compliance (#6358)
add 2ffdb3ccb8 HDDS-10516. Add metrics for chunk read from internal ratis (#6375)
add 3d193fc06e HDDS-10521. ETag field should not be returned during GetObject if the key does not contain ETag field (#6377)
add f4ba2beaaa HDDS-10440. Set new cluster ID and config instance in MiniOzoneCluster#build(). (#6325)
add b82a2a2842 HDDS-10496. Fetch dependencies for cache using actual build (#6359)
add a99c12bd04 HDDS-10523. Extract module-specific gitignore (#6379)
add e2a4609f3c HDDS-10526. Bump follow-redirects to 1.15.6 (#6382)
add 0bf2a29cbb HDDS-10120. BindException in some integration tests with Java 17 (#6384)
add 01deb4f679 HDDS-10534. Removed objenesis dependency (#6391)
add 8efcaffaa4 HDDS-10533. Bump junit5 to 5.10.2 (#6387)
add 238bff09e9 HDDS-10537. Bump docker-maven-plugin to 0.44.0 (#6390)
add d1e37d6458 HDDS-10535. Remove unused RangerRestMultiTenantAccessController. (#6392)
add 9291f89428 HDDS-10543. Bump LMAX Disruptor to 3.4.4 (#6388)
add 89b700d5bf HDDS-10505. Move space reservation logic to VolumeUsage (#6370)
add a122acca97 HDDS-10549. Fix typo in Freon error message (#6397)
add 9afaeacf71 HDDS-10546. OM startup failure as leader is not getting ready (#6395)
add d4606e1795 HDDS-10182. Add P99 quantiles and Min/Max Metrics for Xceiver Client Performance Metrics (#6070)
add 264cbc6192 HDDS-10476. Add metrics about bucket utilization. (#6344)
add e9073166da HDDS-9802. Tool to fix corrupted snapshot chain (#6386)
add 8582214315 HDDS-10515. Reduce OzoneAcl constructor count (#6396)
add ca8f1f76b2 HDDS-10552. Downgrade Surefire to 3.0.0-M4 (#6406)
add e6ed46a262 HDDS-9085. Recon Directories Pending for Deletion Inside OMDB Insight (#5521)
add 8dabb2bb2f HDDS-10554. Bump Zookeeper to 3.8.4 (#6407)
add 8a522f0c3f HDDS-10311. Speed up TestOmMetrics (#6409)
add f399f1ba5c HDDS-10539. Replace GSON with Jackson in multitenancy code. (#6399)
add d45acfb2bb HDDS-10544. Move LMAX Disruptor to runtime scope (#6394)
add 72fef614dc HDDS-10436. datanode status decommission command should have json output option (#6369)
add d70d247a49 HDDS-10551. Improve HTTPFS Documentation for Iterative Liststatus Functionality (#6412)
add c4dc6a072f HDDS-5865. Make read retry interval and attempts in BlockInputStream configurable (#6408)
add e3a7224a81 HDDS-10477. Make Rocksdb tools native lib compatible with all chipset with the same arch (#6341)
add 97038ef097 HDDS-10118. hdds-rocks-native fails to build with Java11+ (#6415)
new 44b12428c2 Merge remote-tracking branch 'asf/master' into HDDS-7593
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (509c970054)
\
N -- N -- N refs/heads/HDDS-7593 (44b12428c2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 16 +-
.github/workflows/intermittent-test-check.yml | 6 +-
.github/workflows/populate-cache.yml | 74 +++
.github/workflows/repeat-acceptance.yml | 12 +-
.gitignore | 35 +-
dev-support/rat/rat-exclusions.txt | 73 +++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +
.../hadoop/hdds/scm/XceiverClientMetrics.java | 44 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 27 +-
.../ozone/client/io/BlockInputStreamFactory.java | 6 +-
.../client/io/BlockInputStreamFactoryImpl.java | 15 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 16 +-
.../ozone/client/io/ECBlockInputStreamFactory.java | 6 +-
.../client/io/ECBlockInputStreamFactoryImpl.java | 13 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 14 +-
.../io/ECBlockReconstructedStripeInputStream.java | 10 +-
.../hdds/scm/storage/DummyBlockInputStream.java | 9 +-
.../storage/DummyBlockInputStreamWithRetry.java | 9 +-
.../hdds/scm/storage/TestBlockInputStream.java | 24 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 6 +-
.../client/io/TestBlockInputStreamFactoryImpl.java | 14 +-
.../ozone/client/io/TestECBlockInputStream.java | 110 +++-
.../client/io/TestECBlockInputStreamProxy.java | 13 +-
.../io/TestECBlockReconstructedInputStream.java | 10 +-
.../TestECBlockReconstructedStripeInputStream.java | 12 +-
hadoop-hdds/common/pom.xml | 5 -
.../java/org/apache/hadoop/hdds/HddsUtils.java | 1 -
.../apache/hadoop/hdds/fs/SpaceUsageSource.java | 4 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 88 +--
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../ContainerCommandResponseBuilders.java | 28 -
.../hdds/scm/storage/ContainerProtocolCalls.java | 38 --
.../org/apache/hadoop/hdds/server/JsonUtils.java | 8 +
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../hadoop/util/PerformanceMetricsInitializer.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 9 +-
.../hadoop/hdds/fs/MockSpaceUsageSource.java | 22 +
.../org/apache/hadoop/ozone/audit/DNAction.java | 3 +-
.../container/common/helpers/ContainerMetrics.java | 19 +
.../container/common/impl/HddsDispatcher.java | 78 +--
.../container/common/volume/StorageVolume.java | 5 -
.../ozone/container/common/volume/VolumeInfo.java | 93 +--
.../container/common/volume/VolumeInfoMetrics.java | 6 +-
.../ozone/container/common/volume/VolumeUsage.java | 125 ++--
.../ECReconstructionCoordinator.java | 7 +-
.../container/keyvalue/KeyValueContainerCheck.java | 6 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 10 +-
.../ozoneimpl/ContainerScannerConfiguration.java | 4 +-
.../container/common/impl/TestHddsDispatcher.java | 6 +-
.../common/volume/TestReservedVolumeSpace.java | 28 +-
.../TestContainerScannerConfiguration.java | 4 +-
hadoop-hdds/docs/.gitignore | 2 +
hadoop-hdds/docs/content/interface/HttpFS.md | 2 +-
hadoop-hdds/docs/content/interface/ReconApi.zh.md | 600 +++++++++++++++++-
hadoop-hdds/docs/pom.xml | 24 -
hadoop-hdds/framework/pom.xml | 2 +
hadoop-hdds/hadoop-dependency-test/pom.xml | 5 +
.../src/main/proto/DatanodeClientProtocol.proto | 13 -
hadoop-hdds/pom.xml | 39 --
hadoop-hdds/rocks-native/pom.xml | 3 +-
.../org/apache/ozone/graph/TestPrintableGraph.java | 84 +++
.../hadoop/hdds/scm/ha/SCMRatisResponse.java | 20 +-
.../hdds/scm/server/SCMContainerMetrics.java | 1 -
.../hdds/scm/cli/ContainerOperationClient.java | 2 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 94 ++-
.../client/checksum/ECBlockChecksumComputer.java | 50 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 23 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 34 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 25 +-
.../ozone/client/io/TestKeyInputStreamEC.java | 12 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 211 +++----
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 20 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 6 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 9 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 14 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 120 ++--
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 29 +-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 154 ++---
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 17 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 8 +-
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 38 +-
hadoop-ozone/dev-support/checks/rat.sh | 8 +-
hadoop-ozone/dist/.gitignore | 1 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 3 +-
.../src/main/smoketest/freon/echoRPCLoad.robot | 21 -
.../dist/src/main/smoketest/recon/recon-api.robot | 12 +-
.../smoketest/security/ozone-secure-tenant.robot | 2 +-
.../dist/src/main/smoketest/topology/cli.robot | 21 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 6 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 5 -
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 1 +
hadoop-ozone/httpfsgateway/pom.xml | 4 -
hadoop-ozone/integration-test/pom.xml | 5 -
.../fs/contract/AbstractContractAppendTest.java | 2 +
.../fs/contract/AbstractContractConcatTest.java | 2 +
.../AbstractContractCopyFromLocalTest.java | 2 +
.../AbstractContractGetFileStatusTest.java | 2 +
.../AbstractContractMultipartUploaderTest.java | 4 +
.../fs/contract/AbstractContractOpenTest.java | 2 +
.../AbstractContractRootDirectoryTest.java | 2 +
.../fs/contract/AbstractContractSeekTest.java | 4 +
.../fs/contract/AbstractContractSetTimesTest.java | 2 +
.../fs/contract/AbstractContractUnbufferTest.java | 2 +
.../ozone/AbstractRootedOzoneFileSystemTest.java | 111 +++-
.../org/apache/hadoop/fs/ozone/TestSafeMode.java | 2 +-
.../hadoop/hdds/scm/TestContainerSmallFile.java | 19 -
.../hdds/scm/TestStorageContainerManager.java | 383 ++++++------
.../scm/TestStorageContainerManagerHelper.java | 198 ------
.../scm/node/TestDecommissionAndMaintenance.java | 2 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 19 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +
.../hadoop/ozone/MiniOzoneClusterProvider.java | 34 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 +
.../org/apache/hadoop/ozone/TestBlockTokens.java | 7 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 29 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 34 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 66 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 4 +-
.../hadoop/ozone/freon/TestDNRPCLoadGenerator.java | 111 ----
.../apache/hadoop/ozone/om/TestBucketOwner.java | 6 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 29 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 557 +++++++++--------
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 67 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 18 +-
.../ozone/om/multitenant/RangerUserRequest.java | 28 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 68 +-
.../tools/contract/AbstractContractDistCpTest.java | 2 +
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 14 +-
.../hadoop/ozone/om/helpers/OzoneAclStorage.java | 15 +-
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 9 +-
.../ozone/om/helpers/TestOmPrefixInfoCodec.java | 2 +-
hadoop-ozone/ozone-manager/pom.xml | 5 -
.../hadoop/ozone/om/BucketUtilizationMetrics.java | 118 ++++
.../org/apache/hadoop/ozone/om/ListIterator.java | 2 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 34 -
.../org/apache/hadoop/ozone/om/OzoneManager.java | 23 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 5 +-
.../RangerRestMultiTenantAccessController.java | 681 ---------------------
.../ozone/om/ratis/OzoneManagerStateMachine.java | 16 +-
.../om/request/bucket/OMBucketCreateRequest.java | 3 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +-
.../ozone/om/TestBucketUtilizationMetrics.java | 126 ++++
.../apache/hadoop/ozone/om/TestChunkStreams.java | 9 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 7 +-
.../request/TestOMClientRequestWithUserInfo.java | 92 ++-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../om/request/key/TestOMKeyCreateRequest.java | 2 +-
.../TestS3InitiateMultipartUploadRequest.java | 11 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../key/acl/prefix/TestOMPrefixAclResponse.java | 4 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 11 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 18 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 6 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 1 +
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 2 +
hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 1 +
hadoop-ozone/pom.xml | 67 +-
hadoop-ozone/recon/.gitignore | 1 +
.../webapps/recon/ozone-recon-web/api/db.json | 67 ++
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 15 +-
.../ozone-recon-web/src/views/insights/om/om.tsx | 120 +++-
.../hadoop/ozone/recon/api/TestEndpoints.java | 12 +-
.../hadoop/ozone/recon/common/CommonUtils.java | 5 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 19 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 14 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 66 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 3 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +-
.../hadoop/ozone/freon/DNRPCLoadGenerator.java | 178 ------
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 2 +-
.../hadoop/ozone/freon/DatanodeSimulator.java | 4 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OmRPCLoadGenerator.java | 6 +-
.../freon/containergenerator/GeneratorOm.java | 4 +-
.../apache/hadoop/ozone/repair/OzoneRepair.java} | 45 +-
.../RDBParser.java => repair/RDBRepair.java} | 36 +-
.../hadoop/ozone/repair/om/SnapshotRepair.java | 200 ++++++
.../{shell/s3 => repair/om}/package-info.java | 5 +-
.../apache/hadoop/ozone/repair}/package-info.java | 7 +-
.../ozone/shell/tenant/GetUserInfoHandler.java | 44 +-
.../shell/tenant/TenantAssignAdminHandler.java | 21 +-
.../ozone/shell/tenant/TenantCreateHandler.java | 15 +-
.../ozone/shell/tenant/TenantDeleteHandler.java | 17 +-
.../ozone/shell/tenant/TenantListHandler.java | 31 +-
.../ozone/shell/tenant/TenantListUsersHandler.java | 19 +-
.../shell/tenant/TenantRevokeAdminHandler.java | 21 +-
pom.xml | 58 +-
193 files changed, 3772 insertions(+), 3414 deletions(-)
create mode 100644 .github/workflows/populate-cache.yml
create mode 100644 dev-support/rat/rat-exclusions.txt
create mode 100644 hadoop-hdds/docs/.gitignore
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/graph/TestPrintableGraph.java
create mode 100644 hadoop-ozone/dist/.gitignore
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHelper.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDNRPCLoadGenerator.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketUtilizationMetrics.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerRestMultiTenantAccessController.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketUtilizationMetrics.java
create mode 100644 hadoop-ozone/recon/.gitignore
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DNRPCLoadGenerator.java
copy hadoop-ozone/{insight/src/main/java/org/apache/hadoop/ozone/insight/Insight.java => tools/src/main/java/org/apache/hadoop/ozone/repair/OzoneRepair.java} (52%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{debug/RDBParser.java => repair/RDBRepair.java} (74%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/SnapshotRepair.java
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{shell/s3 => repair/om}/package-info.java (91%)
copy {hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair}/package-info.java (92%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 01/01: Merge remote-tracking branch 'asf/master' into HDDS-7593
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 44b12428c2bbd150a41f64d34ad5c466ceb6d26e
Merge: 825c340bae 97038ef097
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Thu Mar 21 17:50:35 2024 -0700
Merge remote-tracking branch 'asf/master' into HDDS-7593
Conflicts:
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
.github/workflows/ci.yml | 16 +-
.github/workflows/intermittent-test-check.yml | 6 +-
.github/workflows/populate-cache.yml | 74 +++
.github/workflows/repeat-acceptance.yml | 12 +-
.gitignore | 35 +-
dev-support/rat/rat-exclusions.txt | 73 +++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +
.../hadoop/hdds/scm/XceiverClientMetrics.java | 44 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 27 +-
.../ozone/client/io/BlockInputStreamFactory.java | 6 +-
.../client/io/BlockInputStreamFactoryImpl.java | 15 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 16 +-
.../ozone/client/io/ECBlockInputStreamFactory.java | 6 +-
.../client/io/ECBlockInputStreamFactoryImpl.java | 13 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 14 +-
.../io/ECBlockReconstructedStripeInputStream.java | 10 +-
.../hdds/scm/storage/DummyBlockInputStream.java | 9 +-
.../storage/DummyBlockInputStreamWithRetry.java | 9 +-
.../hdds/scm/storage/TestBlockInputStream.java | 24 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 6 +-
.../client/io/TestBlockInputStreamFactoryImpl.java | 14 +-
.../ozone/client/io/TestECBlockInputStream.java | 110 +++-
.../client/io/TestECBlockInputStreamProxy.java | 13 +-
.../io/TestECBlockReconstructedInputStream.java | 10 +-
.../TestECBlockReconstructedStripeInputStream.java | 12 +-
hadoop-hdds/common/pom.xml | 5 -
.../apache/hadoop/hdds/fs/SpaceUsageSource.java | 4 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 88 +--
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 8 +
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../hadoop/util/PerformanceMetricsInitializer.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 9 +-
.../hadoop/hdds/fs/MockSpaceUsageSource.java | 22 +
.../container/common/helpers/ContainerMetrics.java | 19 +
.../container/common/impl/HddsDispatcher.java | 77 ++-
.../container/common/volume/StorageVolume.java | 5 -
.../ozone/container/common/volume/VolumeInfo.java | 93 +--
.../container/common/volume/VolumeInfoMetrics.java | 6 +-
.../ozone/container/common/volume/VolumeUsage.java | 125 ++--
.../ECReconstructionCoordinator.java | 7 +-
.../container/keyvalue/KeyValueContainerCheck.java | 6 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 2 +
.../ozoneimpl/ContainerScannerConfiguration.java | 4 +-
.../container/common/impl/TestHddsDispatcher.java | 6 +-
.../common/volume/TestReservedVolumeSpace.java | 28 +-
.../TestContainerScannerConfiguration.java | 4 +-
hadoop-hdds/docs/.gitignore | 2 +
hadoop-hdds/docs/content/interface/HttpFS.md | 2 +-
hadoop-hdds/docs/content/interface/ReconApi.zh.md | 600 +++++++++++++++++-
hadoop-hdds/docs/pom.xml | 24 -
hadoop-hdds/framework/pom.xml | 2 +
hadoop-hdds/hadoop-dependency-test/pom.xml | 5 +
hadoop-hdds/pom.xml | 39 --
hadoop-hdds/rocks-native/pom.xml | 3 +-
.../org/apache/ozone/graph/TestPrintableGraph.java | 84 +++
.../hadoop/hdds/scm/ha/SCMRatisResponse.java | 20 +-
.../hdds/scm/server/SCMContainerMetrics.java | 1 -
.../cli/datanode/DecommissionStatusSubCommand.java | 94 ++-
.../client/checksum/ECBlockChecksumComputer.java | 50 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 23 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 34 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 25 +-
.../ozone/client/io/TestKeyInputStreamEC.java | 12 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 211 +++----
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 20 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 6 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 9 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 14 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 120 ++--
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 29 +-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 154 ++---
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 17 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 8 +-
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 38 +-
hadoop-ozone/dev-support/checks/rat.sh | 8 +-
hadoop-ozone/dist/.gitignore | 1 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 3 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 12 +-
.../smoketest/security/ozone-secure-tenant.robot | 2 +-
.../dist/src/main/smoketest/topology/cli.robot | 21 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 6 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 5 -
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 1 +
hadoop-ozone/httpfsgateway/pom.xml | 4 -
hadoop-ozone/integration-test/pom.xml | 5 -
.../fs/contract/AbstractContractAppendTest.java | 2 +
.../fs/contract/AbstractContractConcatTest.java | 2 +
.../AbstractContractCopyFromLocalTest.java | 2 +
.../AbstractContractGetFileStatusTest.java | 2 +
.../AbstractContractMultipartUploaderTest.java | 4 +
.../fs/contract/AbstractContractOpenTest.java | 2 +
.../AbstractContractRootDirectoryTest.java | 2 +
.../fs/contract/AbstractContractSeekTest.java | 4 +
.../fs/contract/AbstractContractSetTimesTest.java | 2 +
.../fs/contract/AbstractContractUnbufferTest.java | 2 +
.../ozone/AbstractRootedOzoneFileSystemTest.java | 111 +++-
.../org/apache/hadoop/fs/ozone/TestSafeMode.java | 2 +-
.../hdds/scm/TestStorageContainerManager.java | 383 ++++++------
.../scm/TestStorageContainerManagerHelper.java | 198 ------
.../scm/node/TestDecommissionAndMaintenance.java | 2 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 19 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +
.../hadoop/ozone/MiniOzoneClusterProvider.java | 34 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 +
.../org/apache/hadoop/ozone/TestBlockTokens.java | 7 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 29 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 34 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 66 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 4 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 6 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 29 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 557 +++++++++--------
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 67 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 18 +-
.../ozone/om/multitenant/RangerUserRequest.java | 28 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 68 +-
.../tools/contract/AbstractContractDistCpTest.java | 2 +
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 14 +-
.../hadoop/ozone/om/helpers/OzoneAclStorage.java | 15 +-
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 9 +-
.../ozone/om/helpers/TestOmPrefixInfoCodec.java | 2 +-
hadoop-ozone/ozone-manager/pom.xml | 5 -
.../hadoop/ozone/om/BucketUtilizationMetrics.java | 118 ++++
.../org/apache/hadoop/ozone/om/ListIterator.java | 2 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 34 -
.../org/apache/hadoop/ozone/om/OzoneManager.java | 23 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 5 +-
.../RangerRestMultiTenantAccessController.java | 681 ---------------------
.../ozone/om/ratis/OzoneManagerStateMachine.java | 16 +-
.../om/request/bucket/OMBucketCreateRequest.java | 3 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +-
.../ozone/om/TestBucketUtilizationMetrics.java | 126 ++++
.../apache/hadoop/ozone/om/TestChunkStreams.java | 9 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 7 +-
.../request/TestOMClientRequestWithUserInfo.java | 92 ++-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../om/request/key/TestOMKeyCreateRequest.java | 2 +-
.../TestS3InitiateMultipartUploadRequest.java | 11 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../key/acl/prefix/TestOMPrefixAclResponse.java | 4 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 11 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 18 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 6 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 1 +
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 2 +
hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 1 +
hadoop-ozone/pom.xml | 67 +-
hadoop-ozone/recon/.gitignore | 1 +
.../webapps/recon/ozone-recon-web/api/db.json | 67 ++
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 15 +-
.../ozone-recon-web/src/views/insights/om/om.tsx | 120 +++-
.../hadoop/ozone/recon/api/TestEndpoints.java | 12 +-
.../hadoop/ozone/recon/common/CommonUtils.java | 5 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 19 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 14 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 66 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 3 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 2 +-
.../hadoop/ozone/freon/DatanodeSimulator.java | 4 +-
.../freon/containergenerator/GeneratorOm.java | 4 +-
.../apache/hadoop/ozone/repair/OzoneRepair.java | 64 ++
.../org/apache/hadoop/ozone/repair/RDBRepair.java | 58 ++
.../hadoop/ozone/repair/om/SnapshotRepair.java | 200 ++++++
.../hadoop/ozone/repair/om/package-info.java | 22 +
.../apache/hadoop/ozone/repair/package-info.java | 22 +
.../ozone/shell/tenant/GetUserInfoHandler.java | 44 +-
.../shell/tenant/TenantAssignAdminHandler.java | 21 +-
.../ozone/shell/tenant/TenantCreateHandler.java | 15 +-
.../ozone/shell/tenant/TenantDeleteHandler.java | 17 +-
.../ozone/shell/tenant/TenantListHandler.java | 31 +-
.../ozone/shell/tenant/TenantListUsersHandler.java | 19 +-
.../shell/tenant/TenantRevokeAdminHandler.java | 21 +-
pom.xml | 58 +-
181 files changed, 3879 insertions(+), 2948 deletions(-)
diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java
index 2c30abd3d5,b62415395d..b66999de1f
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java
@@@ -113,30 -112,31 +114,36 @@@ public class BlockInputStream extends B
private final Function<BlockID, BlockLocationInfo> refreshFunction;
- public BlockInputStream(BlockLocationInfo blockInfo, Pipeline pipeline,
- Token<OzoneBlockTokenIdentifier> token, boolean verifyChecksum,
- public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline,
++ public BlockInputStream(
++ BlockLocationInfo blockInfo,
++ Pipeline pipeline,
+ Token<OzoneBlockTokenIdentifier> token,
XceiverClientFactory xceiverClientFactory,
- Function<BlockID, BlockLocationInfo> refreshFunction) {
+ Function<BlockID, BlockLocationInfo> refreshFunction,
+ OzoneClientConfig config) {
- this.blockID = blockId;
- this.length = blockLen;
+ this.blockInfo = blockInfo;
+ this.blockID = blockInfo.getBlockID();
+ this.length = blockInfo.getLength();
setPipeline(pipeline);
tokenRef.set(token);
- this.verifyChecksum = verifyChecksum;
+ this.verifyChecksum = config.isChecksumVerify();
this.xceiverClientFactory = xceiverClientFactory;
this.refreshFunction = refreshFunction;
+ this.retryPolicy =
+ HddsClientUtils.createRetryPolicy(config.getMaxReadRetryCount(),
+ TimeUnit.SECONDS.toMillis(config.getReadRetryInterval()));
}
+ // only for unit tests
public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline,
Token<OzoneBlockTokenIdentifier> token,
- boolean verifyChecksum,
- XceiverClientFactory xceiverClientFactory) {
+ XceiverClientFactory xceiverClientFactory,
+ OzoneClientConfig config
+ ) {
- this(blockId, blockLen, pipeline, token,
- xceiverClientFactory, null, config);
+ this(new BlockLocationInfo(new BlockLocationInfo.Builder().setBlockID(blockId).setLength(blockLen)),
- pipeline, token, verifyChecksum,
- xceiverClientFactory, null);
++ pipeline, token, xceiverClientFactory, null, config);
}
+
/**
* Initialize the BlockInputStream. Get the BlockData (list of chunks) from
* the Container and create the ChunkInputStreams for each Chunk in the Block.
diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
index b9233f42d5,6bcdc3c481..7edc498cf6
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
@@@ -76,16 -77,18 +77,18 @@@ public class BlockInputStreamFactoryImp
*/
public BlockExtendedInputStream create(ReplicationConfig repConfig,
BlockLocationInfo blockInfo, Pipeline pipeline,
- Token<OzoneBlockTokenIdentifier> token, boolean verifyChecksum,
+ Token<OzoneBlockTokenIdentifier> token,
XceiverClientFactory xceiverFactory,
- Function<BlockID, BlockLocationInfo> refreshFunction) {
+ Function<BlockID, BlockLocationInfo> refreshFunction,
+ OzoneClientConfig config) {
if (repConfig.getReplicationType().equals(HddsProtos.ReplicationType.EC)) {
return new ECBlockInputStreamProxy((ECReplicationConfig)repConfig,
- blockInfo, verifyChecksum, xceiverFactory, refreshFunction,
- ecBlockStreamFactory);
+ blockInfo, xceiverFactory, refreshFunction,
+ ecBlockStreamFactory, config);
} else {
- return new BlockInputStream(blockInfo, pipeline, token, verifyChecksum, xceiverFactory,
- refreshFunction);
- return new BlockInputStream(blockInfo.getBlockID(), blockInfo.getLength(),
++ return new BlockInputStream(blockInfo,
+ pipeline, token, xceiverFactory, refreshFunction,
+ config);
}
}
diff --cc hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java
index ca3199d8ac,a89097533d..2987a9b613
--- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java
+++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java
@@@ -23,8 -23,8 +23,9 @@@ import java.util.Map
import java.util.function.Function;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+ import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
@@@ -48,11 -48,10 +49,11 @@@ class DummyBlockInputStream extends Blo
XceiverClientFactory xceiverClientManager,
Function<BlockID, BlockLocationInfo> refreshFunction,
List<ChunkInfo> chunkList,
- Map<String, byte[]> chunks) {
+ Map<String, byte[]> chunks,
+ OzoneClientConfig config) {
- super(blockId, blockLen, pipeline, token,
+ super(new BlockLocationInfo(new BlockLocationInfo.Builder().setBlockID(blockId).setLength(blockLen)),
- pipeline, token, verifyChecksum,
- xceiverClientManager, refreshFunction);
++ pipeline, token,
+ xceiverClientManager, refreshFunction, config);
this.chunkDataMap = chunks;
this.chunks = chunkList;
diff --cc hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java
index d66c76dcdd,6d12614228..172e62887b
--- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java
+++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java
@@@ -23,8 -23,8 +23,9 @@@ import java.util.Map
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+ import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.MockPipeline;
diff --cc hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
index f755817816,21b088ce85..0012d691f9
--- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
+++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
@@@ -22,9 -22,10 +22,11 @@@ import com.google.common.primitives.Byt
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ContainerBlockID;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+ import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@@ -408,12 -411,14 +419,15 @@@ public class TestBlockInputStream
.thenReturn(blockLocationInfo);
when(blockLocationInfo.getPipeline()).thenReturn(newPipeline);
+ OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
+ clientConfig.setChecksumVerify(false);
- BlockInputStream subject = new BlockInputStream(blockID, blockSize,
+ BlockInputStream subject = new BlockInputStream(
+ new BlockLocationInfo(new BlockLocationInfo.Builder().setBlockID(blockID).setLength(blockSize)),
- pipeline, null, false, clientFactory, refreshFunction) {
+ pipeline, null, clientFactory, refreshFunction,
+ clientConfig) {
@Override
- protected List<ChunkInfo> getChunkInfoListUsingClient() {
- return chunks;
+ protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) {
+ return stream;
}
@Override
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
index 6b6be1abd4,2d40841ee4..15e84cf370
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
@@@ -59,13 -59,11 +60,13 @@@ public class KeyInputStream extends Mul
OmKeyInfo keyInfo,
List<OmKeyLocationInfo> blockInfos,
XceiverClientFactory xceiverClientFactory,
- boolean verifyChecksum,
Function<OmKeyInfo, OmKeyInfo> retryFunction,
- BlockInputStreamFactory blockStreamFactory) {
+ BlockInputStreamFactory blockStreamFactory,
+ OzoneClientConfig config) {
+ boolean isHsyncFile = keyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID);
List<BlockExtendedInputStream> partStreams = new ArrayList<>();
- for (OmKeyLocationInfo omKeyLocationInfo : blockInfos) {
+ for (int i = 0; i < blockInfos.size(); i++) {
+ OmKeyLocationInfo omKeyLocationInfo = blockInfos.get(i);
if (LOG.isDebugEnabled()) {
LOG.debug("Adding stream for accessing {}. The stream will be " +
"initialized later.", omKeyLocationInfo);
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index 85e7c2a76e,47f5314bd6..845140293a
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@@ -304,23 -357,24 +357,24 @@@ public class TestOmMetrics
RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
doKeyOps(keyArgs);
- MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
- assertCounter("NumKeyOps", 8L, omMetrics);
- assertCounter("NumKeyAllocate", 1L, omMetrics);
- assertCounter("NumKeyLookup", 1L, omMetrics);
- assertCounter("NumKeyDeletes", 1L, omMetrics);
- assertCounter("NumKeyLists", 1L, omMetrics);
- assertCounter("NumTrashKeyLists", 1L, omMetrics);
- assertCounter("NumKeys", 0L, omMetrics);
- assertCounter("NumInitiateMultipartUploads", 1L, omMetrics);
- assertCounter("NumListOpenFiles", 1L, omMetrics);
+ omMetrics = getMetrics("OMMetrics");
+
- assertEquals(initialNumKeyOps + 7, getLongCounter("NumKeyOps", omMetrics));
++ assertEquals(initialNumKeyOps + 8, getLongCounter("NumKeyOps", omMetrics));
+ assertEquals(initialNumKeyAllocate + 1, getLongCounter("NumKeyAllocate", omMetrics));
+ assertEquals(initialNumKeyLookup + 1, getLongCounter("NumKeyLookup", omMetrics));
+ assertEquals(initialNumKeyDeletes + 1, getLongCounter("NumKeyDeletes", omMetrics));
+ assertEquals(initialNumKeyLists + 1, getLongCounter("NumKeyLists", omMetrics));
+ assertEquals(initialNumTrashKeyLists + 1, getLongCounter("NumTrashKeyLists", omMetrics));
- assertEquals(initialNumKeys, getLongCounter("NumKeys", omMetrics));
++ assertEquals(initialNumKeys + 1, getLongCounter("NumKeys", omMetrics));
+ assertEquals(initialNumInitiateMultipartUploads + 1, getLongCounter("NumInitiateMultipartUploads", omMetrics));
keyArgs = createKeyArgs(volumeName, bucketName,
new ECReplicationConfig("rs-3-2-1024K"));
doKeyOps(keyArgs);
+
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumKeyOps", 16L, omMetrics);
- assertCounter("EcKeyCreateTotal", 1L, omMetrics);
- assertEquals(initialNumKeys, getLongCounter("NumKeys", omMetrics));
++ assertEquals(initialNumKeys + 2, getLongCounter("NumKeys", omMetrics));
+ assertEquals(initialEcKeyCreateTotal + 1, getLongCounter("EcKeyCreateTotal", omMetrics));
keyArgs = createKeyArgs(volumeName, bucketName,
RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
@@@ -371,21 -425,22 +425,22 @@@
doKeyOps(keyArgs);
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumKeyOps", 31L, omMetrics);
- assertCounter("NumKeyAllocate", 6L, omMetrics);
- assertCounter("NumKeyLookup", 3L, omMetrics);
- assertCounter("NumKeyDeletes", 4L, omMetrics);
- assertCounter("NumKeyLists", 3L, omMetrics);
- assertCounter("NumTrashKeyLists", 3L, omMetrics);
- assertCounter("NumInitiateMultipartUploads", 3L, omMetrics);
-
- assertCounter("NumKeyAllocateFails", 1L, omMetrics);
- assertCounter("NumKeyLookupFails", 1L, omMetrics);
- assertCounter("NumKeyDeleteFails", 1L, omMetrics);
- assertCounter("NumKeyListFails", 1L, omMetrics);
- assertCounter("NumTrashKeyListFails", 1L, omMetrics);
- assertCounter("NumInitiateMultipartUploadFails", 1L, omMetrics);
- assertCounter("NumKeys", 2L, omMetrics);
- assertEquals(initialNumKeyOps + 28, getLongCounter("NumKeyOps", omMetrics));
++ assertEquals(initialNumKeyOps + 31, getLongCounter("NumKeyOps", omMetrics));
+ assertEquals(initialNumKeyAllocate + 6, getLongCounter("NumKeyAllocate", omMetrics));
+ assertEquals(initialNumKeyLookup + 3, getLongCounter("NumKeyLookup", omMetrics));
+ assertEquals(initialNumKeyDeletes + 4, getLongCounter("NumKeyDeletes", omMetrics));
+ assertEquals(initialNumKeyLists + 3, getLongCounter("NumKeyLists", omMetrics));
+ assertEquals(initialNumTrashKeyLists + 3, getLongCounter("NumTrashKeyLists", omMetrics));
+ assertEquals(initialNumInitiateMultipartUploads + 3, getLongCounter("NumInitiateMultipartUploads", omMetrics));
+
+ assertEquals(initialNumKeyAllocateFails + 1, getLongCounter("NumKeyAllocateFails", omMetrics));
+ assertEquals(initialNumKeyLookupFails + 1, getLongCounter("NumKeyLookupFails", omMetrics));
+ assertEquals(initialNumKeyDeleteFails + 1, getLongCounter("NumKeyDeleteFails", omMetrics));
+ assertEquals(initialNumKeyListFails + 1, getLongCounter("NumKeyListFails", omMetrics));
+ assertEquals(initialNumTrashKeyListFails + 1, getLongCounter("NumTrashKeyListFails", omMetrics));
+ assertEquals(initialNumInitiateMultipartUploadFails + 1, getLongCounter(
+ "NumInitiateMultipartUploadFails", omMetrics));
+ assertEquals(initialNumKeys + 2, getLongCounter("NumKeys", omMetrics));
keyArgs = createKeyArgs(volumeName, bucketName,
new ECReplicationConfig("rs-3-2-1024K"));
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
index df7f5b67b4,e0d5ef4084..33a33ad807
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
@@@ -90,7 -94,10 +94,10 @@@ public class TestChunkStreams
}
private BlockInputStream createStream(byte[] buf, int offset) {
- return new BlockInputStream(null, 100L, null, null, true, null) {
+ OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
+ clientConfig.setChecksumVerify(true);
- return new BlockInputStream(null, 100, null, null, null,
++ return new BlockInputStream(null, 100L, null, null, null,
+ clientConfig) {
private long pos;
private final ByteArrayInputStream in =
new ByteArrayInputStream(buf, offset, 100);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org