You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2020/11/20 22:54:00 UTC
[ozone] branch HDDS-3698-upgrade updated (7a26250 -> 4384789)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a change to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 7a26250 HDDS-4342. Add DataNode state and transitions for a node going through upgrade. (#1508)
add 8bcd859 HDDS-4420. Missing a container shouldn't log message at ERROR level (#1554)
add 1f6b0d9 HDDS-4446. Update Jetty version to 9.4.34. (#1566)
add 2ef4cf9 HDDS-4445. Owner info is not passed to authorizer for BUCKET/KEY crea… (#1563)
add 3b53f9f HDDS-4436.TestReplicationManager#testUnderReplicatedQuasiClosedContainerWithUnhealthyReplica fails intermittently. (#1560)
add 50e5765 HDDS-4438. Avoid counting skipped transactions belonging to open containers. (#1565)
add fd9d0ce HDDS-4435. Fix can not insert the second entry when hive on ozone (#1559)
add 90190f5 HDDS-4307.Start Trash Emptier in Ozone Manager (#1507)
add a28cda5 HDDS-4363. Add metric to track the number of RocksDB open/close operations. (#1530)
add fdbd5da HDDS-4417. Simplify Ozone client code with configuration object (#1542)
add d2b7135 HDDS-4437. Avoid unnecessary builder conversion in setting volume Quota/Owner request (#1564)
add f3c8900 HDDS-4441. Add metrics for ACL related operations. (#1571)
add 8523775 HDDS-4455. Fix typo in README.md doc (#1578)
add e56d7bc HDDS-4315. Use Epoch to generate unique ObjectIDs (#1480)
add 7a52990 HDDS-4443. Recon: Using Mysql database throws exception and fails startup (#1570)
add 277a589 HDDS-4408: terminate Datanode when Datanode State Machine Thread got uncaught exception. (#1533)
add 5e8aaee HDDS-4453. Replicate closed container for random selected datanodes. (#1574)
add 5ec98e1 HDDS-4083. Create ZH translation of Recon.md in doc (#1575)
add 760c1e8 HDDS-4337. Implement RocksDB options cache for new datanode DB utilities. (#1544)
add 0e35176 HDDS-4338. Fix the issue that SCM web UI banner shows "HDFS SCM". (#1583)
add b80d9b0 HDDS-4235. Ozone client FS path validation is not present in OFS. (#1582)
add 5f43814 HDDS-4184. Add Features menu for Chinese document. (#1547)
add 45e30d6 HDDS-4079. Create ZH translation of Containers.md in doc. (#1539)
add fcf0a4d HDDS-4080. Create ZH translation of OzoneManager.md in doc. (#1541)
add 7df5dba HDDS-4081. Create ZH translation of StorageContainerManager.md in doc. (#1558)
add 585984e HDDS-4441. Add metrics for ACL related operations.(Addendum for HA). (#1584)
add d203b64 HDDS-4442. Disable the location information of audit logger to reduce overhead (#1567)
add 787e5b6 HDDS-4458. Fix Max Transaction ID value in OM. (#1585)
add e738172 HDDS-4466. Update url in .asf.yaml to use TLP project (#1592)
add 9062c5b HDDS-4467. Acceptance test fails due to new Hadoop 3 image (#1594)
add 1cd5863 HDDS-4464. Upgrade httpclient version due to CVE-2020-13956. (#1590)
add d0aa34c HDDS-4082. Create ZH translation of HA.md in doc. (#1591)
add 04a757a HDDS-4462. Add --frozen-lockfile to pnpm install to prevent ozone-recon-web/pnpm-lock.yaml from being updated automatically (#1589)
add 4e36d03 HDDS-4367. Configuration for deletion service intervals should be different for OM, SCM and datanodes (#1573)
add 8b2c2eb HDDS-4399. Safe mode rule for piplelines should only consider open pipelines. (#1526)
add 40c1944 HDDS-4426. SCM should create transactions using all blocks received from OM (#1561)
add 0aca5c7 HDDS-4346.Ozone specific Trash Policy (#1535)
add 541ae9f HDDS-4450. Cannot run ozone if HADOOP_HOME points to Hadoop install (#1572)
add bbeaf65 HDDS-4448. Duplicate refreshPipeline in listStatus (#1569)
add 58ec7f4 HDDS-4427. Avoid ContainerCache in ContainerReader at Datanode startup (#1549)
add 417082c HDDS-4452. findbugs.sh couldn't be executed after a full build (#1576)
new 4384789 Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 2 +-
README.md | 2 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 213 +
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 66 +-
.../storage/TestBlockOutputStreamCorrectness.java | 17 +-
.../hdds/scm/storage/TestChunkInputStream.java | 11 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 23 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 73 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 22 +-
.../common/src/main/resources/ozone-default.xml | 91 +-
.../ratis/TestContainerCommandRequestMessage.java | 13 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 8 +-
.../common/src/test/resources/auditlog.properties | 2 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 14 +
.../hadoop/hdds/conf/ConfigurationTarget.java | 4 +
.../common/statemachine/DatanodeConfiguration.java | 24 +
.../common/statemachine/DatanodeStateMachine.java | 6 +
.../container/common/utils/ContainerCache.java | 52 +-
.../common/utils/ContainerCacheMetrics.java | 114 +
.../container/keyvalue/helpers/BlockUtils.java | 54 +
.../keyvalue/helpers/KeyValueContainerUtil.java | 107 +-
.../container/metadata/AbstractDatanodeStore.java | 65 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 12 +-
.../replication/GrpcReplicationClient.java | 23 +-
.../replication/SimpleContainerDownloader.java | 36 +-
.../main/resources/webapps/hddsDatanode/index.html | 2 +-
.../ozone/container/ContainerTestHelper.java | 6 +-
.../container/common/TestBlockDeletingService.java | 9 +-
.../ozone/container/common/TestContainerCache.java | 8 +
.../common/impl/TestContainerPersistence.java | 53 +-
.../container/keyvalue/TestKeyValueContainer.java | 54 +-
.../container/ozoneimpl/TestContainerReader.java | 13 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +
.../replication/TestSimpleContainerDownloader.java | 85 +
hadoop-hdds/docs/content/concept/Containers.zh.md | 47 +
.../docs/content/concept/OzoneManager.zh.md | 92 +-
.../content/concept/StorageContainerManager.zh.md | 69 +-
hadoop-hdds/docs/content/feature/GDPR.md | 2 +-
hadoop-hdds/docs/content/feature/GDPR.zh.md | 5 +-
hadoop-hdds/docs/content/feature/HA.md | 4 +-
hadoop-hdds/docs/content/feature/HA.zh.md | 117 +
hadoop-hdds/docs/content/feature/Quota.zh.md | 3 +
hadoop-hdds/docs/content/feature/Recon.zh.md | 49 +
hadoop-hdds/docs/content/feature/_index.zh.md | 7 +-
hadoop-hdds/docs/content/interface/_index.zh.md | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 351 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 21 +-
hadoop-hdds/pom.xml | 50 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 4 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 50 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 29 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 5 +-
.../IncrementalContainerReportHandler.java | 4 +
.../hadoop/hdds/scm/metadata/PipelineCodec.java | 3 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 8 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 42 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 3 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 45 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 85 +-
.../src/main/resources/webapps/scm/index.html | 6 +-
.../hdds/scm/container/TestReplicationManager.java | 12 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 4 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 6 +
.../TestOneReplicaPipelineSafeModeRule.java | 74 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 34 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 135 +-
.../client/io/BlockOutputStreamEntryPool.java | 89 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 115 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 99 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 59 +-
.../org/apache/hadoop/ozone/util/ExitManager.java | 2 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 5 +
hadoop-ozone/datanode/pom.xml | 23 +
hadoop-ozone/dist/pom.xml | 48 +-
.../compose/ozonesecure-om-ha/docker-compose.yaml | 4 +-
.../main/compose/ozonesecure/docker-compose.yaml | 2 +
.../main/smoketest/security/ozone-secure-fs.robot | 17 +
hadoop-ozone/dist/src/shell/ozone/ozone | 11 +-
hadoop-ozone/dist/src/shell/ozone/stop-ozone.sh | 5 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 28 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 13 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 85 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 49 +-
.../hadoop/ozone/TestStorageContainerManager.java | 67 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 34 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 22 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 39 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 34 +-
.../rpc/TestCloseContainerHandlingByClient.java | 35 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 7 +-
.../client/rpc/TestContainerStateMachine.java | 35 +-
.../rpc/TestContainerStateMachineFailures.java | 64 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 6 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 35 +-
.../client/rpc/TestFailureHandlingByClient.java | 7 +-
.../ozone/client/rpc/TestKeyInputStream.java | 33 +-
.../rpc/TestOzoneClientRetriesOnException.java | 43 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 31 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 7 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 23 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 2 +
.../client/rpc/TestValidateBCSIDOnRestart.java | 41 +-
.../ozone/client/rpc/TestWatchForCommit.java | 46 +-
.../commandhandler/TestBlockDeletion.java | 10 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 105 +
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 100 +-
.../src/test/resources/contract/ozone.xml | 5 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 5 +
.../hadoop/ozone/om/ratis/OMTransactionInfo.java | 6 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 18 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 37 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 18 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 112 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 7 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 235 +
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 29 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 2 +-
.../om/request/bucket/OMBucketCreateRequest.java | 4 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 10 +
.../bucket/acl/OMBucketRemoveAclRequest.java | 10 +
.../request/bucket/acl/OMBucketSetAclRequest.java | 9 +
.../om/request/file/OMDirectoryCreateRequest.java | 23 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 1 +
.../ozone/om/request/file/OMFileRequest.java | 30 -
.../ozone/om/request/key/OMKeyCreateRequest.java | 1 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 18 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 9 +
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 9 +
.../om/request/key/acl/OMKeySetAclRequest.java | 9 +
.../S3InitiateMultipartUploadRequest.java | 3 +-
.../om/request/volume/OMVolumeCreateRequest.java | 3 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 9 +
.../volume/acl/OMVolumeRemoveAclRequest.java | 9 +
.../request/volume/acl/OMVolumeSetAclRequest.java | 9 +
...OzoneManagerProtocolServerSideTranslatorPB.java | 9 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 75 +-
.../request/volume/TestOMVolumeCreateRequest.java | 5 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 26 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 28 +-
hadoop-ozone/pom.xml | 50 +-
.../ozone/recon/codegen/ReconSqlDbConfig.java | 2 +-
.../recon/schema/ReconTaskSchemaDefinition.java | 2 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 17167 +++++++++++++++++++
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 7 +-
pom.xml | 6 +-
150 files changed, 20592 insertions(+), 1583 deletions(-)
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCacheMetrics.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java
create mode 100644 hadoop-hdds/docs/content/concept/Containers.zh.md
create mode 100644 hadoop-hdds/docs/content/feature/HA.zh.md
create mode 100644 hadoop-hdds/docs/content/feature/Recon.zh.md
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/pnpm-lock.yaml
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'upstream/master' into
HDDS-3698-upgrade
Posted by av...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4384789415eb7351c674e238d513ee9194a4d7dd
Merge: 7a26250 417082c
Author: Aravindan Vijayan <av...@cloudera.com>
AuthorDate: Thu Nov 19 15:01:51 2020 -0800
Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
.asf.yaml | 2 +-
README.md | 2 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 213 +
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 66 +-
.../storage/TestBlockOutputStreamCorrectness.java | 17 +-
.../hdds/scm/storage/TestChunkInputStream.java | 11 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 23 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 73 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 22 +-
.../common/src/main/resources/ozone-default.xml | 91 +-
.../ratis/TestContainerCommandRequestMessage.java | 13 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 8 +-
.../common/src/test/resources/auditlog.properties | 2 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 14 +
.../hadoop/hdds/conf/ConfigurationTarget.java | 4 +
.../common/statemachine/DatanodeConfiguration.java | 24 +
.../common/statemachine/DatanodeStateMachine.java | 6 +
.../container/common/utils/ContainerCache.java | 52 +-
.../common/utils/ContainerCacheMetrics.java | 114 +
.../container/keyvalue/helpers/BlockUtils.java | 54 +
.../keyvalue/helpers/KeyValueContainerUtil.java | 107 +-
.../container/metadata/AbstractDatanodeStore.java | 65 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 12 +-
.../replication/GrpcReplicationClient.java | 23 +-
.../replication/SimpleContainerDownloader.java | 36 +-
.../main/resources/webapps/hddsDatanode/index.html | 2 +-
.../ozone/container/ContainerTestHelper.java | 6 +-
.../container/common/TestBlockDeletingService.java | 9 +-
.../ozone/container/common/TestContainerCache.java | 8 +
.../common/impl/TestContainerPersistence.java | 53 +-
.../container/keyvalue/TestKeyValueContainer.java | 54 +-
.../container/ozoneimpl/TestContainerReader.java | 13 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +
.../replication/TestSimpleContainerDownloader.java | 85 +
hadoop-hdds/docs/content/concept/Containers.zh.md | 47 +
.../docs/content/concept/OzoneManager.zh.md | 92 +-
.../content/concept/StorageContainerManager.zh.md | 69 +-
hadoop-hdds/docs/content/feature/GDPR.md | 2 +-
hadoop-hdds/docs/content/feature/GDPR.zh.md | 5 +-
hadoop-hdds/docs/content/feature/HA.md | 4 +-
hadoop-hdds/docs/content/feature/HA.zh.md | 117 +
hadoop-hdds/docs/content/feature/Quota.zh.md | 3 +
hadoop-hdds/docs/content/feature/Recon.zh.md | 49 +
hadoop-hdds/docs/content/feature/_index.zh.md | 7 +-
hadoop-hdds/docs/content/interface/_index.zh.md | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 351 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 21 +-
hadoop-hdds/pom.xml | 50 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 4 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 50 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 29 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 5 +-
.../IncrementalContainerReportHandler.java | 4 +
.../hadoop/hdds/scm/metadata/PipelineCodec.java | 3 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 8 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 42 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 3 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 45 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 85 +-
.../src/main/resources/webapps/scm/index.html | 6 +-
.../hdds/scm/container/TestReplicationManager.java | 12 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 4 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 6 +
.../TestOneReplicaPipelineSafeModeRule.java | 74 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 34 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 135 +-
.../client/io/BlockOutputStreamEntryPool.java | 89 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 115 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 99 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 59 +-
.../org/apache/hadoop/ozone/util/ExitManager.java | 2 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 5 +
hadoop-ozone/datanode/pom.xml | 23 +
hadoop-ozone/dist/pom.xml | 48 +-
.../compose/ozonesecure-om-ha/docker-compose.yaml | 4 +-
.../main/compose/ozonesecure/docker-compose.yaml | 2 +
.../main/smoketest/security/ozone-secure-fs.robot | 17 +
hadoop-ozone/dist/src/shell/ozone/ozone | 11 +-
hadoop-ozone/dist/src/shell/ozone/stop-ozone.sh | 5 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 28 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 13 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 85 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 49 +-
.../hadoop/ozone/TestStorageContainerManager.java | 67 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 34 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 22 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 39 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 34 +-
.../rpc/TestCloseContainerHandlingByClient.java | 35 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 7 +-
.../client/rpc/TestContainerStateMachine.java | 35 +-
.../rpc/TestContainerStateMachineFailures.java | 64 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 6 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 35 +-
.../client/rpc/TestFailureHandlingByClient.java | 7 +-
.../ozone/client/rpc/TestKeyInputStream.java | 33 +-
.../rpc/TestOzoneClientRetriesOnException.java | 43 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 31 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 7 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 23 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 2 +
.../client/rpc/TestValidateBCSIDOnRestart.java | 41 +-
.../ozone/client/rpc/TestWatchForCommit.java | 46 +-
.../commandhandler/TestBlockDeletion.java | 10 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 105 +
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 100 +-
.../src/test/resources/contract/ozone.xml | 5 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 5 +
.../hadoop/ozone/om/ratis/OMTransactionInfo.java | 6 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 18 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 37 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 18 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 112 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 7 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 235 +
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 29 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 2 +-
.../om/request/bucket/OMBucketCreateRequest.java | 4 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 10 +
.../bucket/acl/OMBucketRemoveAclRequest.java | 10 +
.../request/bucket/acl/OMBucketSetAclRequest.java | 9 +
.../om/request/file/OMDirectoryCreateRequest.java | 23 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 1 +
.../ozone/om/request/file/OMFileRequest.java | 30 -
.../ozone/om/request/key/OMKeyCreateRequest.java | 1 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 18 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 9 +
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 9 +
.../om/request/key/acl/OMKeySetAclRequest.java | 9 +
.../S3InitiateMultipartUploadRequest.java | 3 +-
.../om/request/volume/OMVolumeCreateRequest.java | 3 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 9 +
.../volume/acl/OMVolumeRemoveAclRequest.java | 9 +
.../request/volume/acl/OMVolumeSetAclRequest.java | 9 +
...OzoneManagerProtocolServerSideTranslatorPB.java | 9 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 75 +-
.../request/volume/TestOMVolumeCreateRequest.java | 5 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 26 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 28 +-
hadoop-ozone/pom.xml | 50 +-
.../ozone/recon/codegen/ReconSqlDbConfig.java | 2 +-
.../recon/schema/ReconTaskSchemaDefinition.java | 2 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 17167 +++++++++++++++++++
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 7 +-
pom.xml | 6 +-
150 files changed, 20592 insertions(+), 1583 deletions(-)
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index fc5b1f5,135fb51..4b68f7e
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@@ -55,10 -28,9 +28,11 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+ import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 0afa44d,2d2c9f4..a36cf2f
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -139,11 -144,7 +144,10 @@@ import org.apache.hadoop.ozone.om.ratis
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
- import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.snapshot.OzoneManagerSnapshotProvider;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManagerImpl;
+import org.apache.hadoop.ozone.om.upgrade.OmLayoutVersionManager;
+import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizer;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
@@@ -347,9 -341,11 +354,10 @@@ public final class OzoneManager extend
// Used in MiniOzoneCluster testing
private State omState;
-
+ private Thread emptier;
- private OzoneManager(OzoneConfiguration conf) throws IOException,
- AuthenticationException {
+ private OzoneManager(OzoneConfiguration conf, boolean forUpgrade)
+ throws IOException, AuthenticationException {
super(OzoneVersionInfo.OZONE_VERSION_INFO);
Preconditions.checkNotNull(conf);
configuration = conf;
@@@ -1230,11 -1183,12 +1238,16 @@@
// Allow OM to start as Http Server failure is not fatal.
LOG.error("OM HttpServer failed to start.", ex);
}
- omRpcServer.start();
- isOmRpcServerRunning = true;
+
+ if (!prepareForUpgrade) {
+ omRpcServer.start();
+ isOmRpcServerRunning = true;
+ }
++
+ // TODO: Start this thread only on the leader node.
+ // Should be fixed after HDDS-4451.
+ startTrashEmptier(configuration);
+
registerMXBean();
startJVMPauseMonitor();
@@@ -3762,10 -3776,6 +3853,9 @@@
}
return omVolumeArgs.build();
-
}
+ public OmLayoutVersionManager getVersionManager() {
+ return versionManager;
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java
index 4a727d1,18f7bef..565871e
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java
@@@ -26,8 -24,8 +26,9 @@@ import java.util.List
import com.google.common.collect.Lists;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType;
import org.apache.hadoop.ozone.util.BooleanBiFunction;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@@ -133,8 -131,12 +134,17 @@@ public class OMBucketAddAclRequest exte
}
}
+ public static String getRequestType() {
+ return AddAcl.name() + "-" + ObjectType.BUCKET.name();
+ }
++
+ @Override
- public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
- long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
++ public OMClientResponse validateAndUpdateCache(
++ OzoneManager ozoneManager, long trxnLogIndex,
++ OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumAddAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java
index b9d924b,cf5ee42..3932c0f
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java
@@@ -24,8 -22,8 +24,9 @@@ import java.io.IOException
import java.util.List;
import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -131,8 -129,12 +132,17 @@@ public class OMBucketRemoveAclRequest e
}
}
+ public static String getRequestType() {
+ return RemoveAcl.name() + "-" + ObjectType.BUCKET.name();
+ }
++
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumRemoveAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
++
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java
index 5d4bfcf,b3d2289..e4e64ba
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java
@@@ -25,8 -23,8 +25,9 @@@ import java.util.ArrayList
import java.util.List;
import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -132,8 -130,12 +133,16 @@@ public class OMBucketSetAclRequest exte
}
}
+ public static String getRequestType() {
+ return SetAcl.name() + "-" + ObjectType.BUCKET.name();
+ }
++
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumSetAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java
index 9a0a3fa,499562b..6a4922c
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java
@@@ -120,8 -118,12 +121,16 @@@ public class OMKeyAddAclRequest extend
return omKeyInfo.addAcl(ozoneAcls.get(0));
}
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumAddAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
++
+ public static String getRequestType() {
+ return AddAcl.name() + "-" + ObjectType.KEY;
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java
index 8b17fe4,d95be51..2484958
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java
@@@ -121,8 -119,12 +122,16 @@@ public class OMKeyRemoveAclRequest exte
return omKeyInfo.removeAcl(ozoneAcls.get(0));
}
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumRemoveAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
++
+ public static String getRequestType() {
+ return RemoveAcl.name() + "-" + ObjectType.KEY;
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java
index c6da84f,9ba1a76..a5d736f
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java
@@@ -117,8 -115,12 +118,16 @@@ public class OMKeySetAclRequest extend
return omKeyInfo.setAcls(ozoneAcls);
}
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumSetAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
++
+ public static String getRequestType() {
+ return SetAcl.name() + "-" + ObjectType.KEY;
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java
index 0a5fadd,a3b9f19..50afa1a
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java
@@@ -132,7 -130,11 +133,15 @@@ public class OMVolumeAddAclRequest exte
}
}
+ public static String getRequestType() {
+ return AddAcl.name() + "-" + ObjectType.VOLUME;
+ }
++
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumAddAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java
index d20e1a9,b4bb1f8..cc5ac72
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java
@@@ -131,7 -129,11 +132,15 @@@ public class OMVolumeRemoveAclRequest e
}
}
+ public static String getRequestType() {
+ return RemoveAcl.name() + "-" + ObjectType.VOLUME;
+ }
++
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumRemoveAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java
index 848a243,a1b0ca6..0c56af5
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java
@@@ -128,7 -126,11 +129,15 @@@ public class OMVolumeSetAclRequest exte
}
}
+ public static String getRequestType() {
+ return SetAcl.name() + "-" + ObjectType.VOLUME;
+ }
++
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
+ ozoneManager.getMetrics().incNumSetAcl();
+ return super.validateAndUpdateCache(ozoneManager, trxnLogIndex,
+ omDoubleBufferHelper);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org