You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2021/06/16 17:50:44 UTC
[ozone] branch HDDS-3698-nonrolling-upgrade updated (b335431 ->
92eb6a2)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a change to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from b335431 HDDS-5336. Fix datanode capacity related race condition. (#2327)
add f7ac8c1 HDDS-5307. Remove checkAclRight method, duplicates checkAclRights (#2323)
add 6e6aefe HDDS-5315 Skip storing unwanted block tokens on OM DB (#2311)
add ac7166b HDDS-5263. SCM may stay in safe mode forever after a unclean shutdown of SCM. (#2294)
add 2262395 HDDS-4913. Refine the native authorizer parent context right check. (#2012)
add 7e59cb8 HDDS-5334. Remove unncessary log added durig HDDS-5263. (#2326)
add 838d5d8 HDDS-5324. Shouldn't dectNumKeys when delete keys fail (#2319)
add d45819f HDDS-5329. Remove lockmanager and synchronize on ContainerInfo in Replication Manager (#2325)
add ad02d6c HDDS-5316. Fix datanode reserved space calculation. (#2310)
add 9cf7d31 HDDS-5268. Ensure disk checker also scans the ratis log disks periodically (#2290)
add 623e1d9 HDDS-5344. Adding debug log for block token verification. (#2333)
new 92eb6a2 Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 12 -
.../apache/hadoop/ozone/HddsDatanodeService.java | 7 +-
.../common/impl/StorageLocationReport.java | 55 ++-
.../common/statemachine/DatanodeConfiguration.java | 115 +++++-
.../common/statemachine/DatanodeStateMachine.java | 4 +-
.../states/endpoint/VersionEndpointTask.java | 14 +-
.../transport/server/ratis/XceiverServerRatis.java | 51 +--
.../container/common/utils/HddsVolumeUtil.java | 11 -
.../StorageVolumeUtil.java} | 47 +--
.../ozone/container/common/volume/HddsVolume.java | 174 ++-------
.../container/common/volume/HddsVolumeFactory.java | 93 +++++
.../common/volume/ImmutableVolumeSet.java | 8 +-
.../container/common/volume/MetadataVolume.java | 42 ++-
.../common/volume/MetadataVolumeFactory.java | 56 +++
.../container/common/volume/MutableVolumeSet.java | 344 +++++++----------
.../container/common/volume/StorageVolume.java | 214 +++++++++++
...olumeChecker.java => StorageVolumeChecker.java} | 138 +++----
.../common/volume/StorageVolumeFactory.java | 60 +++
.../ozone/container/common/volume/VolumeInfo.java | 56 ++-
.../ozone/container/common/volume/VolumeSet.java | 4 +-
.../ozone/container/common/volume/VolumeUsage.java | 30 +-
.../container/keyvalue/KeyValueContainer.java | 8 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 6 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../container/keyvalue/helpers/ChunkUtils.java | 2 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 49 ++-
.../container/common/TestBlockDeletingService.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 15 +-
.../common/impl/TestContainerPersistence.java | 6 +-
.../container/common/impl/TestHddsDispatcher.java | 11 +-
.../statemachine/TestDatanodeConfiguration.java | 55 ++-
.../container/common/volume/TestHddsVolume.java | 116 ++++++
...eChecker.java => TestStorageVolumeChecker.java} | 40 +-
.../container/common/volume/TestVolumeSet.java | 48 +--
.../common/volume/TestVolumeSetDiskChecks.java | 70 ++--
.../keyvalue/TestKeyValueBlockIterator.java | 6 +-
.../container/keyvalue/TestKeyValueContainer.java | 9 +-
.../keyvalue/TestKeyValueContainerCheck.java | 6 +-
.../container/keyvalue/TestKeyValueHandler.java | 5 +-
.../container/ozoneimpl/TestContainerReader.java | 11 +-
.../container/ozoneimpl/TestOzoneContainer.java | 16 +-
.../hdds/security/token/BlockTokenVerifier.java | 8 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 16 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 +
.../hdds/scm/block/SCMBlockDeletingService.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 221 ++++++-----
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 60 +++
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 69 +++-
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 2 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 56 ++-
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 8 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 85 +++--
.../safemode/OneReplicaPipelineSafeModeRule.java | 70 ++--
.../hdds/scm/safemode/SCMSafeModeManager.java | 41 +-
.../hadoop/hdds/scm/safemode/SafeModeExitRule.java | 9 +
.../hdds/scm/server/StorageContainerManager.java | 11 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 2 +-
.../hdds/scm/container/TestReplicationManager.java | 3 -
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 3 +
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 5 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 6 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 6 +-
.../TestOneReplicaPipelineSafeModeRule.java | 2 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 33 +-
.../ozone/container/common/TestEndPoint.java | 5 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 6 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 25 --
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 6 +
.../container/metrics/TestContainerMetrics.java | 11 +-
.../server/TestSecureContainerServer.java | 6 +-
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 19 +-
.../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 4 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 22 +-
.../TestDatanodeHddsVolumeFailureToleration.java | 16 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 1 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 6 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 1 -
.../ozone/security/acl/OzoneNativeAuthorizer.java | 35 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 411 +++++++++++++++++++++
.../apache/hadoop/ozone/debug/DatanodeLayout.java | 8 +-
.../ozone/debug/container/ContainerCommands.java | 14 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 4 +-
.../ozone/freon/ClosedContainerReplicator.java | 6 +-
.../containergenerator/GeneratorDatanode.java | 7 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 4 +-
86 files changed, 2365 insertions(+), 998 deletions(-)
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/{interfaces/VolumeChoosingPolicy.java => utils/StorageVolumeUtil.java} (50%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolumeFactory.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneTrash.java => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MetadataVolume.java (54%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MetadataVolumeFactory.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
rename hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/{HddsVolumeChecker.java => StorageVolumeChecker.java} (77%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeFactory.java
rename hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/{TestHddsVolumeChecker.java => TestStorageVolumeChecker.java} (89%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'ssh-upstream/master'
into ssh-upstream-upgrade-branch
Posted by av...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 92eb6a2b13f7aa28c1652ee9467e117b725acfcf
Merge: b335431 623e1d9
Author: Aravindan Vijayan <av...@cloudera.com>
AuthorDate: Wed Jun 16 10:50:09 2021 -0700
Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
.../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 12 -
.../apache/hadoop/ozone/HddsDatanodeService.java | 7 +-
.../common/impl/StorageLocationReport.java | 55 ++-
.../common/statemachine/DatanodeConfiguration.java | 115 +++++-
.../common/statemachine/DatanodeStateMachine.java | 4 +-
.../states/endpoint/VersionEndpointTask.java | 14 +-
.../transport/server/ratis/XceiverServerRatis.java | 51 +--
.../container/common/utils/HddsVolumeUtil.java | 11 -
.../container/common/utils/StorageVolumeUtil.java | 51 +++
.../ozone/container/common/volume/HddsVolume.java | 174 ++-------
.../container/common/volume/HddsVolumeFactory.java | 93 +++++
.../common/volume/ImmutableVolumeSet.java | 8 +-
...ImmutableVolumeSet.java => MetadataVolume.java} | 57 ++-
.../common/volume/MetadataVolumeFactory.java | 56 +++
.../container/common/volume/MutableVolumeSet.java | 344 +++++++----------
.../container/common/volume/StorageVolume.java | 214 +++++++++++
...olumeChecker.java => StorageVolumeChecker.java} | 138 +++----
.../common/volume/StorageVolumeFactory.java | 60 +++
.../ozone/container/common/volume/VolumeInfo.java | 56 ++-
.../ozone/container/common/volume/VolumeSet.java | 4 +-
.../ozone/container/common/volume/VolumeUsage.java | 30 +-
.../container/keyvalue/KeyValueContainer.java | 8 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 6 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../container/keyvalue/helpers/ChunkUtils.java | 2 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 49 ++-
.../container/common/TestBlockDeletingService.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 15 +-
.../common/impl/TestContainerPersistence.java | 6 +-
.../container/common/impl/TestHddsDispatcher.java | 11 +-
.../statemachine/TestDatanodeConfiguration.java | 55 ++-
.../container/common/volume/TestHddsVolume.java | 116 ++++++
...eChecker.java => TestStorageVolumeChecker.java} | 40 +-
.../container/common/volume/TestVolumeSet.java | 48 +--
.../common/volume/TestVolumeSetDiskChecks.java | 70 ++--
.../keyvalue/TestKeyValueBlockIterator.java | 6 +-
.../container/keyvalue/TestKeyValueContainer.java | 9 +-
.../keyvalue/TestKeyValueContainerCheck.java | 6 +-
.../container/keyvalue/TestKeyValueHandler.java | 5 +-
.../container/ozoneimpl/TestContainerReader.java | 11 +-
.../container/ozoneimpl/TestOzoneContainer.java | 16 +-
.../hdds/security/token/BlockTokenVerifier.java | 8 +
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 16 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 +
.../hdds/scm/block/SCMBlockDeletingService.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 221 ++++++-----
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 60 +++
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 69 +++-
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 2 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 56 ++-
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 8 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 85 +++--
.../safemode/OneReplicaPipelineSafeModeRule.java | 70 ++--
.../hdds/scm/safemode/SCMSafeModeManager.java | 41 +-
.../hadoop/hdds/scm/safemode/SafeModeExitRule.java | 9 +
.../hdds/scm/server/StorageContainerManager.java | 11 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 2 +-
.../hdds/scm/container/TestReplicationManager.java | 3 -
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 3 +
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 5 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 6 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 6 +-
.../TestOneReplicaPipelineSafeModeRule.java | 2 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 33 +-
.../ozone/container/common/TestEndPoint.java | 5 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 6 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 25 --
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 6 +
.../container/metrics/TestContainerMetrics.java | 11 +-
.../server/TestSecureContainerServer.java | 6 +-
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 19 +-
.../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 4 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 22 +-
.../TestDatanodeHddsVolumeFailureToleration.java | 16 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 1 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 6 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 1 -
.../ozone/security/acl/OzoneNativeAuthorizer.java | 35 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 411 +++++++++++++++++++++
.../apache/hadoop/ozone/debug/DatanodeLayout.java | 8 +-
.../ozone/debug/container/ContainerCommands.java | 14 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 4 +-
.../ozone/freon/ClosedContainerReplicator.java | 6 +-
.../containergenerator/GeneratorDatanode.java | 7 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 4 +-
86 files changed, 2390 insertions(+), 992 deletions(-)
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
index 44d4c41,9d3810e..35dab34
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
@@@ -22,10 -22,8 +22,8 @@@ import com.google.common.annotations.Vi
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
-import org.apache.hadoop.ozone.container.common.DataNodeLayoutVersion;
+import org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
- import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
- import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index 677d25f,6da4599..0099182
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@@ -18,28 -18,18 +18,20 @@@
package org.apache.hadoop.ozone.container.common.volume;
+import static org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion;
+
+import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
- import java.util.Objects;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
- import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
- import org.apache.hadoop.hdds.conf.ConfigurationSource;
- import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
- import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
- import org.apache.hadoop.hdfs.server.datanode.checker.Checkable;
- import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
-import org.apache.hadoop.ozone.container.common.DataNodeLayoutVersion;
import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile;
import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
- import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 408b83d,f0d388b..0f8ea75
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -126,9 -122,6 +126,8 @@@ import org.apache.hadoop.ozone.OzoneCon
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.lease.LeaseManager;
- import org.apache.hadoop.ozone.lock.LockManager;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org