You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2022/06/02 03:37:16 UTC
[ozone] branch HDDS-3630 updated (f94b7eb12d -> 4f58545294)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a change to branch HDDS-3630
in repository https://gitbox.apache.org/repos/asf/ozone.git
from f94b7eb12d Merge remote-tracking branch 'origin/HDDS-3630'
add af3a80d041 HDDS-6808. EC: Fix datanode exclusion check in client (#3460)
add e8abd0f6e2 HDDS-6280. Support Container Balancer HA (#3423)
add e021a06b4c HDDS-6474. Add test to cover the FSO bucket list status with beyond batch boundary and cache. (#3379). Contributed by aswinshakil
add 24ca66b067 HDDS-6705 Add metrics for volume statistics including disk capacity, usage, Reserved (#3430)
new 4f58545294 Merge remote-tracking branch 'origin/HDDS-3630'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../ozone/container/common/volume/HddsVolume.java | 31 +++
.../container/common/volume/MetadataVolume.java | 6 +
.../container/common/volume/VolumeInfoMetrics.java | 137 ++++++++++
.../interface-client/src/main/proto/hdds.proto | 19 ++
.../scm/container/balancer/ContainerBalancer.java | 291 ++++++++++++++++-----
.../balancer/ContainerBalancerConfiguration.java | 84 ++++++
.../IllegalContainerBalancerStateException.java | 5 +-
...lidContainerBalancerConfigurationException.java | 11 +-
.../org/apache/hadoop/hdds/scm/ha/SCMService.java | 2 +-
.../hadoop/hdds/scm/ha/SCMServiceException.java} | 25 +-
.../hadoop/hdds/scm/ha/SCMServiceManager.java | 6 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 23 +-
.../scm/ha/StatefulServiceStateManagerImpl.java | 14 +
.../hadoop/hdds/scm/ha/io/ByteStringCodec.java} | 28 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +
.../hdds/scm/server/SCMClientProtocolServer.java | 16 +-
.../hdds/scm/server/StorageContainerManager.java | 11 +-
.../container/balancer/TestContainerBalancer.java | 157 +++++++----
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 3 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 10 +-
.../ozone/client/MockXceiverClientFactory.java | 60 +++--
.../hadoop/ozone/client/MockXceiverClientSpi.java | 3 +
.../hadoop/ozone/client/TestOzoneECClient.java | 67 ++++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 26 ++
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 93 +++++++
25 files changed, 945 insertions(+), 185 deletions(-)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/utils/db/SequenceNumberNotFoundException.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMServiceException.java} (63%)
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/utils/db/ShortCodec.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java} (62%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'origin/HDDS-3630'
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-3630
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4f58545294e3de92929d2fe814bcb98b98929733
Merge: 24ca66b067 f94b7eb12d
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Thu Jun 2 11:21:52 2022 +0800
Merge remote-tracking branch 'origin/HDDS-3630'
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 6 +-
.../upgrade/AbstractLayoutVersionManager.java | 12 +-
.../common/src/main/resources/ozone-default.xml | 14 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 6 +-
.../container/common/interfaces/DBHandle.java | 48 ++
.../common/statemachine/DatanodeConfiguration.java | 66 ++
.../common/statemachine/DatanodeStateMachine.java | 11 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 90 ++-
.../states/endpoint/VersionEndpointTask.java | 68 +-
.../container/common/utils/ContainerCache.java | 2 +-
.../container/common/utils/DatanodeStoreCache.java | 88 +++
.../container/common/utils/HddsVolumeUtil.java | 230 ++-----
.../hadoop/ozone/container/common/utils/RawDB.java | 42 ++
.../container/common/utils/ReferenceCountedDB.java | 25 +-
.../container/common/utils/StorageVolumeUtil.java | 202 ++++++
.../ozone/container/common/volume/DbVolume.java | 153 +++++
...dataVolumeFactory.java => DbVolumeFactory.java} | 35 +-
.../ozone/container/common/volume/HddsVolume.java | 407 +++++------
.../container/common/volume/HddsVolumeFactory.java | 37 +-
.../container/common/volume/MetadataVolume.java | 5 +
.../common/volume/MetadataVolumeFactory.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 11 +
.../container/common/volume/StorageVolume.java | 278 +++++++-
.../common/volume/StorageVolumeFactory.java | 41 +-
.../container/keyvalue/KeyValueContainer.java | 121 ++--
.../container/keyvalue/KeyValueContainerCheck.java | 24 +-
.../container/keyvalue/KeyValueContainerData.java | 109 ++-
.../KeyValueContainerMetadataInspector.java | 62 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 11 +-
.../container/keyvalue/TarContainerPacker.java | 15 +-
.../container/keyvalue/helpers/BlockUtils.java | 179 ++++-
.../helpers/KeyValueContainerLocationUtil.java | 12 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 194 +++---
.../container/keyvalue/impl/BlockManagerImpl.java | 57 +-
.../background/BlockDeletingService.java | 95 ++-
.../metadata/AbstractDatanodeDBDefinition.java | 11 +-
.../container/metadata/AbstractDatanodeStore.java | 20 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 158 +++++
.../metadata/DatanodeSchemaTwoDBDefinition.java | 6 +-
.../ozone/container/metadata/DatanodeStore.java | 9 +-
.../metadata/DatanodeStoreSchemaOneImpl.java | 7 +-
.../metadata/DatanodeStoreSchemaThreeImpl.java | 130 ++++
.../metadata/DatanodeStoreSchemaTwoImpl.java | 13 +-
.../ozone/container/metadata/DatanodeTable.java | 34 +-
.../container/metadata/DeleteTransactionStore.java | 29 +
.../metadata/SchemaOneDeletedBlocksTable.java | 8 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 33 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 82 +++
.../ScmHAFinalizeUpgradeActionDatanode.java | 2 +-
.../upgrade/VersionedDatanodeFeatures.java | 28 +-
.../ozone/container/common/ContainerTestUtils.java | 32 +
.../container/common/TestBlockDeletingService.java | 167 ++---
.../ozone/container/common/TestContainerCache.java | 2 +-
.../container/common/TestDatanodeStoreCache.java | 78 +++
.../common/TestKeyValueContainerData.java | 20 +-
.../TestSchemaOneBackwardsCompatibility.java | 150 ++--
.../TestSchemaTwoBackwardsCompatibility.java | 367 ++++++++++
.../common/helpers/TestDatanodeVersionFile.java | 18 +-
.../common/impl/TestContainerPersistence.java | 30 +-
.../statemachine/TestDatanodeConfiguration.java | 11 +
.../container/common/utils/TestHddsVolumeUtil.java | 238 +++++++
.../common/utils/TestStorageVolumeUtil.java | 99 +++
.../container/common/volume/TestDbVolume.java | 172 +++++
.../container/common/volume/TestHddsVolume.java | 164 ++++-
.../container/common/volume/TestStorageVolume.java | 83 +++
.../common/volume/TestVolumeSetDiskChecks.java | 30 +
.../keyvalue/ContainerTestVersionInfo.java | 79 +++
.../keyvalue/TestKeyValueBlockIterator.java | 82 ++-
.../container/keyvalue/TestKeyValueContainer.java | 47 +-
.../keyvalue/TestKeyValueContainerCheck.java | 19 +-
.../TestKeyValueContainerIntegrityChecks.java | 44 +-
.../TestKeyValueContainerMetadataInspector.java | 20 +-
.../container/keyvalue/TestTarContainerPacker.java | 82 ++-
.../keyvalue/impl/TestBlockManagerImpl.java | 23 +-
.../container/ozoneimpl/TestContainerReader.java | 85 ++-
.../container/ozoneimpl/TestOzoneContainer.java | 99 +--
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 759 +++++++++++++++++++++
.../upgrade/TestDatanodeUpgradeToScmHA.java | 5 +
.../docs/content/design/dn-merge-rocksdb.md | 29 +
.../docs/content/feature/dn-merge-rocksdb.md | 70 ++
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 7 +
.../hadoop/hdds/utils/MetadataKeyFilters.java | 1 +
.../hdds/utils/db/DBColumnFamilyDefinition.java | 13 +
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 2 +-
.../hadoop/hdds/utils/db/DumpFileLoader.java | 38 ++
.../hadoop/hdds/utils/db/DumpFileWriter.java | 46 ++
.../hdds/utils/db/FixedLengthStringCodec.java | 50 ++
.../hdds/utils/db/FixedLengthStringUtils.java | 57 ++
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 64 ++
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 99 +++
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 54 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 82 ++-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 16 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 43 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 44 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 79 +++
.../hdds/utils/db/TestFixedLengthStringUtils.java | 45 ++
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 54 ++
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 226 ++++++
.../src/main/proto/DatanodeClientProtocol.proto | 2 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../hadoop/hdds/scm/server/SCMCertStore.java | 6 +-
.../main/resources/webapps/scm/scm-overview.html | 2 +-
.../dist/src/main/compose/compatibility/test.sh | 1 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 2 +-
.../smoketest/compatibility/dn-one-rocksdb.robot | 29 +
.../ozone/TestStorageContainerManagerHelper.java | 80 +--
.../client/rpc/TestFailureHandlingByClient.java | 14 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 21 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 18 +-
.../commandhandler/TestBlockDeletion.java | 25 +-
.../TestCloseContainerByPipeline.java | 6 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 14 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 3 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 2 +-
.../ozone/debug/container/ContainerCommands.java | 3 +-
.../containergenerator/GeneratorDatanode.java | 8 +-
.../ozone/debug/TestDBDefinitionFactory.java | 13 +-
121 files changed, 6330 insertions(+), 1387 deletions(-)
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index e817202846,9fc07840da..bcdedf5654
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@@ -18,12 -18,11 +18,12 @@@
package org.apache.hadoop.ozone.container.common.volume;
- import static org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion;
-
import java.io.File;
import java.io.IOException;
- import java.util.Properties;
+ import java.util.List;
+import java.util.UUID;
+ import java.util.concurrent.ThreadLocalRandom;
+ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@@ -65,20 -62,18 +69,21 @@@ public class HddsVolume extends Storage
public static final String HDDS_VOLUME_DIR = "hdds";
- private VolumeState state;
private final VolumeIOStats volumeIOStats;
+ private final VolumeInfoMetrics volumeInfoMetrics;
- // VERSION file properties
- private String storageID; // id of the file system
- private String clusterID; // id of the cluster
- private String datanodeUuid; // id of the DataNode
- private long cTime; // creation time of the file system state
- private int layoutVersion; // layout version of the storage data
private final AtomicLong committedBytes; // till Open containers become full
+ // Mentions the type of volume
+ private final VolumeType type = VolumeType.DATA_VOLUME;
+ // The dedicated DbVolume that the db instance of this HddsVolume resides.
+ // This is optional, if null then the db instance resides on this HddsVolume.
+ private DbVolume dbVolume;
+ // The subdirectory with storageID as its name, used to build the
+ // container db path. This is initialized only once together with dbVolume,
+ // and stored as a member to prevent spawning lots of File objects.
+ private File dbParentDir;
+ private AtomicBoolean dbLoaded = new AtomicBoolean(false);
/**
* Builder for HddsVolume.
@@@ -115,12 -98,7 +108,10 @@@
super(b);
if (!b.getFailedVolume()) {
- this.state = VolumeState.NOT_INITIALIZED;
- this.clusterID = b.clusterID;
- this.datanodeUuid = b.datanodeUuid;
++ this.setState(VolumeState.NOT_INITIALIZED);
this.volumeIOStats = new VolumeIOStats(b.getVolumeRootStr());
+ this.volumeInfoMetrics =
+ new VolumeInfoMetrics(b.getVolumeRootStr(), this);
this.committedBytes = new AtomicLong(0);
LOG.info("Creating HddsVolume: {} of storage type : {} capacity : {}",
@@@ -130,10 -108,7 +121,9 @@@
} else {
// Builder is called with failedVolume set, so create a failed volume
// HddsVolume Object.
++ this.setState(VolumeState.FAILED);
volumeIOStats = null;
+ volumeInfoMetrics = null;
- storageID = UUID.randomUUID().toString();
- state = VolumeState.FAILED;
committedBytes = null;
}
@@@ -288,61 -130,17 +145,28 @@@
return super.getStorageDir();
}
- @Override
- public String getStorageID() {
- return storageID;
- }
-
- public String getClusterID() {
- return clusterID;
- }
-
- public String getDatanodeUuid() {
- return datanodeUuid;
- }
-
- public long getCTime() {
- return cTime;
- }
-
- public int getLayoutVersion() {
- return layoutVersion;
- }
-
+ public VolumeType getType() {
+ return type;
+ }
+
- public VolumeState getStorageState() {
- return state;
- }
-
- public void setState(VolumeState state) {
- this.state = state;
- }
-
- public boolean isFailed() {
- return (state == VolumeState.FAILED);
- }
-
public VolumeIOStats getVolumeIOStats() {
return volumeIOStats;
}
+ public VolumeInfoMetrics getVolumeInfoStats() {
+ return volumeInfoMetrics;
+ }
+
@Override
public void failVolume() {
- setState(VolumeState.FAILED);
super.failVolume();
if (volumeIOStats != null) {
volumeIOStats.unregister();
}
+ if (volumeInfoMetrics != null) {
+ volumeInfoMetrics.unregister();
+ }
+ closeDbStore();
}
@Override
@@@ -352,30 -149,7 +175,10 @@@
if (volumeIOStats != null) {
volumeIOStats.unregister();
}
+ if (volumeInfoMetrics != null) {
+ volumeInfoMetrics.unregister();
+ }
- }
-
- /**
- * VolumeState represents the different states a HddsVolume can be in.
- * NORMAL => Volume can be used for storage
- * FAILED => Volume has failed due and can no longer be used for
- * storing containers.
- * NON_EXISTENT => Volume Root dir does not exist
- * INCONSISTENT => Volume Root dir is not empty but VERSION file is
- * missing or Volume Root dir is not a directory
- * NOT_FORMATTED => Volume Root exists but not formatted(no VERSION file)
- * NOT_INITIALIZED => VERSION file exists but has not been verified for
- * correctness.
- */
- public enum VolumeState {
- NORMAL,
- FAILED,
- NON_EXISTENT,
- INCONSISTENT,
- NOT_FORMATTED,
- NOT_INITIALIZED
+ closeDbStore();
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org