You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2022/06/02 03:37:17 UTC
[ozone] 01/01: Merge remote-tracking branch 'origin/HDDS-3630'
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-3630
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4f58545294e3de92929d2fe814bcb98b98929733
Merge: 24ca66b067 f94b7eb12d
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Thu Jun 2 11:21:52 2022 +0800
Merge remote-tracking branch 'origin/HDDS-3630'
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 6 +-
.../upgrade/AbstractLayoutVersionManager.java | 12 +-
.../common/src/main/resources/ozone-default.xml | 14 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 6 +-
.../container/common/interfaces/DBHandle.java | 48 ++
.../common/statemachine/DatanodeConfiguration.java | 66 ++
.../common/statemachine/DatanodeStateMachine.java | 11 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 90 ++-
.../states/endpoint/VersionEndpointTask.java | 68 +-
.../container/common/utils/ContainerCache.java | 2 +-
.../container/common/utils/DatanodeStoreCache.java | 88 +++
.../container/common/utils/HddsVolumeUtil.java | 230 ++-----
.../hadoop/ozone/container/common/utils/RawDB.java | 42 ++
.../container/common/utils/ReferenceCountedDB.java | 25 +-
.../container/common/utils/StorageVolumeUtil.java | 202 ++++++
.../ozone/container/common/volume/DbVolume.java | 153 +++++
...dataVolumeFactory.java => DbVolumeFactory.java} | 35 +-
.../ozone/container/common/volume/HddsVolume.java | 407 +++++------
.../container/common/volume/HddsVolumeFactory.java | 37 +-
.../container/common/volume/MetadataVolume.java | 5 +
.../common/volume/MetadataVolumeFactory.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 11 +
.../container/common/volume/StorageVolume.java | 278 +++++++-
.../common/volume/StorageVolumeFactory.java | 41 +-
.../container/keyvalue/KeyValueContainer.java | 121 ++--
.../container/keyvalue/KeyValueContainerCheck.java | 24 +-
.../container/keyvalue/KeyValueContainerData.java | 109 ++-
.../KeyValueContainerMetadataInspector.java | 62 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 11 +-
.../container/keyvalue/TarContainerPacker.java | 15 +-
.../container/keyvalue/helpers/BlockUtils.java | 179 ++++-
.../helpers/KeyValueContainerLocationUtil.java | 12 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 194 +++---
.../container/keyvalue/impl/BlockManagerImpl.java | 57 +-
.../background/BlockDeletingService.java | 95 ++-
.../metadata/AbstractDatanodeDBDefinition.java | 11 +-
.../container/metadata/AbstractDatanodeStore.java | 20 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 158 +++++
.../metadata/DatanodeSchemaTwoDBDefinition.java | 6 +-
.../ozone/container/metadata/DatanodeStore.java | 9 +-
.../metadata/DatanodeStoreSchemaOneImpl.java | 7 +-
.../metadata/DatanodeStoreSchemaThreeImpl.java | 130 ++++
.../metadata/DatanodeStoreSchemaTwoImpl.java | 13 +-
.../ozone/container/metadata/DatanodeTable.java | 34 +-
.../container/metadata/DeleteTransactionStore.java | 29 +
.../metadata/SchemaOneDeletedBlocksTable.java | 8 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 33 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 82 +++
.../ScmHAFinalizeUpgradeActionDatanode.java | 2 +-
.../upgrade/VersionedDatanodeFeatures.java | 28 +-
.../ozone/container/common/ContainerTestUtils.java | 32 +
.../container/common/TestBlockDeletingService.java | 167 ++---
.../ozone/container/common/TestContainerCache.java | 2 +-
.../container/common/TestDatanodeStoreCache.java | 78 +++
.../common/TestKeyValueContainerData.java | 20 +-
.../TestSchemaOneBackwardsCompatibility.java | 150 ++--
.../TestSchemaTwoBackwardsCompatibility.java | 367 ++++++++++
.../common/helpers/TestDatanodeVersionFile.java | 18 +-
.../common/impl/TestContainerPersistence.java | 30 +-
.../statemachine/TestDatanodeConfiguration.java | 11 +
.../container/common/utils/TestHddsVolumeUtil.java | 238 +++++++
.../common/utils/TestStorageVolumeUtil.java | 99 +++
.../container/common/volume/TestDbVolume.java | 172 +++++
.../container/common/volume/TestHddsVolume.java | 164 ++++-
.../container/common/volume/TestStorageVolume.java | 83 +++
.../common/volume/TestVolumeSetDiskChecks.java | 30 +
.../keyvalue/ContainerTestVersionInfo.java | 79 +++
.../keyvalue/TestKeyValueBlockIterator.java | 82 ++-
.../container/keyvalue/TestKeyValueContainer.java | 47 +-
.../keyvalue/TestKeyValueContainerCheck.java | 19 +-
.../TestKeyValueContainerIntegrityChecks.java | 44 +-
.../TestKeyValueContainerMetadataInspector.java | 20 +-
.../container/keyvalue/TestTarContainerPacker.java | 82 ++-
.../keyvalue/impl/TestBlockManagerImpl.java | 23 +-
.../container/ozoneimpl/TestContainerReader.java | 85 ++-
.../container/ozoneimpl/TestOzoneContainer.java | 99 +--
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 759 +++++++++++++++++++++
.../upgrade/TestDatanodeUpgradeToScmHA.java | 5 +
.../docs/content/design/dn-merge-rocksdb.md | 29 +
.../docs/content/feature/dn-merge-rocksdb.md | 70 ++
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 7 +
.../hadoop/hdds/utils/MetadataKeyFilters.java | 1 +
.../hdds/utils/db/DBColumnFamilyDefinition.java | 13 +
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 2 +-
.../hadoop/hdds/utils/db/DumpFileLoader.java | 38 ++
.../hadoop/hdds/utils/db/DumpFileWriter.java | 46 ++
.../hdds/utils/db/FixedLengthStringCodec.java | 50 ++
.../hdds/utils/db/FixedLengthStringUtils.java | 57 ++
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 64 ++
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 99 +++
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 54 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 82 ++-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 16 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 43 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 44 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 79 +++
.../hdds/utils/db/TestFixedLengthStringUtils.java | 45 ++
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 54 ++
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 226 ++++++
.../src/main/proto/DatanodeClientProtocol.proto | 2 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../hadoop/hdds/scm/server/SCMCertStore.java | 6 +-
.../main/resources/webapps/scm/scm-overview.html | 2 +-
.../dist/src/main/compose/compatibility/test.sh | 1 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 2 +-
.../smoketest/compatibility/dn-one-rocksdb.robot | 29 +
.../ozone/TestStorageContainerManagerHelper.java | 80 +--
.../client/rpc/TestFailureHandlingByClient.java | 14 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 21 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 18 +-
.../commandhandler/TestBlockDeletion.java | 25 +-
.../TestCloseContainerByPipeline.java | 6 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 14 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 3 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 2 +-
.../ozone/debug/container/ContainerCommands.java | 3 +-
.../containergenerator/GeneratorDatanode.java | 8 +-
.../ozone/debug/TestDBDefinitionFactory.java | 13 +-
121 files changed, 6330 insertions(+), 1387 deletions(-)
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index e817202846,9fc07840da..bcdedf5654
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@@ -18,12 -18,11 +18,12 @@@
package org.apache.hadoop.ozone.container.common.volume;
- import static org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion;
-
import java.io.File;
import java.io.IOException;
- import java.util.Properties;
+ import java.util.List;
+import java.util.UUID;
+ import java.util.concurrent.ThreadLocalRandom;
+ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@@ -65,20 -62,18 +69,21 @@@ public class HddsVolume extends Storage
public static final String HDDS_VOLUME_DIR = "hdds";
- private VolumeState state;
private final VolumeIOStats volumeIOStats;
+ private final VolumeInfoMetrics volumeInfoMetrics;
- // VERSION file properties
- private String storageID; // id of the file system
- private String clusterID; // id of the cluster
- private String datanodeUuid; // id of the DataNode
- private long cTime; // creation time of the file system state
- private int layoutVersion; // layout version of the storage data
private final AtomicLong committedBytes; // till Open containers become full
+ // Mentions the type of volume
+ private final VolumeType type = VolumeType.DATA_VOLUME;
+ // The dedicated DbVolume that the db instance of this HddsVolume resides.
+ // This is optional, if null then the db instance resides on this HddsVolume.
+ private DbVolume dbVolume;
+ // The subdirectory with storageID as its name, used to build the
+ // container db path. This is initialized only once together with dbVolume,
+ // and stored as a member to prevent spawning lots of File objects.
+ private File dbParentDir;
+ private AtomicBoolean dbLoaded = new AtomicBoolean(false);
/**
* Builder for HddsVolume.
@@@ -115,12 -98,7 +108,10 @@@
super(b);
if (!b.getFailedVolume()) {
- this.state = VolumeState.NOT_INITIALIZED;
- this.clusterID = b.clusterID;
- this.datanodeUuid = b.datanodeUuid;
++ this.setState(VolumeState.NOT_INITIALIZED);
this.volumeIOStats = new VolumeIOStats(b.getVolumeRootStr());
+ this.volumeInfoMetrics =
+ new VolumeInfoMetrics(b.getVolumeRootStr(), this);
this.committedBytes = new AtomicLong(0);
LOG.info("Creating HddsVolume: {} of storage type : {} capacity : {}",
@@@ -130,10 -108,7 +121,9 @@@
} else {
// Builder is called with failedVolume set, so create a failed volume
// HddsVolume Object.
++ this.setState(VolumeState.FAILED);
volumeIOStats = null;
+ volumeInfoMetrics = null;
- storageID = UUID.randomUUID().toString();
- state = VolumeState.FAILED;
committedBytes = null;
}
@@@ -288,61 -130,17 +145,28 @@@
return super.getStorageDir();
}
- @Override
- public String getStorageID() {
- return storageID;
- }
-
- public String getClusterID() {
- return clusterID;
- }
-
- public String getDatanodeUuid() {
- return datanodeUuid;
- }
-
- public long getCTime() {
- return cTime;
- }
-
- public int getLayoutVersion() {
- return layoutVersion;
- }
-
+ public VolumeType getType() {
+ return type;
+ }
+
- public VolumeState getStorageState() {
- return state;
- }
-
- public void setState(VolumeState state) {
- this.state = state;
- }
-
- public boolean isFailed() {
- return (state == VolumeState.FAILED);
- }
-
public VolumeIOStats getVolumeIOStats() {
return volumeIOStats;
}
+ public VolumeInfoMetrics getVolumeInfoStats() {
+ return volumeInfoMetrics;
+ }
+
@Override
public void failVolume() {
- setState(VolumeState.FAILED);
super.failVolume();
if (volumeIOStats != null) {
volumeIOStats.unregister();
}
+ if (volumeInfoMetrics != null) {
+ volumeInfoMetrics.unregister();
+ }
+ closeDbStore();
}
@Override
@@@ -352,30 -149,7 +175,10 @@@
if (volumeIOStats != null) {
volumeIOStats.unregister();
}
+ if (volumeInfoMetrics != null) {
+ volumeInfoMetrics.unregister();
+ }
- }
-
- /**
- * VolumeState represents the different states a HddsVolume can be in.
- * NORMAL => Volume can be used for storage
- * FAILED => Volume has failed due and can no longer be used for
- * storing containers.
- * NON_EXISTENT => Volume Root dir does not exist
- * INCONSISTENT => Volume Root dir is not empty but VERSION file is
- * missing or Volume Root dir is not a directory
- * NOT_FORMATTED => Volume Root exists but not formatted(no VERSION file)
- * NOT_INITIALIZED => VERSION file exists but has not been verified for
- * correctness.
- */
- public enum VolumeState {
- NORMAL,
- FAILED,
- NON_EXISTENT,
- INCONSISTENT,
- NOT_FORMATTED,
- NOT_INITIALIZED
+ closeDbStore();
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org