You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2022/06/02 14:39:25 UTC
[ozone] branch master updated (c923a266cb -> fdf7376d3f)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
from c923a266cb HDDS-6819. Add LEGACY to AllowedBucketLayouts in CreateBucketHandler (#3473)
add 0bc3321950 HDDS-5920. Allow to add ColumnFamilyOptions to tables. (#2790)
add fcf74fcd96 HDDS-5921. [Merge rocksdb in datanode]Decouple DatanodeStore impl classes from container. (#2791)
add 83bb7034b4 HDDS-5923. Add DBHandle abstraction over ReferenceCountedDB. (#2794)
add db4734237b HDDS-6404. Format table key according to schema in KeyValueContainerData. (#3172)
add 482886d77b HDDS-6428. Add prefix iterator support to RDBTable. (#3176)
add d693014b02 HDDS-6486. [Merge rocksdb in datanode] Add new container schema v3 definitions. (#3253)
add 4e7a0c395b HDDS-6540. Add a Cache for per-disk RocksDB Instance. (#3283)
add 7c8d51dca5 HDDS-6541. [Merge rocksdb in datanode] Per-disk DB location management. (#3292)
add 07ad6080f4 Update non-rolling-upgrade test mlv to the newest 4.
add de18c59022 HDDS-6542. [Merge rocksdb in datanode] KeyValueContainer operation adaptation for schema v3 containers. (#3346)
add 562a242095 HDDS-6543. [Merge rocksdb in datanode] BlockDeletingService adaptation for schema v3 containers. (#3385)
add 3998c1da1f HDDS-6745: [Merge rocksdb in datanode] Flush RocksDB WAL when closing container in Schema V3. (#3419)
add 4a94b609fa HDDS-6544. [Merge rocksdb in datanode] New container replication operations for schema v3 container. (#3398)
add 3178ba5a94 HDDS-6597. [Merge rocksdb in datanode] Non-rolling upgrade supports container Schema V3. (#3392)
add 78cff9c031 HDDS-6754. [Merge rocksdb in datanode] Add compatibility test between schema v3 & v2. (#3422)
add 9b3de4101b HDDS-6747. Support configurable separator between container ID and container meta data key (#3420)
add 0e7b095c30 HDDS-6791: Add smoke test for Merge RocksDB in datanode feature. (#3448)
add 48525d793d HDDS-6792: [Merge RocksDB in Datanode] Fix issues reportd by sonar. (#3446)
add a6b5174fbb HDDS-6790: Add document for Merge RocksDB in datanode feature. (#3451)
add f94b7eb12d Merge remote-tracking branch 'origin/HDDS-3630'
add 55c3848624 Merge remote-tracking branch 'origin/HDDS-3630'
add fdf7376d3f Merge remote-tracking branch 'origin/master' into HDDS-3630
No new revisions were added by this update.
Summary of changes:
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 6 +-
.../upgrade/AbstractLayoutVersionManager.java | 12 +-
.../common/src/main/resources/ozone-default.xml | 14 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 6 +-
...nerLocationManagerMXBean.java => DBHandle.java} | 32 +-
.../common/statemachine/DatanodeConfiguration.java | 66 ++
.../common/statemachine/DatanodeStateMachine.java | 11 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 90 ++-
.../states/endpoint/VersionEndpointTask.java | 68 +-
.../container/common/utils/ContainerCache.java | 2 +-
.../container/common/utils/DatanodeStoreCache.java | 88 +++
.../container/common/utils/HddsVolumeUtil.java | 230 ++-----
.../hadoop/ozone/container/common/utils/RawDB.java | 32 +-
.../container/common/utils/ReferenceCountedDB.java | 25 +-
.../container/common/utils/StorageVolumeUtil.java | 202 ++++++
.../ozone/container/common/volume/DbVolume.java | 153 +++++
...dataVolumeFactory.java => DbVolumeFactory.java} | 35 +-
.../ozone/container/common/volume/HddsVolume.java | 408 +++++------
.../container/common/volume/HddsVolumeFactory.java | 37 +-
.../container/common/volume/MetadataVolume.java | 5 +
.../common/volume/MetadataVolumeFactory.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 11 +
.../container/common/volume/StorageVolume.java | 278 +++++++-
.../common/volume/StorageVolumeFactory.java | 41 +-
.../container/keyvalue/KeyValueContainer.java | 121 ++--
.../container/keyvalue/KeyValueContainerCheck.java | 24 +-
.../container/keyvalue/KeyValueContainerData.java | 109 ++-
.../KeyValueContainerMetadataInspector.java | 62 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 11 +-
.../container/keyvalue/TarContainerPacker.java | 15 +-
.../container/keyvalue/helpers/BlockUtils.java | 179 ++++-
.../helpers/KeyValueContainerLocationUtil.java | 12 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 194 +++---
.../container/keyvalue/impl/BlockManagerImpl.java | 57 +-
.../background/BlockDeletingService.java | 95 ++-
.../metadata/AbstractDatanodeDBDefinition.java | 11 +-
.../container/metadata/AbstractDatanodeStore.java | 20 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 158 +++++
.../metadata/DatanodeSchemaTwoDBDefinition.java | 6 +-
.../ozone/container/metadata/DatanodeStore.java | 9 +-
.../metadata/DatanodeStoreSchemaOneImpl.java | 7 +-
.../metadata/DatanodeStoreSchemaThreeImpl.java | 130 ++++
.../metadata/DatanodeStoreSchemaTwoImpl.java | 13 +-
.../ozone/container/metadata/DatanodeTable.java | 34 +-
.../metadata/DeleteTransactionStore.java} | 17 +-
.../metadata/SchemaOneDeletedBlocksTable.java | 8 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 33 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 82 +++
.../ScmHAFinalizeUpgradeActionDatanode.java | 2 +-
.../upgrade/VersionedDatanodeFeatures.java | 28 +-
.../ozone/container/common/ContainerTestUtils.java | 32 +
.../container/common/TestBlockDeletingService.java | 167 ++---
.../ozone/container/common/TestContainerCache.java | 2 +-
.../container/common/TestDatanodeStoreCache.java | 78 +++
.../common/TestKeyValueContainerData.java | 20 +-
.../TestSchemaOneBackwardsCompatibility.java | 150 ++--
.../TestSchemaTwoBackwardsCompatibility.java | 367 ++++++++++
.../common/helpers/TestDatanodeVersionFile.java | 18 +-
.../common/impl/TestContainerPersistence.java | 30 +-
.../statemachine/TestDatanodeConfiguration.java | 11 +
.../container/common/utils/TestHddsVolumeUtil.java | 238 +++++++
.../common/utils/TestStorageVolumeUtil.java | 99 +++
.../container/common/volume/TestDbVolume.java | 172 +++++
.../container/common/volume/TestHddsVolume.java | 164 ++++-
.../container/common/volume/TestStorageVolume.java | 83 +++
.../common/volume/TestVolumeSetDiskChecks.java | 30 +
.../keyvalue/ContainerTestVersionInfo.java | 79 +++
.../keyvalue/TestKeyValueBlockIterator.java | 82 ++-
.../container/keyvalue/TestKeyValueContainer.java | 47 +-
.../keyvalue/TestKeyValueContainerCheck.java | 19 +-
.../TestKeyValueContainerIntegrityChecks.java | 44 +-
.../TestKeyValueContainerMetadataInspector.java | 20 +-
.../container/keyvalue/TestTarContainerPacker.java | 82 ++-
.../keyvalue/impl/TestBlockManagerImpl.java | 23 +-
.../container/ozoneimpl/TestContainerReader.java | 85 ++-
.../container/ozoneimpl/TestOzoneContainer.java | 99 +--
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 759 +++++++++++++++++++++
.../upgrade/TestDatanodeUpgradeToScmHA.java | 5 +
.../design/{locks.md => dn-merge-rocksdb.md} | 14 +-
.../docs/content/feature/dn-merge-rocksdb.md | 70 ++
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 7 +
.../hadoop/hdds/utils/MetadataKeyFilters.java | 1 +
.../hdds/utils/db/DBColumnFamilyDefinition.java | 13 +
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 2 +-
...rNotFoundException.java => DumpFileLoader.java} | 25 +-
...rNotFoundException.java => DumpFileWriter.java} | 31 +-
...tringCodec.java => FixedLengthStringCodec.java} | 16 +-
.../hdds/utils/db/FixedLengthStringUtils.java | 57 ++
...DBBatchOperation.java => RDBSstFileLoader.java} | 60 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 98 +++
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 54 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 82 ++-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 16 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 43 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 44 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 79 +++
.../hdds/utils/db/TestFixedLengthStringUtils.java} | 40 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 54 ++
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 226 ++++++
.../src/main/proto/DatanodeClientProtocol.proto | 2 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../hadoop/hdds/scm/server/SCMCertStore.java | 6 +-
.../dist/src/main/compose/compatibility/test.sh | 1 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 2 +-
.../dn-one-rocksdb.robot} | 11 +-
.../ozone/TestStorageContainerManagerHelper.java | 80 +--
.../client/rpc/TestFailureHandlingByClient.java | 14 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 21 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 18 +-
.../commandhandler/TestBlockDeletion.java | 25 +-
.../TestCloseContainerByPipeline.java | 6 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 14 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 3 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 2 +-
.../ozone/debug/container/ContainerCommands.java | 3 +-
.../containergenerator/GeneratorDatanode.java | 8 +-
.../ozone/debug/TestDBDefinitionFactory.java | 13 +-
120 files changed, 6055 insertions(+), 1518 deletions(-)
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/{ContainerLocationManagerMXBean.java => DBHandle.java} (60%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DatanodeStoreCache.java
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/RawDB.java (55%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/{MetadataVolumeFactory.java => DbVolumeFactory.java} (64%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreSchemaThreeImpl.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/CheckedFunction.java => container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DeleteTransactionStore.java} (67%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV3FinalizeAction.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStoreCache.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestHddsVolumeUtil.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestStorageVolumeUtil.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestDbVolume.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolume.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java
copy hadoop-hdds/docs/content/design/{locks.md => dn-merge-rocksdb.md} (62%)
create mode 100644 hadoop-hdds/docs/content/feature/dn-merge-rocksdb.md
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/{SequenceNumberNotFoundException.java => DumpFileLoader.java} (73%)
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/{SequenceNumberNotFoundException.java => DumpFileWriter.java} (64%)
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/{StringCodec.java => FixedLengthStringCodec.java} (77%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/FixedLengthStringUtils.java
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/{RDBBatchOperation.java => RDBSstFileLoader.java} (52%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBSstFileWriter.java
copy hadoop-hdds/framework/src/{main/java/org/apache/hadoop/hdds/utils/db/ShortCodec.java => test/java/org/apache/hadoop/hdds/utils/db/TestFixedLengthStringUtils.java} (54%)
copy hadoop-ozone/dist/src/main/smoketest/{topology/readdata.robot => compatibility/dn-one-rocksdb.robot} (74%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org