You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2023/04/05 20:43:22 UTC
[ozone] branch master updated: HDDS-8237. [Snapshot] loadDb() used by SstFiltering service creates extraneous directories. (#4446)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 05d9d85b8e HDDS-8237. [Snapshot] loadDb() used by SstFiltering service creates extraneous directories. (#4446)
05d9d85b8e is described below
commit 05d9d85b8ecf88c88a35e93bb315497940471330
Author: GeorgeJahad <gi...@blackbirdsystems.net>
AuthorDate: Wed Apr 5 13:43:15 2023 -0700
HDDS-8237. [Snapshot] loadDb() used by SstFiltering service creates extraneous directories. (#4446)
Co-authored-by: Hemant Kumar <he...@gmail.com>
Co-authored-by: George Jahad <ge...@georgejahad.com>
Co-authored-by: Siyao Meng <50...@users.noreply.github.com>
---
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 8 +++-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 43 ++++++++++------------
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 17 +++++++--
.../hadoop/ozone/om/SstFilteringService.java | 2 +-
4 files changed, 42 insertions(+), 28 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
index 3d5d785f86..5427f20936 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
@@ -107,6 +107,7 @@ public final class DBStoreBuilder {
private boolean enableCompactionLog;
private long maxTimeAllowedForSnapshotInDag;
private long pruneCompactionDagDaemonRunInterval;
+ private boolean createCheckpointDirs = true;
// this is to track the total size of dbUpdates data since sequence
// number in request to avoid increase in heap memory.
private long maxDbUpdatesSizeThreshold;
@@ -228,7 +229,8 @@ public final class DBStoreBuilder {
return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs,
registry, openReadOnly, maxFSSnapshots, dbJmxBeanNameName,
enableCompactionLog, maxTimeAllowedForSnapshotInDag,
- pruneCompactionDagDaemonRunInterval, maxDbUpdatesSizeThreshold);
+ pruneCompactionDagDaemonRunInterval, maxDbUpdatesSizeThreshold,
+ createCheckpointDirs);
} finally {
tableConfigs.forEach(TableConfig::close);
}
@@ -290,6 +292,10 @@ public final class DBStoreBuilder {
return this;
}
+ public DBStoreBuilder setCreateCheckpointDirs(boolean createCheckpointDirs) {
+ this.createCheckpointDirs = createCheckpointDirs;
+ return this;
+ }
/**
* Set the {@link ManagedDBOptions} and default
* {@link ManagedColumnFamilyOptions} based on {@code prof}.
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index aa95bd32e5..92e95fc3a4 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.hdds.utils.db;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
@@ -84,7 +86,7 @@ public class RDBStore implements DBStore {
this(dbFile, options, new ManagedWriteOptions(), families,
new CodecRegistry(), false, 1000, null, false,
TimeUnit.DAYS.toMillis(1), TimeUnit.HOURS.toMillis(1),
- maxDbUpdatesSizeThreshold);
+ maxDbUpdatesSizeThreshold, true);
}
@SuppressWarnings("parameternumber")
@@ -94,7 +96,9 @@ public class RDBStore implements DBStore {
String dbJmxBeanNameName, boolean enableCompactionLog,
long maxTimeAllowedForSnapshotInDag,
long compactionDagDaemonInterval,
- long maxDbUpdatesSizeThreshold)
+ long maxDbUpdatesSizeThreshold,
+ boolean createCheckpointDirs)
+
throws IOException {
Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
Preconditions.checkNotNull(families);
@@ -133,29 +137,22 @@ public class RDBStore implements DBStore {
}
//create checkpoints directory if not exists.
- checkpointsParentDir =
- dbLocation.getParent() + OM_KEY_PREFIX + OM_CHECKPOINT_DIR;
- File checkpointsDir = new File(checkpointsParentDir);
- if (!checkpointsDir.exists()) {
- boolean success = checkpointsDir.mkdir();
- if (!success) {
- throw new IOException(
- "Unable to create RocksDB checkpoint directory: " +
- checkpointsParentDir);
- }
+ if (!createCheckpointDirs) {
+ checkpointsParentDir = null;
+ } else {
+ Path checkpointsParentDirPath =
+ Paths.get(dbLocation.getParent(), OM_CHECKPOINT_DIR);
+ checkpointsParentDir = checkpointsParentDirPath.toString();
+ Files.createDirectories(checkpointsParentDirPath);
}
-
//create snapshot checkpoint directory if does not exist.
- snapshotsParentDir = Paths.get(dbLocation.getParent(),
- OM_SNAPSHOT_CHECKPOINT_DIR).toString();
- File snapshotsDir = new File(snapshotsParentDir);
- if (!snapshotsDir.exists()) {
- boolean success = snapshotsDir.mkdirs();
- if (!success) {
- throw new IOException(
- "Unable to create RocksDB snapshot checkpoint directory: " +
- snapshotsParentDir);
- }
+ if (!createCheckpointDirs) {
+ snapshotsParentDir = null;
+ } else {
+ Path snapshotsParentDirPath =
+ Paths.get(dbLocation.getParent(), OM_SNAPSHOT_CHECKPOINT_DIR);
+ snapshotsParentDir = snapshotsParentDirPath.toString();
+ Files.createDirectories(snapshotsParentDirPath);
}
if (enableCompactionLog) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 1816d4c872..5d4d28d94d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -382,7 +382,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
RDBCheckpointUtils.waitForCheckpointDirectoryExist(checkpoint);
}
setStore(loadDB(conf, metaDir, dbName, false,
- java.util.Optional.of(Boolean.TRUE)));
+ java.util.Optional.of(Boolean.TRUE), false));
initializeOmTables(false);
}
@@ -520,7 +520,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir)
throws IOException {
return loadDB(configuration, metaDir, OM_DB_NAME, false,
- java.util.Optional.empty());
+ java.util.Optional.empty(), true);
}
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
@@ -528,6 +528,16 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
java.util.Optional<Boolean>
disableAutoCompaction)
throws IOException {
+ return loadDB(configuration, metaDir, dbName, readOnly,
+ disableAutoCompaction, true);
+ }
+
+ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
+ String dbName, boolean readOnly,
+ java.util.Optional<Boolean>
+ disableAutoCompaction,
+ boolean createCheckpointDirs)
+ throws IOException {
final int maxFSSnapshots = configuration.getInt(
OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT);
RocksDBConfiguration rocksDBConfiguration =
@@ -537,7 +547,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
.setOpenReadOnly(readOnly)
.setPath(Paths.get(metaDir.getPath()))
.setMaxFSSnapshots(maxFSSnapshots)
- .setEnableCompactionLog(true);
+ .setEnableCompactionLog(true)
+ .setCreateCheckpointDirs(createCheckpointDirs);
disableAutoCompaction.ifPresent(
dbStoreBuilder::disableDefaultCFAutoCompaction);
return addOMTablesAndCodecs(dbStoreBuilder).build();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
index cdc473646b..5d3b988bc6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
@@ -150,7 +150,7 @@ public class SstFilteringService extends BackgroundService {
try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl
.loadDB(ozoneManager.getConfiguration(),
new File(snapshotCheckpointDir),
- dbName, true, Optional.of(Boolean.TRUE))) {
+ dbName, true, Optional.of(Boolean.TRUE), false)) {
RocksDatabase db = rdbStore.getDb();
db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org