You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "smengcl (via GitHub)" <gi...@apache.org> on 2025/10/07 21:24:44 UTC
[PR] HDDS-13009. Background snapshot defrag service [ozone]
smengcl opened a new pull request, #9117:
URL: https://github.com/apache/ozone/pull/9117
## What changes were proposed in this pull request?
Implement Background Snapshot Defragmentation Service outlined in the [design](https://github.com/apache/ozone/pull/8514)
Some commits are cherry-picked from the POC and rebased/changed.
- WIP
## What is the link to the Apache JIRA
https://issues.apache.org/jira/browse/HDDS-13009
## How was this patch tested?
- [ ] To cherry-pick test cases from POC
- [ ] Add additional test cases as we see fit
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2480254529
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2480264270
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2479975912
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
Review Comment:
Note: Step 2-5 is similar to `SSTFilteringService` but improved because of Step 5 compaction.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "jojochuang (via GitHub)" <gi...@apache.org>.
jojochuang commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2433864650
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
Review Comment:
can leverage #9141 to load from in-memory OmSnapshotLocalDataManager
##########
hadoop-hdds/common/src/main/resources/ozone-default.xml:
##########
@@ -3758,13 +3758,28 @@
Snapshot Deleting Service per run.
</description>
</property>
+ <property>
+ <name>ozone.snapshot.defrag.limit.per.task</name>
+ <value>1</value>
Review Comment:
we need to benchmark to find a more appropriate value.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ // Get the SnapshotChainManager to iterate through the global snapshot chain
+ // Set this each time the task runs just in case OmMetadataManager is restarted
+ snapshotChainManager = ((OmMetadataManagerImpl) ozoneManager.getMetadataManager()).getSnapshotChainManager();
+
+ final Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, snapshotToDefrag.getSnapshotId().toString());
+ LOG.debug("Acquired SNAPSHOT_GC_LOCK for snapshot: {}, ID: {}",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ if (!gcLockDetails.isLockAcquired()) {
+ LOG.warn("Failed to acquire SNAPSHOT_GC_LOCK for snapshot: {}",
+ snapshotToDefrag.getName());
+ break;
+ }
+
+ try {
+ LOG.info("Processing snapshot defragmentation for: {} (ID: {})",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ // Get snapshot through SnapshotCache for proper locking
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshotSupplier =
+ snapshotManager.get().getActiveSnapshot(
+ snapshotToDefrag.getVolumeName(),
+ snapshotToDefrag.getBucketName(),
+ snapshotToDefrag.getName())) {
+
+ OmSnapshot omSnapshot = snapshotSupplier.get();
+
+ UUID pathPreviousSnapshotId = snapshotToDefrag.getPathPreviousSnapshotId();
+ boolean isFirstSnapshotInPath = pathPreviousSnapshotId == null;
+ if (isFirstSnapshotInPath) {
+ LOG.info("Performing full defragmentation for first snapshot (in path): {}",
+ snapshotToDefrag.getName());
+ performFullDefragmentation(snapshotToDefrag, omSnapshot);
+ } else {
+ final String psIdtableKey = snapshotChainManager.getTableKey(pathPreviousSnapshotId);
+ SnapshotInfo previousDefraggedSnapshot = snapshotInfoTable.get(psIdtableKey);
+
+ LOG.info("Performing incremental defragmentation for snapshot: {} " +
+ "based on previous defragmented snapshot: {}",
+ snapshotToDefrag.getName(), previousDefraggedSnapshot.getName());
+
+ // If previous path snapshot is not null, it must have been defragmented already
+ // Sanity check to ensure previous snapshot exists and is defragmented
+ if (needsDefragmentation(previousDefraggedSnapshot)) {
+ LOG.error("Fatal error before defragging snapshot: {}. " +
+ "Previous snapshot in path {} was not defragged while it is expected to be.",
+ snapshotToDefrag.getName(), previousDefraggedSnapshot.getName());
+ break;
+ }
+
+ performIncrementalDefragmentation(snapshotToDefrag,
+ previousDefraggedSnapshot, omSnapshot);
+ }
+
+ // Update snapshot metadata
+ updateSnapshotMetadata(snapshotToDefrag);
+
+ // Close and evict the original snapshot DB from SnapshotCache
+ // TODO: Implement proper eviction from SnapshotCache
+ LOG.info("Defragmentation completed for snapshot: {}",
+ snapshotToDefrag.getName());
+
+ snapshotLimit--;
+ snapshotsDefraggedCount.getAndIncrement();
+
+ } catch (OMException ome) {
+ if (ome.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
+ LOG.info("Snapshot {} was deleted during defragmentation",
+ snapshotToDefrag.getName());
+ } else {
+ LOG.error("OMException during snapshot defragmentation for: {}",
+ snapshotToDefrag.getName(), ome);
+ }
Review Comment:
return value is not used. But in this case (OMException), it would return true.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2418018496
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
Review Comment:
I've modified `findFirstSnapshotNeedingDefrag` to use snapshot chain now. ptal
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, "defrag-" + snapshotToDefrag.getSnapshotId());
+
+ if (!gcLockDetails.isLockAcquired()) {
+ LOG.warn("Failed to acquire SNAPSHOT_GC_LOCK for snapshot: {}",
+ snapshotToDefrag.getName());
+ break;
+ }
+
+ try {
+ LOG.info("Processing snapshot defragmentation for: {} (ID: {})",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ // Get snapshot through SnapshotCache for proper locking
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshotSupplier =
+ snapshotManager.get().getActiveSnapshot(
+ snapshotToDefrag.getVolumeName(),
+ snapshotToDefrag.getBucketName(),
+ snapshotToDefrag.getName())) {
+
+ OmSnapshot omSnapshot = snapshotSupplier.get();
+
+ // Check if this is the first snapshot in the chain
+ SnapshotInfo previousDefraggedSnapshot = findPreviousDefraggedSnapshot(
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl closed pull request #9117: HDDS-13009. Background snapshot defrag service
URL: https://github.com/apache/ozone/pull/9117
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2418011052
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java:
##########
@@ -968,7 +1027,16 @@ public boolean isSstFilteringSvcEnabled() {
// any interval <= 0 causes IllegalArgumentException from scheduleWithFixedDelay
return serviceInterval > 0;
}
-
+
+ public boolean isDefragSvcEnabled() {
+ long serviceInterval = ozoneManager.getConfiguration()
+ .getTimeDuration(OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL,
+ OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ // any interval <= 0 causes IllegalArgumentException from scheduleWithFixedDelay
Review Comment:
yes that's why the caller handles this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2480263665
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ // Get the SnapshotChainManager to iterate through the global snapshot chain
+ // Set this each time the task runs just in case OmMetadataManager is restarted
+ snapshotChainManager = ((OmMetadataManagerImpl) ozoneManager.getMetadataManager()).getSnapshotChainManager();
+
+ final Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, snapshotToDefrag.getSnapshotId().toString());
+ LOG.debug("Acquired SNAPSHOT_GC_LOCK for snapshot: {}, ID: {}",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ if (!gcLockDetails.isLockAcquired()) {
+ LOG.warn("Failed to acquire SNAPSHOT_GC_LOCK for snapshot: {}",
+ snapshotToDefrag.getName());
+ break;
+ }
+
+ try {
+ LOG.info("Processing snapshot defragmentation for: {} (ID: {})",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ // Get snapshot through SnapshotCache for proper locking
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshotSupplier =
+ snapshotManager.get().getActiveSnapshot(
+ snapshotToDefrag.getVolumeName(),
+ snapshotToDefrag.getBucketName(),
+ snapshotToDefrag.getName())) {
+
+ OmSnapshot omSnapshot = snapshotSupplier.get();
+
+ UUID pathPreviousSnapshotId = snapshotToDefrag.getPathPreviousSnapshotId();
+ boolean isFirstSnapshotInPath = pathPreviousSnapshotId == null;
+ if (isFirstSnapshotInPath) {
+ LOG.info("Performing full defragmentation for first snapshot (in path): {}",
+ snapshotToDefrag.getName());
+ performFullDefragmentation(snapshotToDefrag, omSnapshot);
+ } else {
+ final String psIdtableKey = snapshotChainManager.getTableKey(pathPreviousSnapshotId);
+ SnapshotInfo previousDefraggedSnapshot = snapshotInfoTable.get(psIdtableKey);
+
+ LOG.info("Performing incremental defragmentation for snapshot: {} " +
+ "based on previous defragmented snapshot: {}",
+ snapshotToDefrag.getName(), previousDefraggedSnapshot.getName());
+
+ // If previous path snapshot is not null, it must have been defragmented already
+ // Sanity check to ensure previous snapshot exists and is defragmented
+ if (needsDefragmentation(previousDefraggedSnapshot)) {
+ LOG.error("Fatal error before defragging snapshot: {}. " +
+ "Previous snapshot in path {} was not defragged while it is expected to be.",
+ snapshotToDefrag.getName(), previousDefraggedSnapshot.getName());
+ break;
+ }
+
+ performIncrementalDefragmentation(snapshotToDefrag,
+ previousDefraggedSnapshot, omSnapshot);
+ }
+
+ // Update snapshot metadata
+ updateSnapshotMetadata(snapshotToDefrag);
+
+ // Close and evict the original snapshot DB from SnapshotCache
+ // TODO: Implement proper eviction from SnapshotCache
+ LOG.info("Defragmentation completed for snapshot: {}",
+ snapshotToDefrag.getName());
+
+ snapshotLimit--;
+ snapshotsDefraggedCount.getAndIncrement();
+
+ } catch (OMException ome) {
+ if (ome.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
+ LOG.info("Snapshot {} was deleted during defragmentation",
+ snapshotToDefrag.getName());
+ } else {
+ LOG.error("OMException during snapshot defragmentation for: {}",
+ snapshotToDefrag.getName(), ome);
+ }
Review Comment:
good catch. fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "Copilot (via GitHub)" <gi...@apache.org>.
Copilot commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2433325670
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
Review Comment:
[nitpick] The TODO comment questions whether this constant should be moved to OzoneConsts. Since this is a directory name used internally by the defragmentation process and is similar to `OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR` which was added to OzoneConsts, consider moving this constant to OzoneConsts for consistency.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
Review Comment:
This TODO indicates that the snapshot checkpoint DB should be retrieved via SnapshotCache instead of directly accessing it. This is important for proper resource management and locking. Consider implementing this before production use.
```suggestion
// Get snapshot checkpoint DB via SnapshotCache for proper resource management
RDBStore originalStore = null;
RocksDatabase originalDb = null;
try {
// Obtain SnapshotCache from ozoneManager or omSnapshotManager
// This assumes ozoneManager has a getSnapshotCache() method; adjust as needed
SnapshotCache snapshotCache = ozoneManager.getSnapshotCache();
originalStore = snapshotCache.getSnapshotDB(snapshotInfo);
originalDb = originalStore.getDb();
} catch (Exception e) {
LOG.error("Failed to get snapshot DB from SnapshotCache for snapshot: {}", snapshotInfo.getName(), e);
throw new IOException("Failed to get snapshot DB from SnapshotCache for snapshot: " + snapshotInfo.getName(), e);
}
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
Review Comment:
[nitpick] This commented-out compaction code should either be removed or properly implemented with a TODO comment explaining why it's disabled. Leaving large blocks of commented code reduces code maintainability.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
Review Comment:
The TODO indicates that pagination is not handled when the diff list exceeds the server page size. The log message shows this can occur in practice (1000 entries). This could lead to incomplete defragmentation and should be implemented.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
Review Comment:
This TODO suggests optimizing the defragmentation by only processing column families relevant to the bucket layout (e.g., keyTable for OBJECT_STORE, fileTable for FSO). This optimization would reduce unnecessary work.
```suggestion
// Use bucket layout to determine which column families to process
List<String> relevantColumnFamilies = getRelevantColumnFamiliesForLayout(currentSnapshot.getBucketLayout());
for (String cfName : relevantColumnFamilies) {
```
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDefragService.java:
##########
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DEFRAG_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.ozone.test.LambdaTestUtils.await;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.DBProfile;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.SnapshotDefragService;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+/**
+ * Test SnapshotDefragService functionality using MiniOzoneCluster.
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestSnapshotDefragService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotDefragService.class);
+
+ private MiniOzoneCluster cluster;
+ private OzoneClient client;
+ private ObjectStore store;
+ private OzoneManager ozoneManager;
+ private SnapshotDefragService defragService;
+
+ @BeforeAll
+ void setup() throws Exception {
+ // Enable debug logging for SnapshotDefragService
+ GenericTestUtils.setLogLevel(LoggerFactory.getLogger(SnapshotDefragService.class), Level.DEBUG);
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 30, TimeUnit.SECONDS);
+ conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST);
+ conf.setBoolean(OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
+ conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.OBJECT_STORE.name());
+ // TODO: Add SNAPSHOT_DEFRAGMENTATION layout feature version
+ conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY,
+ OMLayoutFeature.DELEGATION_TOKEN_SYMMETRIC_SIGN.layoutVersion());
Review Comment:
This TODO indicates that a layout feature version for snapshot defragmentation needs to be added. This is important for proper upgrade handling and ensuring the feature is only enabled when the cluster is ready.
```suggestion
// Set layout version to SNAPSHOT_DEFRAGMENTATION to enable the feature
conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY,
OMLayoutFeature.SNAPSHOT_DEFRAGMENTATION.layoutVersion());
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
Review Comment:
This TODO correctly identifies a performance optimization: triggering snapshot diff before acquiring locks would prevent holding locks during the potentially long-running diff computation. Consider implementing this to reduce lock contention.
```suggestion
// First, trigger and wait for snapshot diff computation before acquiring any locks
SnapshotDiffResponse diffResponse;
LOG.info("Computing snapshot diff between {} and {}",
previousSnapshot.getName(), currentSnapshot.getName());
try {
// Call snapshotDiff to get the diff list
diffResponse = ozoneManager.snapshotDiff(
currentSnapshot.getVolumeName(),
currentSnapshot.getBucketName(),
previousSnapshot.getName(),
currentSnapshot.getName(),
null, // token - start from beginning
0, // pageSize - get all diffs at once
false, // forceFullDiff
false // disableNativeDiff
);
// Wait for snapshotDiff computation to complete if it's still in progress
while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
LOG.info("Snapshot diff computation in progress, waiting {} ms...",
diffResponse.getWaitTimeInMs());
Thread.sleep(diffResponse.getWaitTimeInMs());
// Poll for updated status
diffResponse = ozoneManager.snapshotDiff(
currentSnapshot.getVolumeName(),
currentSnapshot.getBucketName(),
previousSnapshot.getName(),
currentSnapshot.getName(),
null, // token
0, // pageSize
false, // forceFullDiff
false // disableNativeDiff
);
}
if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
}
LOG.info("Snapshot diff computation completed successfully");
} catch (Exception e) {
// Handle exception as before
throw new RocksDatabaseException("Snapshot diff computation failed", e);
}
// Now, proceed with the rest of the method, including lock acquisition and processing
try {
Files.createDirectories(Paths.get(tempSstDir));
LOG.info("Created temporary SST directory: {}", tempSstDir);
```
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDefragService.java:
##########
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DEFRAG_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.ozone.test.LambdaTestUtils.await;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.DBProfile;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.SnapshotDefragService;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+/**
+ * Test SnapshotDefragService functionality using MiniOzoneCluster.
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestSnapshotDefragService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotDefragService.class);
+
+ private MiniOzoneCluster cluster;
+ private OzoneClient client;
+ private ObjectStore store;
+ private OzoneManager ozoneManager;
+ private SnapshotDefragService defragService;
+
+ @BeforeAll
+ void setup() throws Exception {
+ // Enable debug logging for SnapshotDefragService
+ GenericTestUtils.setLogLevel(LoggerFactory.getLogger(SnapshotDefragService.class), Level.DEBUG);
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 30, TimeUnit.SECONDS);
+ conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST);
+ conf.setBoolean(OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
+ conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.OBJECT_STORE.name());
+ // TODO: Add SNAPSHOT_DEFRAGMENTATION layout feature version
+ conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY,
+ OMLayoutFeature.DELEGATION_TOKEN_SYMMETRIC_SIGN.layoutVersion());
+
+ conf.setInt(OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL, -1);
+ conf.setInt(OZONE_SNAPSHOT_DEFRAG_SERVICE_TIMEOUT, 3000000);
+ conf.setInt(SNAPSHOT_DEFRAG_LIMIT_PER_TASK, 5);
+
+ conf.setQuietMode(false);
+
+ // Create MiniOzoneCluster
+ cluster = MiniOzoneCluster.newBuilder(conf).build();
+ cluster.waitForClusterToBeReady();
+
+ client = cluster.newClient();
+ store = client.getObjectStore();
+ ozoneManager = cluster.getOzoneManager();
+
+ // Create SnapshotDefragService for manual triggering
+ defragService = new SnapshotDefragService(
+ 10000, // interval
+ TimeUnit.MILLISECONDS,
+ 30000, // service timeout
+ ozoneManager,
+ conf
+ );
+ }
+
+ @AfterAll
+ public void cleanup() throws Exception {
+ if (defragService != null) {
+ defragService.shutdown();
+ }
+ if (client != null) {
+ client.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Create keys in a bucket.
+ */
+ private void createKeys(String volumeName, String bucketName, int keyCount) throws Exception {
+ OzoneVolume volume = store.getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ for (int i = 0; i < keyCount; i++) {
+ String keyName = "key-" + i;
+ String data = RandomStringUtils.randomAlphabetic(100);
+
+ try (OzoneOutputStream outputStream = bucket.createKey(keyName, data.length(),
+ StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ java.util.Collections.emptyMap(), java.util.Collections.emptyMap())) {
+ outputStream.write(data.getBytes());
+ }
+ }
+ LOG.info("Created {} keys in bucket {}/{}", keyCount, volumeName, bucketName);
+ }
+
+ /**
+ * Create a snapshot and wait for it to be available.
+ */
+ private void createSnapshot(String volumeName, String bucketName, String snapshotName)
+ throws Exception {
+ // Get existing checkpoint directories before creating snapshot
+ Set<String> existingCheckpoints = getExistingCheckpointDirectories();
+
+ store.createSnapshot(volumeName, bucketName, snapshotName);
+
+ // Wait for snapshot to be created
+ GenericTestUtils.waitFor(() -> {
+ try {
+ SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager()
+ .getSnapshotInfoTable()
+ .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName));
+ return snapshotInfo != null;
+ } catch (IOException e) {
+ return false;
+ }
+ }, 1000, 30000);
+
+ // Wait for checkpoint DB to be created
+ waitForCheckpointDB(snapshotName, existingCheckpoints);
+
+ LOG.info("Created snapshot: {}", snapshotName);
+ }
+
+ /**
+ * Get existing checkpoint directories before snapshot creation.
+ */
+ private Set<String> getExistingCheckpointDirectories() {
+ String metadataDir = ozoneManager.getConfiguration().get("ozone.om.db.dirs");
+ if (metadataDir == null) {
+ metadataDir = ozoneManager.getConfiguration().get("ozone.metadata.dirs");
+ }
+
+ String checkpointStateDir = metadataDir + "/db.snapshots/checkpointState";
+ File checkpointParentDir = new File(checkpointStateDir);
+
+ Set<String> existingDirs = new java.util.HashSet<>();
+ if (checkpointParentDir.exists()) {
+ File[] checkpointDirs = checkpointParentDir.listFiles(File::isDirectory);
+ if (checkpointDirs != null) {
+ for (File dir : checkpointDirs) {
+ existingDirs.add(dir.getName());
+ }
+ }
+ }
+
+ LOG.debug("Existing checkpoint directories: {}", existingDirs);
+ return existingDirs;
+ }
+
+ /**
+ * Wait for checkpoint DB to be created under checkpointState directory.
+ */
+ private void waitForCheckpointDB(String snapshotName, Set<String> existingCheckpoints) throws Exception {
+ String metadataDir = ozoneManager.getConfiguration().get("ozone.om.db.dirs");
+ if (metadataDir == null) {
+ metadataDir = ozoneManager.getConfiguration().get("ozone.metadata.dirs");
+ }
+
+ String checkpointStateDir = metadataDir + "/db.snapshots/checkpointState";
+ File checkpointParentDir = new File(checkpointStateDir);
+
+ LOG.info("Waiting for new checkpoint DB to be created under: {}", checkpointStateDir);
+
+ GenericTestUtils.waitFor(() -> {
+ if (!checkpointParentDir.exists()) {
+ LOG.debug("CheckpointState directory does not exist yet: {}", checkpointStateDir);
+ return false;
+ }
+
+ // List all directories in checkpointState
+ File[] checkpointDirs = checkpointParentDir.listFiles(File::isDirectory);
+ if (checkpointDirs == null || checkpointDirs.length == 0) {
+ LOG.debug("No checkpoint directories found in: {}", checkpointStateDir);
+ return false;
+ }
+
+ // Look for new checkpoint directories that weren't there before
+ for (File checkpointDir : checkpointDirs) {
+ String dirName = checkpointDir.getName();
+
+ // Skip if this directory existed before snapshot creation
+ if (existingCheckpoints.contains(dirName)) {
+ continue;
+ }
+
+ // Check if the new directory contains database files
+ File[] dbFiles = checkpointDir.listFiles();
+ if (dbFiles != null && dbFiles.length > 0) {
+ for (File dbFile : dbFiles) {
+ if (dbFile.isFile() && (dbFile.getName().endsWith(".sst") ||
+ dbFile.getName().equals("CURRENT") ||
+ dbFile.getName().startsWith("MANIFEST"))) {
+ LOG.info("New checkpoint DB found for snapshot {} in directory: {}",
+ snapshotName, checkpointDir.getAbsolutePath());
+ return true;
+ }
+ }
+ }
+
+ LOG.debug("New checkpoint directory found but no DB files yet: {}", checkpointDir.getAbsolutePath());
+ }
+
+ LOG.debug("Waiting for new checkpoint DB files to appear in checkpointState directories");
+ return false;
+ }, 1000, 60000); // Wait up to 60 seconds for checkpoint DB creation
+
+ LOG.info("Checkpoint DB created successfully for snapshot: {}", snapshotName);
+ }
+
+ /**
+ * Lists the contents of the db.snapshots directory.
+ */
+ private void printSnapshotDirectoryListing(String description) {
+ LOG.info("=== {} ===", description);
+ String metadataDir = ozoneManager.getConfiguration().get("ozone.om.db.dirs");
+ if (metadataDir == null) {
+ metadataDir = ozoneManager.getConfiguration().get("ozone.metadata.dirs");
+ }
+
+ String snapshotDir = metadataDir + "/db.snapshots";
+ File snapshotsDir = new File(snapshotDir);
+
+ if (!snapshotsDir.exists()) {
+ LOG.info("Snapshots directory does not exist: {}", snapshotDir);
+ return;
+ }
+
+ try (Stream<Path> paths = Files.walk(Paths.get(snapshotDir))) {
+ paths.sorted()
+ .forEach(path -> {
+ File file = path.toFile();
+ String relativePath = Paths.get(snapshotDir).relativize(path).toString();
+ if (file.isDirectory()) {
+ LOG.info("Directory: {}/", relativePath.isEmpty() ? "." : relativePath);
+ } else {
+ LOG.info("File: {} (size: {} bytes)", relativePath, file.length());
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("Error listing snapshot directory: {}", snapshotDir, e);
+ }
+ }
+
+ /**
+ * Mark a snapshot as needing defragmentation by updating its YAML metadata.
+ * TODO: This is a workaround as the current logic does not
+ * automatically mark snapshots as needing defragmentation.
+ * This is not needed.
+ */
+ private void markSnapshotAsNeedingDefragmentation(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+ File yamlFile = new File(snapshotPath + ".yaml");
+
+ if (yamlFile.exists()) {
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+ yamlData.setNeedsDefrag(true);
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile);
+ LOG.info("Marked snapshot {} as needing defragmentation", snapshotInfo.getName());
+ } else {
+ LOG.warn("YAML file not found for snapshot {}: {}", snapshotInfo.getName(), yamlFile.getPath());
+ }
+ }
+
+ /**
+ * Trigger the SnapshotDefragService by starting it and waiting for it to process snapshots.
+ */
+ private void triggerSnapshotDefragService() throws Exception {
+ LOG.info("Triggering SnapshotDefragService...");
+
+ // Mark all snapshots as needing defragmentation first
+ OMMetadataManager metadataManager = ozoneManager.getMetadataManager();
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ metadataManager.getSnapshotInfoTable().iterator()) {
+ iterator.seekToFirst();
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> entry = iterator.next();
+ SnapshotInfo snapshotInfo = entry.getValue();
+ markSnapshotAsNeedingDefragmentation(snapshotInfo);
+ }
+ }
+
+ long initialDefragCount = defragService.getSnapshotsDefraggedCount().get();
+ LOG.info("Initial defragmented count: {}", initialDefragCount);
+
+ // Start the service
+ defragService.start();
+
+ // Wait for the service to process snapshots
+ try {
+ await(30000, 1000, () -> {
+ long currentCount = defragService.getSnapshotsDefraggedCount().get();
+ LOG.info("Current defragmented count: {}", currentCount);
+ return currentCount > initialDefragCount;
+ });
+ } catch (TimeoutException e) {
+ LOG.warn("Timeout waiting for defragmentation to complete, continuing with test");
+ }
+
+ LOG.info("SnapshotDefragService execution completed. Snapshots defragmented: {}",
+ defragService.getSnapshotsDefraggedCount().get());
+ }
+
+ @Test
+ public void testSnapshotDefragmentation() throws Exception {
+ String volumeName = "test-volume";
+ String bucketName = "test-bucket";
+
+ // Create volume and bucket
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ // TODO: Test FSO bucket as well, default is LEGACY / OBJECT_STORE
+ volume.createBucket(bucketName);
+
+ LOG.info("Starting snapshot defragmentation test...");
+
+ // Print initial state
+ printSnapshotDirectoryListing("Initial state - no snapshots");
+
+ // Step 1: Create 2 keys, then create snap-1
+ createKeys(volumeName, bucketName, 2);
+ createSnapshot(volumeName, bucketName, "snap-1");
+ printSnapshotDirectoryListing("After creating snap-1 (2 keys)");
+
+ // Step 2: Create 2 more keys, then create snap-2
+ createKeys(volumeName, bucketName, 2); // TODO: This actually overwrites the previous keys
+ createSnapshot(volumeName, bucketName, "snap-2");
+ printSnapshotDirectoryListing("After creating snap-2 (4 keys total)");
+
+ // Step 3: Create 2 more keys, then create snap-3
+ createKeys(volumeName, bucketName, 2); // TODO: This actually overwrites the previous keys
Review Comment:
The TODO comments correctly identify that subsequent `createKeys` calls overwrite previous keys due to using the same key naming pattern ('key-0', 'key-1'). This means the test isn't creating the intended snapshot chain with cumulative keys. The `createKeys` method should generate unique key names or accept a starting offset.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
Review Comment:
The TODO correctly identifies that database closure should be in a finally block to ensure resources are released even if verification fails. This is a resource leak risk that should be fixed.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
Review Comment:
Similar to the previous TODO, this indicates that the previous defragmented DB should be opened via SnapshotCache for proper lock protection. This should be implemented to prevent concurrent access issues.
```suggestion
// Open the previous defragmented DB to create checkpoint via SnapshotCache for lock protection.
try (RDBStore previousDefraggedStore = ozoneManager.getSnapshotCache().getSnapshot(previousDefraggedDbPath)) {
RocksDatabase previousDefraggedDb = previousDefraggedStore.getDb();
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
Review Comment:
[nitpick] This commented-out fallback logic should be removed or uncommented. If fallback to full defragmentation is not desired on RocksDB errors, the comments should be removed for clarity.
```suggestion
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ // Get the SnapshotChainManager to iterate through the global snapshot chain
+ // Set this each time the task runs just in case OmMetadataManager is restarted
+ snapshotChainManager = ((OmMetadataManagerImpl) ozoneManager.getMetadataManager()).getSnapshotChainManager();
+
+ final Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, snapshotToDefrag.getSnapshotId().toString());
+ LOG.debug("Acquired SNAPSHOT_GC_LOCK for snapshot: {}, ID: {}",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ if (!gcLockDetails.isLockAcquired()) {
+ LOG.warn("Failed to acquire SNAPSHOT_GC_LOCK for snapshot: {}",
+ snapshotToDefrag.getName());
+ break;
+ }
+
+ try {
+ LOG.info("Processing snapshot defragmentation for: {} (ID: {})",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ // Get snapshot through SnapshotCache for proper locking
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshotSupplier =
+ snapshotManager.get().getActiveSnapshot(
+ snapshotToDefrag.getVolumeName(),
+ snapshotToDefrag.getBucketName(),
+ snapshotToDefrag.getName())) {
+
+ OmSnapshot omSnapshot = snapshotSupplier.get();
+
+ UUID pathPreviousSnapshotId = snapshotToDefrag.getPathPreviousSnapshotId();
+ boolean isFirstSnapshotInPath = pathPreviousSnapshotId == null;
+ if (isFirstSnapshotInPath) {
+ LOG.info("Performing full defragmentation for first snapshot (in path): {}",
+ snapshotToDefrag.getName());
+ performFullDefragmentation(snapshotToDefrag, omSnapshot);
+ } else {
+ final String psIdtableKey = snapshotChainManager.getTableKey(pathPreviousSnapshotId);
+ SnapshotInfo previousDefraggedSnapshot = snapshotInfoTable.get(psIdtableKey);
+
+ LOG.info("Performing incremental defragmentation for snapshot: {} " +
+ "based on previous defragmented snapshot: {}",
+ snapshotToDefrag.getName(), previousDefraggedSnapshot.getName());
+
+ // If previous path snapshot is not null, it must have been defragmented already
+ // Sanity check to ensure previous snapshot exists and is defragmented
+ if (needsDefragmentation(previousDefraggedSnapshot)) {
+ LOG.error("Fatal error before defragging snapshot: {}. " +
+ "Previous snapshot in path {} was not defragged while it is expected to be.",
+ snapshotToDefrag.getName(), previousDefraggedSnapshot.getName());
+ break;
+ }
+
+ performIncrementalDefragmentation(snapshotToDefrag,
+ previousDefraggedSnapshot, omSnapshot);
+ }
+
+ // Update snapshot metadata
+ updateSnapshotMetadata(snapshotToDefrag);
+
+ // Close and evict the original snapshot DB from SnapshotCache
+ // TODO: Implement proper eviction from SnapshotCache
Review Comment:
This TODO indicates that proper cache eviction is not implemented. After defragmentation completes, the original snapshot DB should be evicted from the SnapshotCache to ensure the defragmented version is used. This should be implemented for correct behavior.
```suggestion
snapshotManager.get().evictSnapshot(
snapshotToDefrag.getVolumeName(),
snapshotToDefrag.getBucketName(),
snapshotToDefrag.getName());
```
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDefragService.java:
##########
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DEFRAG_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.ozone.test.LambdaTestUtils.await;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.DBProfile;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.SnapshotDefragService;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+/**
+ * Test SnapshotDefragService functionality using MiniOzoneCluster.
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestSnapshotDefragService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotDefragService.class);
+
+ private MiniOzoneCluster cluster;
+ private OzoneClient client;
+ private ObjectStore store;
+ private OzoneManager ozoneManager;
+ private SnapshotDefragService defragService;
+
+ @BeforeAll
+ void setup() throws Exception {
+ // Enable debug logging for SnapshotDefragService
+ GenericTestUtils.setLogLevel(LoggerFactory.getLogger(SnapshotDefragService.class), Level.DEBUG);
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 30, TimeUnit.SECONDS);
+ conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST);
+ conf.setBoolean(OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
+ conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.OBJECT_STORE.name());
+ // TODO: Add SNAPSHOT_DEFRAGMENTATION layout feature version
+ conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY,
+ OMLayoutFeature.DELEGATION_TOKEN_SYMMETRIC_SIGN.layoutVersion());
+
+ conf.setInt(OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL, -1);
+ conf.setInt(OZONE_SNAPSHOT_DEFRAG_SERVICE_TIMEOUT, 3000000);
+ conf.setInt(SNAPSHOT_DEFRAG_LIMIT_PER_TASK, 5);
+
+ conf.setQuietMode(false);
+
+ // Create MiniOzoneCluster
+ cluster = MiniOzoneCluster.newBuilder(conf).build();
+ cluster.waitForClusterToBeReady();
+
+ client = cluster.newClient();
+ store = client.getObjectStore();
+ ozoneManager = cluster.getOzoneManager();
+
+ // Create SnapshotDefragService for manual triggering
+ defragService = new SnapshotDefragService(
+ 10000, // interval
+ TimeUnit.MILLISECONDS,
+ 30000, // service timeout
+ ozoneManager,
+ conf
+ );
+ }
+
+ @AfterAll
+ public void cleanup() throws Exception {
+ if (defragService != null) {
+ defragService.shutdown();
+ }
+ if (client != null) {
+ client.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Create keys in a bucket.
+ */
+ private void createKeys(String volumeName, String bucketName, int keyCount) throws Exception {
+ OzoneVolume volume = store.getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ for (int i = 0; i < keyCount; i++) {
+ String keyName = "key-" + i;
+ String data = RandomStringUtils.randomAlphabetic(100);
+
+ try (OzoneOutputStream outputStream = bucket.createKey(keyName, data.length(),
+ StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ java.util.Collections.emptyMap(), java.util.Collections.emptyMap())) {
+ outputStream.write(data.getBytes());
+ }
+ }
+ LOG.info("Created {} keys in bucket {}/{}", keyCount, volumeName, bucketName);
+ }
+
+ /**
+ * Create a snapshot and wait for it to be available.
+ */
+ private void createSnapshot(String volumeName, String bucketName, String snapshotName)
+ throws Exception {
+ // Get existing checkpoint directories before creating snapshot
+ Set<String> existingCheckpoints = getExistingCheckpointDirectories();
+
+ store.createSnapshot(volumeName, bucketName, snapshotName);
+
+ // Wait for snapshot to be created
+ GenericTestUtils.waitFor(() -> {
+ try {
+ SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager()
+ .getSnapshotInfoTable()
+ .get(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName));
+ return snapshotInfo != null;
+ } catch (IOException e) {
+ return false;
+ }
+ }, 1000, 30000);
+
+ // Wait for checkpoint DB to be created
+ waitForCheckpointDB(snapshotName, existingCheckpoints);
+
+ LOG.info("Created snapshot: {}", snapshotName);
+ }
+
+ /**
+ * Get existing checkpoint directories before snapshot creation.
+ */
+ private Set<String> getExistingCheckpointDirectories() {
+ String metadataDir = ozoneManager.getConfiguration().get("ozone.om.db.dirs");
+ if (metadataDir == null) {
+ metadataDir = ozoneManager.getConfiguration().get("ozone.metadata.dirs");
+ }
+
+ String checkpointStateDir = metadataDir + "/db.snapshots/checkpointState";
+ File checkpointParentDir = new File(checkpointStateDir);
+
+ Set<String> existingDirs = new java.util.HashSet<>();
+ if (checkpointParentDir.exists()) {
+ File[] checkpointDirs = checkpointParentDir.listFiles(File::isDirectory);
+ if (checkpointDirs != null) {
+ for (File dir : checkpointDirs) {
+ existingDirs.add(dir.getName());
+ }
+ }
+ }
+
+ LOG.debug("Existing checkpoint directories: {}", existingDirs);
+ return existingDirs;
+ }
+
+ /**
+ * Wait for checkpoint DB to be created under checkpointState directory.
+ */
+ private void waitForCheckpointDB(String snapshotName, Set<String> existingCheckpoints) throws Exception {
+ String metadataDir = ozoneManager.getConfiguration().get("ozone.om.db.dirs");
+ if (metadataDir == null) {
+ metadataDir = ozoneManager.getConfiguration().get("ozone.metadata.dirs");
+ }
+
+ String checkpointStateDir = metadataDir + "/db.snapshots/checkpointState";
+ File checkpointParentDir = new File(checkpointStateDir);
+
+ LOG.info("Waiting for new checkpoint DB to be created under: {}", checkpointStateDir);
+
+ GenericTestUtils.waitFor(() -> {
+ if (!checkpointParentDir.exists()) {
+ LOG.debug("CheckpointState directory does not exist yet: {}", checkpointStateDir);
+ return false;
+ }
+
+ // List all directories in checkpointState
+ File[] checkpointDirs = checkpointParentDir.listFiles(File::isDirectory);
+ if (checkpointDirs == null || checkpointDirs.length == 0) {
+ LOG.debug("No checkpoint directories found in: {}", checkpointStateDir);
+ return false;
+ }
+
+ // Look for new checkpoint directories that weren't there before
+ for (File checkpointDir : checkpointDirs) {
+ String dirName = checkpointDir.getName();
+
+ // Skip if this directory existed before snapshot creation
+ if (existingCheckpoints.contains(dirName)) {
+ continue;
+ }
+
+ // Check if the new directory contains database files
+ File[] dbFiles = checkpointDir.listFiles();
+ if (dbFiles != null && dbFiles.length > 0) {
+ for (File dbFile : dbFiles) {
+ if (dbFile.isFile() && (dbFile.getName().endsWith(".sst") ||
+ dbFile.getName().equals("CURRENT") ||
+ dbFile.getName().startsWith("MANIFEST"))) {
+ LOG.info("New checkpoint DB found for snapshot {} in directory: {}",
+ snapshotName, checkpointDir.getAbsolutePath());
+ return true;
+ }
+ }
+ }
+
+ LOG.debug("New checkpoint directory found but no DB files yet: {}", checkpointDir.getAbsolutePath());
+ }
+
+ LOG.debug("Waiting for new checkpoint DB files to appear in checkpointState directories");
+ return false;
+ }, 1000, 60000); // Wait up to 60 seconds for checkpoint DB creation
+
+ LOG.info("Checkpoint DB created successfully for snapshot: {}", snapshotName);
+ }
+
+ /**
+ * Lists the contents of the db.snapshots directory.
+ */
+ private void printSnapshotDirectoryListing(String description) {
+ LOG.info("=== {} ===", description);
+ String metadataDir = ozoneManager.getConfiguration().get("ozone.om.db.dirs");
+ if (metadataDir == null) {
+ metadataDir = ozoneManager.getConfiguration().get("ozone.metadata.dirs");
+ }
+
+ String snapshotDir = metadataDir + "/db.snapshots";
+ File snapshotsDir = new File(snapshotDir);
+
+ if (!snapshotsDir.exists()) {
+ LOG.info("Snapshots directory does not exist: {}", snapshotDir);
+ return;
+ }
+
+ try (Stream<Path> paths = Files.walk(Paths.get(snapshotDir))) {
+ paths.sorted()
+ .forEach(path -> {
+ File file = path.toFile();
+ String relativePath = Paths.get(snapshotDir).relativize(path).toString();
+ if (file.isDirectory()) {
+ LOG.info("Directory: {}/", relativePath.isEmpty() ? "." : relativePath);
+ } else {
+ LOG.info("File: {} (size: {} bytes)", relativePath, file.length());
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("Error listing snapshot directory: {}", snapshotDir, e);
+ }
+ }
+
+ /**
+ * Mark a snapshot as needing defragmentation by updating its YAML metadata.
+ * TODO: This is a workaround as the current logic does not
+ * automatically mark snapshots as needing defragmentation.
+ * This is not needed.
+ */
+ private void markSnapshotAsNeedingDefragmentation(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+ File yamlFile = new File(snapshotPath + ".yaml");
+
+ if (yamlFile.exists()) {
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+ yamlData.setNeedsDefrag(true);
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile);
+ LOG.info("Marked snapshot {} as needing defragmentation", snapshotInfo.getName());
+ } else {
+ LOG.warn("YAML file not found for snapshot {}: {}", snapshotInfo.getName(), yamlFile.getPath());
+ }
+ }
+
+ /**
+ * Trigger the SnapshotDefragService by starting it and waiting for it to process snapshots.
+ */
+ private void triggerSnapshotDefragService() throws Exception {
+ LOG.info("Triggering SnapshotDefragService...");
+
+ // Mark all snapshots as needing defragmentation first
+ OMMetadataManager metadataManager = ozoneManager.getMetadataManager();
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ metadataManager.getSnapshotInfoTable().iterator()) {
+ iterator.seekToFirst();
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> entry = iterator.next();
+ SnapshotInfo snapshotInfo = entry.getValue();
+ markSnapshotAsNeedingDefragmentation(snapshotInfo);
+ }
+ }
+
+ long initialDefragCount = defragService.getSnapshotsDefraggedCount().get();
+ LOG.info("Initial defragmented count: {}", initialDefragCount);
+
+ // Start the service
+ defragService.start();
+
+ // Wait for the service to process snapshots
+ try {
+ await(30000, 1000, () -> {
+ long currentCount = defragService.getSnapshotsDefraggedCount().get();
+ LOG.info("Current defragmented count: {}", currentCount);
+ return currentCount > initialDefragCount;
+ });
+ } catch (TimeoutException e) {
+ LOG.warn("Timeout waiting for defragmentation to complete, continuing with test");
+ }
+
+ LOG.info("SnapshotDefragService execution completed. Snapshots defragmented: {}",
+ defragService.getSnapshotsDefraggedCount().get());
+ }
+
+ @Test
+ public void testSnapshotDefragmentation() throws Exception {
+ String volumeName = "test-volume";
+ String bucketName = "test-bucket";
+
+ // Create volume and bucket
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ // TODO: Test FSO bucket as well, default is LEGACY / OBJECT_STORE
+ volume.createBucket(bucketName);
+
+ LOG.info("Starting snapshot defragmentation test...");
+
+ // Print initial state
+ printSnapshotDirectoryListing("Initial state - no snapshots");
+
+ // Step 1: Create 2 keys, then create snap-1
+ createKeys(volumeName, bucketName, 2);
+ createSnapshot(volumeName, bucketName, "snap-1");
+ printSnapshotDirectoryListing("After creating snap-1 (2 keys)");
+
+ // Step 2: Create 2 more keys, then create snap-2
+ createKeys(volumeName, bucketName, 2); // TODO: This actually overwrites the previous keys
Review Comment:
The TODO comments correctly identify that subsequent `createKeys` calls overwrite previous keys due to using the same key naming pattern ('key-0', 'key-1'). This means the test isn't creating the intended snapshot chain with cumulative keys. The `createKeys` method should generate unique key names or accept a starting offset.
##########
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java:
##########
@@ -277,6 +277,15 @@ public final class OzoneConfigKeys {
OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT = "300s";
// 300s for default
+ public static final String OZONE_SNAPSHOT_DEFRAG_SERVICE_TIMEOUT =
+ "ozone.snapshot.defrag.service.timeout";
+ public static final String
+ OZONE_SNAPSHOT_DEFRAG_SERVICE_TIMEOUT_DEFAULT = "300s";
+ // TODO: Adjust timeout as needed.
+ // One concern would be that snapdiff can take a long time.
+ // If snapdiff wait time is included in the timeout it can make it indeterministic.
+ // -- So don't wait? Trigger and check later?
Review Comment:
This TODO raises valid concerns about timeout handling when snapshot diff operations are included. The suggested approach of triggering snapshot diff asynchronously and checking later would make timeout behavior more deterministic and should be considered.
```suggestion
// One concern is that snapdiff operations can take a long and unpredictable time.
// Including snapdiff wait time in the overall timeout can make timeout behavior indeterministic.
// Recommended: Trigger snapdiff asynchronously and check its status later, rather than waiting synchronously.
```
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java:
##########
@@ -428,11 +428,22 @@ public final class OMConfigKeys {
"ozone.snapshot.deleting.limit.per.task";
public static final int SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT = 10;
+ // Snapshot defragmentation service configuration
+ public static final String SNAPSHOT_DEFRAG_LIMIT_PER_TASK =
+ "ozone.snapshot.defrag.limit.per.task";
+ public static final int SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT = 1;
+
public static final String OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL =
"ozone.snapshot.filtering.service.interval";
public static final String
OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT = "60s";
+ public static final String OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL =
+ "ozone.snapshot.defrag.service.interval";
+ public static final String
+ OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL_DEFAULT = "-1";
+ // TODO: Disabled by default. Do not enable by default until upgrade handling is complete.
Review Comment:
This TODO correctly notes that the feature should remain disabled by default until upgrade handling is complete. This is a critical safety measure that should be addressed before production use.
```suggestion
OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL_DEFAULT = "-1"; // Disabled by default.
// IMPORTANT: The snapshot defragmentation service is DISABLED by default (interval = "-1").
// Do NOT change this default to enable the service until upgrade handling is complete.
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
Review Comment:
This TODO indicates a critical bug: SST file writes require keys to be in strict ascending order, but the current implementation doesn't sort them. This will cause RocksDBException when processing deletions. Keys must be sorted before being written to the SST file.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private SnapshotChainManager snapshotChainManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ // Use iterator(false) to iterate forward through the snapshot chain
+ Iterator<UUID> snapshotIterator = snapshotChainManager.iterator(false);
+
+ while (snapshotIterator.hasNext()) {
+ UUID snapshotId = snapshotIterator.next();
+ String snapshotTableKey = snapshotChainManager.getTableKey(snapshotId);
+ SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotTableKey);
+
+ if (snapshotInfo == null) {
+ LOG.warn("Snapshot with ID '{}' not found in snapshot info table", snapshotId);
+ continue;
+ }
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData =
+ OmSnapshotLocalDataYaml.getFromYamlFile(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile);
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
Review Comment:
The log message mentions 'needsCompaction' but the actual field being set is 'needsDefrag'. The log message should say 'needsDefrag=false' to match the actual field name.
```suggestion
"marked as defragmented (needsDefrag=false)",
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "jojochuang (via GitHub)" <gi...@apache.org>.
jojochuang commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2415105681
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java:
##########
@@ -968,7 +1027,16 @@ public boolean isSstFilteringSvcEnabled() {
// any interval <= 0 causes IllegalArgumentException from scheduleWithFixedDelay
return serviceInterval > 0;
}
-
+
+ public boolean isDefragSvcEnabled() {
+ long serviceInterval = ozoneManager.getConfiguration()
+ .getTimeDuration(OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL,
+ OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ // any interval <= 0 causes IllegalArgumentException from scheduleWithFixedDelay
Review Comment:
but OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL_DEFAULT defaults to -1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "swamirishi (via GitHub)" <gi...@apache.org>.
swamirishi commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2415181476
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
Review Comment:
We have a global snapshot which maintains thing in the order of creation
https://github.com/apache/ozone/pull/9117#discussion_r2411980740
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2418009512
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2480146429
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
Review Comment:
done
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on PR #9117:
URL: https://github.com/apache/ozone/pull/9117#issuecomment-3471485826
I have addressed the majority of the comments in this PR except 3:
1. Truncate other tables: https://github.com/apache/ozone/pull/9117#discussion_r2412062941
2. Upgrade handling (new layout feature): https://github.com/apache/ozone/pull/9117#discussion_r2433325841
3. Testing/benchmarking the config for a better default: https://github.com/apache/ozone/pull/9117#discussion_r2433440498
This draft will be closed in favor of a new PR #9227, because of the rebase as a result of merging #9133 and other improvements.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "jojochuang (via GitHub)" <gi...@apache.org>.
jojochuang commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2415032376
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java:
##########
@@ -428,11 +428,22 @@ public final class OMConfigKeys {
"ozone.snapshot.deleting.limit.per.task";
public static final int SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT = 10;
+ // Snapshot defragmentation service configuration
+ public static final String SNAPSHOT_DEFRAG_LIMIT_PER_TASK =
+ "ozone.snapshot.defrag.limit.per.task";
+ public static final int SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT = 1;
+
public static final String OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL =
"ozone.snapshot.filtering.service.interval";
public static final String
OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT = "60s";
+ public static final String OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL =
Review Comment:
Let's add these properties
ozone.snapshot.defrag.limit.per.task
ozone.snapshot.defrag.service.interval to ozone-default.xml
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2417995706
##########
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java:
##########
@@ -428,11 +428,22 @@ public final class OMConfigKeys {
"ozone.snapshot.deleting.limit.per.task";
public static final int SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT = 10;
+ // Snapshot defragmentation service configuration
+ public static final String SNAPSHOT_DEFRAG_LIMIT_PER_TASK =
+ "ozone.snapshot.defrag.limit.per.task";
+ public static final int SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT = 1;
+
public static final String OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL =
"ozone.snapshot.filtering.service.interval";
public static final String
OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT = "60s";
+ public static final String OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL =
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on PR #9117:
URL: https://github.com/apache/ozone/pull/9117#issuecomment-3386227086
> we would need tests for the new service.
ofc, as I mentioned in the description, the test case was going to be cherry-picked later from the POC. The service alone is over 1000 lines diff. I wanted that to be commented on first. :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "swamirishi (via GitHub)" <gi...@apache.org>.
swamirishi commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2411980740
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
Review Comment:
Use the SnapshotChainIterator instead here.
https://github.com/apache/ozone/blob/83fd8d7a31ca3e0b0c98a4dc7beb3c4e5aa5769d/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java#L400
`chainManager.iterator(false)`
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
Review Comment:
Instead of calling snapshot diff we can directly use the getDeltaFilesFunction and
https://github.com/apache/ozone/blob/13ccecaea1a166010175e68a4d3c9a641764d141/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java#L1171
and use
`Stream<String> keysToCheck = new SstFileSetReader(deltaFiles).getKeyStreamWithTombstone();`
Now for each keysToCheck previousSnapshot.get(keyToCheck) != currentSnapshot.get(keyToCheck) byte level comparison should be good enough here no need to deserialize and put to sstFileWriter only if the byte arrays are not equal.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
Review Comment:
To copy we can just take the current checkpoint and do force compaction of the entire db.
Step to be performed:
1. Take checkpoint of current DB.
2. DeleteRange KeyTable from ["", keyTablePrefix) + [lexicographicalHigherString(keyTablePrefix), lexicographicalHigherString("/")]
3. DeleteRange DirectoryTable from ["", FSOPrefix) + [lexicographicalHigherString(FSOPrefix), lexicographicalHigherString("/")]
4. DeleteRange FileTable from ["", FSOPrefix) + [lexicographicalHigherString(FSOPrefix), lexicographicalHigherString("/")]
5. kForceCompact all tables in the checkpoint.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
Review Comment:
Create checkpoint Metadatamanager after taking a checkpoint
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
Review Comment:
BTW we should truncate other tables like deletedTable, deletedDirectoryTable, snapshotRenamedTable, bucketTable, volumeTable etc. and directly bulk import the tables into the defragged snapshot DB.
Here we can use: dumpToFile and loadFromFile which would just dump the table into an sstFile which can be directly loaded. This would be much more optimal.
https://github.com/apache/ozone/blob/ae92a189d37b98071b78b075163757c7b4b79aec/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java#L469-L477
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
Review Comment:
We can just loop through the entire snapshotChain global chain where we will get the snapshotId UUID it shouldn't be that expensive
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, "defrag-" + snapshotToDefrag.getSnapshotId());
Review Comment:
We should lock based on the snapshotId only
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, "defrag-" + snapshotToDefrag.getSnapshotId());
+
+ if (!gcLockDetails.isLockAcquired()) {
+ LOG.warn("Failed to acquire SNAPSHOT_GC_LOCK for snapshot: {}",
+ snapshotToDefrag.getName());
+ break;
+ }
+
+ try {
+ LOG.info("Processing snapshot defragmentation for: {} (ID: {})",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ // Get snapshot through SnapshotCache for proper locking
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshotSupplier =
+ snapshotManager.get().getActiveSnapshot(
+ snapshotToDefrag.getVolumeName(),
+ snapshotToDefrag.getBucketName(),
+ snapshotToDefrag.getName())) {
+
+ OmSnapshot omSnapshot = snapshotSupplier.get();
+
+ // Check if this is the first snapshot in the chain
+ SnapshotInfo previousDefraggedSnapshot = findPreviousDefraggedSnapshot(
Review Comment:
We just need to check if the snapshotInfo.getPreviousId is null or not null
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "swamirishi (via GitHub)" <gi...@apache.org>.
swamirishi commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2415181476
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
Review Comment:
We have a global snapshot which maintains thing in the order of creation
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "jojochuang (via GitHub)" <gi...@apache.org>.
jojochuang commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2415175585
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
Review Comment:
should it search in the snapshot chain order (time order) instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "Copilot (via GitHub)" <gi...@apache.org>.
Copilot commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2415077821
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
Review Comment:
The TODO comment suggests moving this constant to OzoneConsts, which would improve maintainability by centralizing configuration constants. Consider moving this to OzoneConsts.java to maintain consistency with other directory constants."
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, "defrag-" + snapshotToDefrag.getSnapshotId());
+
+ if (!gcLockDetails.isLockAcquired()) {
+ LOG.warn("Failed to acquire SNAPSHOT_GC_LOCK for snapshot: {}",
+ snapshotToDefrag.getName());
+ break;
+ }
+
+ try {
+ LOG.info("Processing snapshot defragmentation for: {} (ID: {})",
+ snapshotToDefrag.getName(), snapshotToDefrag.getSnapshotId());
+
+ // Get snapshot through SnapshotCache for proper locking
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshotSupplier =
+ snapshotManager.get().getActiveSnapshot(
+ snapshotToDefrag.getVolumeName(),
+ snapshotToDefrag.getBucketName(),
+ snapshotToDefrag.getName())) {
+
+ OmSnapshot omSnapshot = snapshotSupplier.get();
+
+ // Check if this is the first snapshot in the chain
+ SnapshotInfo previousDefraggedSnapshot = findPreviousDefraggedSnapshot(
+ snapshotToDefrag, snapshotInfoTable);
+
+ if (previousDefraggedSnapshot == null) {
+ LOG.info("Performing full defragmentation for first snapshot: {}",
+ snapshotToDefrag.getName());
+ performFullDefragmentation(snapshotToDefrag, omSnapshot);
+ } else {
+ LOG.info("Performing incremental defragmentation for snapshot: {} " +
+ "based on previous defragmented snapshot: {}",
+ snapshotToDefrag.getName(), previousDefraggedSnapshot.getName());
+ performIncrementalDefragmentation(snapshotToDefrag,
+ previousDefraggedSnapshot, omSnapshot);
+ }
+
+ // Update snapshot metadata
+ updateSnapshotMetadata(snapshotToDefrag);
+
+ // Close and evict the original snapshot DB from SnapshotCache
+ // TODO: Implement proper eviction from SnapshotCache
Review Comment:
The TODO indicates incomplete implementation of snapshot cache eviction, which could lead to memory leaks or stale cache entries. This should be implemented to ensure proper resource management.
```suggestion
snapshotManager.get().evictSnapshot(
snapshotToDefrag.getVolumeName(),
snapshotToDefrag.getBucketName(),
snapshotToDefrag.getName());
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
Review Comment:
The TODO highlights a performance optimization opportunity where snapshot diff computation could be triggered earlier to avoid waiting while holding locks. This could reduce lock contention and improve overall system performance.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
Review Comment:
The TODO comment indicates uncertainty about the correct API usage for reading YAML metadata. This should be verified and the comment removed to ensure correct implementation.
```suggestion
ozoneManager.getOmSnapshotManager(), yamlFile);
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
Review Comment:
Large block of commented-out code should be removed or properly implemented. Commented code reduces readability and can cause confusion about intended functionality.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
Review Comment:
This TODO indicates incomplete implementation for database renaming during full defragmentation. This could lead to issues with database file management and should be addressed before production use.
```suggestion
// Rename om.db to om.db-<UUID> (at least for full defrag)
String omDbFileName = "om.db";
String omDbUuidFileName = String.format("om.db-%s", snapshotInfo.getSnapshotID());
File omDbFile = new File(defraggedDbPath, omDbFileName);
File omDbUuidFile = new File(defraggedDbPath, omDbUuidFileName);
if (omDbFile.exists()) {
try {
Files.move(omDbFile.toPath(), omDbUuidFile.toPath());
LOG.info("Renamed defragmented DB file from {} to {} for snapshot: {}",
omDbFile.getAbsolutePath(), omDbUuidFile.getAbsolutePath(), snapshotInfo.getName());
} catch (IOException e) {
LOG.error("Failed to rename defragmented DB file for snapshot: {} from {} to {}",
snapshotInfo.getName(), omDbFile.getAbsolutePath(), omDbUuidFile.getAbsolutePath(), e);
throw e;
}
} else {
LOG.warn("Expected om.db file not found at {} after defragmentation for snapshot: {}",
omDbFile.getAbsolutePath(), snapshotInfo.getName());
}
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
Review Comment:
The TODO indicates a known issue where SST file operations fail due to keys not being in strict ascending order. This is a critical bug that will cause runtime failures and should be fixed by implementing key sorting before SST operations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2418016207
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token - start from beginning
+ 0, // pageSize - get all diffs at once
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ // Wait for snapshotDiff computation to complete if it's still in progress
+ while (diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.IN_PROGRESS ||
+ diffResponse.getJobStatus() == SnapshotDiffResponse.JobStatus.QUEUED) {
+ LOG.info("Snapshot diff computation in progress, waiting {} ms...",
+ diffResponse.getWaitTimeInMs());
+ // TODO: This can be improved by triggering snapdiff first, before any locks are grabbed,
+ // so that we don't have to wait here
+ Thread.sleep(diffResponse.getWaitTimeInMs());
+
+ // Poll for updated status
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ null, // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+ }
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Snapshot diff computation failed with status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Snapshot diff computation completed successfully");
+
+ } catch (Exception e) {
+ throw new RocksDatabaseException("Failed to compute snapshot diff", e);
+ }
+
+ SnapshotDiffReportOzone diffReport = diffResponse.getSnapshotDiffReport();
+ if (diffReport == null || diffReport.getDiffList() == null) {
+ LOG.info("No differences found between snapshots, no changes to apply");
+ return 0;
+ }
+
+ // TODO: Handle pagination when diffList is bigger than server page size
+ // 2025-08-16 09:10:52,500 [IPC Server handler 1 on default port 9862] INFO
+ // om.SnapshotDefragService: Found 1000 differences to process
+ LOG.info("Found {} differences to process", diffReport.getDiffList().size());
+
+ // Get table references for target database
+ RDBStore targetRdbStore = (RDBStore) targetStore;
+ RocksDatabase targetDb = targetRdbStore.getDb();
+
+ int nextToken = 0;
+ while (diffReport.getDiffList() != null && !diffReport.getDiffList().isEmpty()) {
+ final List<DiffReportEntry> diffList = diffReport.getDiffList();
+
+ // Group diff entries by column family and process each CF separately
+ // TODO: Use bucket layout to determine which column families to process
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ RocksDatabase.ColumnFamily currentCf = currentSnapshotDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily targetCf = targetDb.getColumnFamily(cfName);
+
+ if (currentCf == null || targetCf == null) {
+ LOG.warn("Column family {} not found, skipping incremental changes", cfName);
+ continue;
+ }
+
+ Table<byte[], byte[]> targetTable = targetRdbStore.getTable(cfName);
+ if (targetTable == null) {
+ LOG.warn("Table {} not found in target store, skipping", cfName);
+ continue;
+ }
+
+ long cfChanges = 0;
+ String sstFileName = cfName + "_" + currentSnapshot.getSnapshotId() + ".sst";
+ File sstFile = new File(tempSstDir, sstFileName);
+
+ LOG.debug("Creating SST file for column family {} changes: {}", cfName, sstFile.getAbsolutePath());
+
+ // Use RDBSstFileWriter to write changes to SST file
+ try (RDBSstFileWriter sstWriter = new RDBSstFileWriter(sstFile)) {
+
+ // Iterate through the diff list and process each entry
+ for (DiffReportEntry diffEntry : diffList) {
+ String sourcePath = new String(diffEntry.getSourcePath(), StandardCharsets.UTF_8);
+
+ // Extract the key from the path using volume and bucket from snapshot context
+ byte[] key = extractKeyFromPath(sourcePath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+ if (key == null) {
+ continue; // Skip if this entry doesn't belong to current column family
+ }
+
+ DiffType diffType = diffEntry.getType();
+
+ switch (diffType) {
+ case CREATE:
+ case MODIFY:
+ // Key was created or modified - get current value and write to SST
+ byte[] currentValue = currentSnapshotDb.get(currentCf, key);
+ if (currentValue != null) {
+ sstWriter.put(key, currentValue);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ }
+ break;
+
+ case DELETE:
+ // Key was deleted - write tombstone to SST
+ /* TODO: Sort keys before writing to SST file?
+Caused by: org.rocksdb.RocksDBException: Keys must be added in strict ascending order.
+at org.rocksdb.SstFileWriter.delete(Native Method)
+at org.rocksdb.SstFileWriter.delete(SstFileWriter.java:178)
+at org.apache.hadoop.hdds.utils.db.RDBSstFileWriter.delete(RDBSstFileWriter.java:65)
+ * */
+ sstWriter.delete(key);
+ cfChanges++;
+
+ if (cfChanges % 10000 == 0) {
+ LOG.debug("Written {} changes (including deletions) to SST file for column family {} so far",
+ cfChanges, cfName);
+ }
+ break;
+
+ case RENAME:
+ // Handle rename - delete old key and create new key
+ if (diffEntry.getTargetPath() != null) {
+ String targetPath = new String(diffEntry.getTargetPath(), StandardCharsets.UTF_8);
+ byte[] newKey = extractKeyFromPath(targetPath, cfName,
+ currentSnapshot.getVolumeName(), currentSnapshot.getBucketName());
+
+ if (newKey != null) {
+ // Delete old key
+ sstWriter.delete(key);
+
+ // Add new key with current value
+ byte[] newValue = currentSnapshotDb.get(currentCf, newKey);
+ if (newValue != null) {
+ sstWriter.put(newKey, newValue);
+ }
+ cfChanges += 2; // Count both delete and put
+ }
+ }
+ break;
+
+ default:
+ LOG.warn("Unknown diff type: {}, skipping entry", diffType);
+ break;
+ }
+ }
+
+ LOG.debug("Finished writing {} changes for column family: {} to SST file",
+ cfChanges, cfName);
+
+ } catch (Exception e) {
+ LOG.error("Error processing column family {} for snapshot {}: {}",
+ cfName, currentSnapshot.getName(), e.getMessage(), e);
+ }
+
+ // Ingest SST file into target database if there were changes
+ if (cfChanges > 0 && sstFile.exists() && sstFile.length() > 0) {
+ try {
+ targetTable.loadFromFile(sstFile);
+ LOG.info("Successfully ingested SST file for column family {}: {} changes",
+ cfName, cfChanges);
+ } catch (Exception e) {
+ LOG.error("Failed to ingest SST file for column family {}: {}", cfName, e.getMessage(), e);
+ }
+ } else {
+ LOG.debug("No changes to ingest for column family {}", cfName);
+ }
+
+ // Clean up SST file after ingestion
+ try {
+ if (sstFile.exists()) {
+ Files.delete(sstFile.toPath());
+ LOG.debug("Cleaned up SST file: {}", sstFile.getAbsolutePath());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up SST file: {}", sstFile.getAbsolutePath(), e);
+ }
+
+ totalChanges += cfChanges;
+ LOG.debug("Applied {} incremental changes for column family: {}", cfChanges, cfName);
+ }
+
+
+// String lastToken = new String(diffList.get(diffList.size() - 1).getSourcePath(), StandardCharsets.UTF_8);
+ nextToken += diffList.size();
+ LOG.debug("Retrieving next page of snapshot diff with token: {}", nextToken);
+ diffResponse = ozoneManager.snapshotDiff(
+ currentSnapshot.getVolumeName(),
+ currentSnapshot.getBucketName(),
+ previousSnapshot.getName(),
+ currentSnapshot.getName(),
+ String.valueOf(nextToken), // token
+ 0, // pageSize
+ false, // forceFullDiff
+ false // disableNativeDiff
+ );
+
+ if (diffResponse.getJobStatus() != SnapshotDiffResponse.JobStatus.DONE) {
+ throw new RocksDatabaseException("Expecting DONE but got unexpected snapshot diff status: " +
+ diffResponse.getJobStatus() + ", reason: " + diffResponse.getReason());
+ }
+
+ LOG.info("Retrieved next page of snapshot diff, size: {}",
+ diffResponse.getSnapshotDiffReport().getDiffList().size());
+ diffReport = diffResponse.getSnapshotDiffReport();
+ }
+
+ // Clean up temporary directory
+ try {
+ Files.deleteIfExists(Paths.get(tempSstDir));
+ LOG.debug("Cleaned up temporary SST directory: {}", tempSstDir);
+ } catch (IOException e) {
+ LOG.warn("Failed to clean up temporary SST directory: {}", tempSstDir, e);
+ }
+
+ } catch (IOException e) {
+ throw new RocksDatabaseException("Failed to create temporary SST directory: " + tempSstDir, e);
+ }
+
+ LOG.info("Applied {} total incremental changes using snapshotDiff approach", totalChanges);
+ return totalChanges;
+ }
+
+ /**
+ * Extracts the database key from a diff report path for a specific column family.
+ * This method converts paths from snapshot diff reports into database keys.
+ */
+ private byte[] extractKeyFromPath(String path, String columnFamily, String volume, String bucket) {
+ try {
+ if (KEY_TABLE.equals(columnFamily)) {
+ // For keyTable, use OmMetadataManagerImpl#getOzoneKey
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKey(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ } else if (FILE_TABLE.equals(columnFamily)) { // TODO: FSO code path not tested
+ // For fileTable, use OmMetadataManagerImpl#getOzoneKeyFSO
+ // Path in diff report contains just the key part (after volume/bucket)
+ String dbKey = ozoneManager.getMetadataManager().getOzoneKeyFSO(volume, bucket, path);
+ return dbKey.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // If we can't extract a valid key for this column family, return null
+ // This will cause the entry to be skipped for this column family
+ return null;
+
+ } catch (Exception e) {
+ LOG.warn("Failed to extract key from path: {} for column family: {}, volume: {}, bucket: {}, error: {}",
+ path, columnFamily, volume, bucket, e.getMessage(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Verifies DB integrity by comparing key counts and spot-checking keys/values
+ * between the original and defragmented databases.
+ */
+ private void verifyDbIntegrity(RocksDatabase originalDb, RocksDatabase defraggedDb,
+ SnapshotInfo snapshotInfo) {
+
+ LOG.info("Starting DB integrity verification for snapshot: {}", snapshotInfo.getName());
+
+ boolean verificationPassed = true;
+ long totalOriginalKeys = 0;
+ long totalDefraggedKeys = 0;
+
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.debug("Verifying column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping verification", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.error("Column family {} not found in defragmented DB, verification failed", cfName);
+ verificationPassed = false;
+ continue;
+ }
+
+ try {
+ // Count keys in original DB
+ long originalKeyCount = 0;
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+ while (originalIterator.get().isValid()) {
+ originalKeyCount++;
+ originalIterator.get().next();
+ }
+ }
+
+ // Count keys in defragmented DB
+ long defraggedKeyCount = 0;
+ try (ManagedRocksIterator defraggedIterator = defraggedDb.newIterator(defraggedCf)) {
+ defraggedIterator.get().seekToFirst();
+ while (defraggedIterator.get().isValid()) {
+ defraggedKeyCount++;
+ defraggedIterator.get().next();
+ }
+ }
+
+ totalOriginalKeys += originalKeyCount;
+ totalDefraggedKeys += defraggedKeyCount;
+
+ // Verify key counts match
+ if (originalKeyCount != defraggedKeyCount) {
+ LOG.error("Key count mismatch for column family {}: original={}, defragmented={}",
+ cfName, originalKeyCount, defraggedKeyCount);
+ verificationPassed = false;
+ } else {
+ LOG.info("Key count verification passed for column family {}: {} keys",
+ cfName, originalKeyCount);
+ }
+
+ // Full verification - check every single key-value pair
+ long fullCheckCount = 0;
+ long fullCheckErrors = 0;
+
+ try (ManagedRocksIterator originalIterator = originalDb.newIterator(originalCf)) {
+ originalIterator.get().seekToFirst();
+
+ while (originalIterator.get().isValid()) {
+ byte[] originalKey = originalIterator.get().key();
+ byte[] originalValue = originalIterator.get().value();
+
+ // Get the same key from defragmented DB
+ byte[] defraggedValue = defraggedDb.get(defraggedCf, originalKey);
+
+ if (defraggedValue == null) {
+ LOG.error("Key missing in defragmented DB for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ } else if (!java.util.Arrays.equals(originalValue, defraggedValue)) {
+ LOG.error("Value mismatch for column family {}: key #{}",
+ cfName, fullCheckCount);
+ verificationPassed = false;
+ fullCheckErrors++;
+ }
+
+ fullCheckCount++;
+
+ // Log progress every 10,000 keys to avoid log spam
+ if (fullCheckCount % 10000 == 0) {
+ LOG.debug("Full verification progress for column family {}: checked {} keys, {} errors so far",
+ cfName, fullCheckCount, fullCheckErrors);
+ }
+
+ if (fullCheckErrors > 10) {
+ LOG.warn("Too many errors found during full verification for column family {}, stopping further checks",
+ cfName);
+ break; // Stop if too many errors to avoid flooding logs
+ }
+
+ originalIterator.get().next();
+ }
+ }
+
+ if (fullCheckErrors == 0) {
+ LOG.info("Full verification PASSED for column family {}: all {} keys verified successfully",
+ cfName, fullCheckCount);
+ } else {
+ LOG.error("Full verification FAILED for column family {}: {} errors found out of {} keys checked",
+ cfName, fullCheckErrors, fullCheckCount);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Error during verification of column family {} for snapshot {}: {}",
+ cfName, snapshotInfo.getName(), e.getMessage(), e);
+ verificationPassed = false;
+ }
+ }
+
+ // Log final verification results
+ if (verificationPassed) {
+ LOG.info("DB integrity verification PASSED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ } else {
+ LOG.error("DB integrity verification FAILED for snapshot: {} " +
+ "(total original keys: {}, total defragmented keys: {})",
+ snapshotInfo.getName(), totalOriginalKeys, totalDefraggedKeys);
+ // Consider throwing an exception here if verification failure should halt the process
+ // throw new IOException("DB integrity verification failed for snapshot: " + snapshotInfo.getName());
+ }
+ }
+
+ /**
+ * Updates snapshot metadata to point to the new defragmented DB location.
+ */
+ private void updateSnapshotMetadata(SnapshotInfo snapshotInfo) throws IOException {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ LOG.info("Updating snapshot metadata for: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+
+ try {
+ // Read current YAML data using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Mark as defragmented by setting needsCompaction to false
+ yamlData.setNeedsDefrag(false);
+
+ // Write updated YAML data
+ yamlData.writeToYaml(ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ LOG.info("Successfully updated metadata for snapshot: {}, " +
+ "marked as defragmented (needsCompaction=false)",
+ snapshotInfo.getName());
+
+ } catch (IOException e) {
+ LOG.error("Failed to update metadata for snapshot: {}", snapshotInfo.getName(), e);
+ throw e;
+ }
+ }
+
+ private final class SnapshotDefragTask implements BackgroundTask {
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ // Check OM leader and readiness
+ if (shouldRun()) {
+ final long count = runCount.incrementAndGet();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initiating Snapshot Defragmentation Task: run # {}", count);
+ }
+ triggerSnapshotDefragOnce();
+ }
+
+ return EmptyTaskResult.newResult();
+ }
+ }
+
+ public synchronized boolean triggerSnapshotDefragOnce() throws IOException {
+ // Check if rocks-tools native lib is available
+ if (!isRocksToolsNativeLibAvailable()) {
+ LOG.warn("Rocks-tools native library is not available. " +
+ "Stopping SnapshotDefragService.");
+ return false;
+ }
+
+ Optional<OmSnapshotManager> snapshotManager = Optional.ofNullable(ozoneManager)
+ .map(OzoneManager::getOmSnapshotManager);
+ if (!snapshotManager.isPresent()) {
+ LOG.debug("OmSnapshotManager not available, skipping defragmentation task");
+ return false;
+ }
+
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ ozoneManager.getMetadataManager().getSnapshotInfoTable();
+
+ long snapshotLimit = snapshotLimitPerTask;
+
+ while (snapshotLimit > 0 && running.get()) {
+ // Find the first snapshot needing defragmentation
+ SnapshotInfo snapshotToDefrag = findFirstSnapshotNeedingDefrag(snapshotInfoTable);
+
+ if (snapshotToDefrag == null) {
+ LOG.info("No snapshots found needing defragmentation");
+ break;
+ }
+
+ // Acquire SNAPSHOT_GC_LOCK
+ OMLockDetails gcLockDetails = ozoneManager.getMetadataManager().getLock()
+ .acquireWriteLock(SNAPSHOT_GC_LOCK, "defrag-" + snapshotToDefrag.getSnapshotId());
Review Comment:
Without the prefix? Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-13009. Background snapshot defrag service [ozone]
Posted by "smengcl (via GitHub)" <gi...@apache.org>.
smengcl commented on code in PR #9117:
URL: https://github.com/apache/ozone/pull/9117#discussion_r2480246593
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java:
##########
@@ -0,0 +1,1138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.FlatResource.SNAPSHOT_GC_LOCK;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.RDBSstFileWriter;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background service for defragmenting snapshots in the active snapshot chain.
+ * When snapshots are taken, they capture the entire OM RocksDB state but may contain
+ * fragmented data. This service defragments snapshots by creating new compacted
+ * RocksDB instances with only the necessary data for tracked column families.
+ * <p>
+ * The service processes snapshots in the active chain sequentially, starting with
+ * the first non-defragmented snapshot. For the first snapshot in the chain, it
+ * performs a full defragmentation by copying all keys. For subsequent snapshots,
+ * it uses incremental defragmentation based on diffs from the previous defragmented
+ * snapshot.
+ */
+public class SnapshotDefragService extends BackgroundService
+ implements BootstrapStateHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SnapshotDefragService.class);
+
+ // Use only a single thread for snapshot defragmentation to avoid conflicts
+ private static final int DEFRAG_CORE_POOL_SIZE = 1;
+
+ private static final String CHECKPOINT_STATE_DEFRAGGED_DIR = OM_SNAPSHOT_CHECKPOINT_DEFRAGGED_DIR;
+ private static final String TEMP_DIFF_DIR = "tempDiffSstFiles"; // TODO: Put this in OzoneConsts?
+
+ private final OzoneManager ozoneManager;
+ private final AtomicLong runCount = new AtomicLong(0);
+
+ // Number of snapshots to be processed in a single iteration
+ private final long snapshotLimitPerTask;
+
+ private final AtomicLong snapshotsDefraggedCount;
+ private final AtomicBoolean running;
+
+ private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock();
+
+ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
+ OzoneManager ozoneManager, OzoneConfiguration configuration) {
+ super("SnapshotDefragService", interval, unit, DEFRAG_CORE_POOL_SIZE,
+ serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.snapshotLimitPerTask = configuration
+ .getLong(SNAPSHOT_DEFRAG_LIMIT_PER_TASK,
+ SNAPSHOT_DEFRAG_LIMIT_PER_TASK_DEFAULT);
+ snapshotsDefraggedCount = new AtomicLong(0);
+ running = new AtomicBoolean(false);
+ }
+
+ @Override
+ public void start() {
+ running.set(true);
+ super.start();
+ }
+
+ @VisibleForTesting
+ public void pause() {
+ running.set(false);
+ }
+
+ @VisibleForTesting
+ public void resume() {
+ running.set(true);
+ }
+
+ /**
+ * Checks if rocks-tools native library is available.
+ */
+ private boolean isRocksToolsNativeLibAvailable() {
+ try {
+ return ManagedRawSSTFileReader.tryLoadLibrary();
+ } catch (Exception e) {
+ LOG.warn("Failed to check native code availability", e);
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a snapshot needs defragmentation by examining its YAML metadata.
+ */
+ private boolean needsDefragmentation(SnapshotInfo snapshotInfo) {
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ try {
+ // Read YAML metadata using the correct API
+ File yamlFile = new File(snapshotPath + ".yaml");
+ OmSnapshotLocalDataYaml yamlData = OmSnapshotLocalDataYaml.getFromYamlFile(
+ ozoneManager.getOmSnapshotManager(), yamlFile); // TODO: Verify new usage
+
+ // Check if snapshot needs compaction (defragmentation)
+ boolean needsDefrag = yamlData.getNeedsDefrag();
+ LOG.debug("Snapshot {} needsDefragmentation field value: {}",
+ snapshotInfo.getName(), needsDefrag);
+
+ return needsDefrag;
+ } catch (IOException e) {
+ LOG.warn("Failed to read YAML metadata for snapshot {}, assuming defrag needed",
+ snapshotInfo.getName(), e);
+ return true;
+ }
+ }
+
+ /**
+ * Finds the first active snapshot in the chain that needs defragmentation.
+ */
+ private SnapshotInfo findFirstSnapshotNeedingDefrag(
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for first snapshot needing defragmentation in active chain");
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ // Skip deleted snapshots
+ if (snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED) {
+ LOG.debug("Skipping deleted snapshot: {}", snapshotInfo.getName());
+ continue;
+ }
+
+ // Check if this snapshot needs defragmentation
+ if (needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found snapshot needing defragmentation: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ LOG.debug("Snapshot {} already defragmented, continuing search",
+ snapshotInfo.getName());
+ }
+ }
+
+ LOG.debug("No snapshots found needing defragmentation");
+ return null;
+ }
+
+ /**
+ * Finds the previous defragmented snapshot in the chain.
+ */
+ private SnapshotInfo findPreviousDefraggedSnapshot(SnapshotInfo currentSnapshot,
+ Table<String, SnapshotInfo> snapshotInfoTable) throws IOException {
+
+ LOG.debug("Searching for previous defragmented snapshot before: {}",
+ currentSnapshot.getName());
+
+ // Walk backwards through the snapshot chain using pathPreviousSnapshotId
+ String previousSnapshotId = currentSnapshot.getPathPreviousSnapshotId() != null ?
+ currentSnapshot.getPathPreviousSnapshotId().toString() : null;
+
+ while (previousSnapshotId != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>> iterator =
+ snapshotInfoTable.iterator()) {
+ iterator.seekToFirst();
+
+ while (iterator.hasNext()) {
+ Table.KeyValue<String, SnapshotInfo> keyValue = iterator.next();
+ SnapshotInfo snapshotInfo = keyValue.getValue();
+
+ if (snapshotInfo.getSnapshotId().toString().equals(previousSnapshotId)) {
+ if (!needsDefragmentation(snapshotInfo)) {
+ LOG.info("Found previous defragmented snapshot: {} (ID: {})",
+ snapshotInfo.getName(), snapshotInfo.getSnapshotId());
+ return snapshotInfo;
+ }
+
+ // Continue searching with this snapshot's previous
+ previousSnapshotId = snapshotInfo.getPathPreviousSnapshotId() != null ?
+ snapshotInfo.getPathPreviousSnapshotId().toString() : null;
+ break;
+ }
+ }
+ }
+ }
+
+ LOG.debug("No previous defragmented snapshot found");
+ return null;
+ }
+
+ /**
+ * Performs full defragmentation for the first snapshot in the chain.
+ * This is a simplified implementation that demonstrates the concept.
+ */
+ private void performFullDefragmentation(SnapshotInfo snapshotInfo,
+ OmSnapshot omSnapshot) throws IOException {
+
+ String snapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), snapshotInfo);
+
+ // For defraggedDbPath, we need to go up to the parent directory and use checkpointStateDefragged
+ String parentDir = Paths.get(snapshotPath).getParent().getParent().toString();
+ String checkpointDirName = Paths.get(snapshotPath).getFileName().toString();
+ String defraggedDbPath = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, checkpointDirName).toString();
+
+ LOG.info("Starting full defragmentation for snapshot: {} at path: {}",
+ snapshotInfo.getName(), snapshotPath);
+ LOG.info("Target defragmented DB path: {}", defraggedDbPath);
+
+ // Create defragmented directory
+ Files.createDirectories(Paths.get(defraggedDbPath));
+
+ // TODO: Get snapshot checkpoint DB via SnapshotCache
+ RDBStore originalStore = (RDBStore) omSnapshot.getMetadataManager().getStore();
+ RocksDatabase originalDb = originalStore.getDb();
+
+ LOG.info("Starting defragmentation process for snapshot: {}", snapshotInfo.getName());
+ LOG.info("Original DB path: {}", snapshotPath);
+ LOG.info("Defragmented DB path: {}", defraggedDbPath);
+
+ // Implement actual RocksDB defragmentation
+ try {
+ // 1. Create a new RocksDB instance at defraggedDbPath
+ DBStoreBuilder defraggedDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(checkpointDirName)
+ .setPath(Paths.get(defraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add all the tracked column families to the new DB
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ defraggedDbBuilder.addTable(cfName);
+ LOG.debug("Added column family {} to defragmented DB", cfName);
+ }
+
+ // Build the new defragmented database
+ DBStore defraggedStore = defraggedDbBuilder.build();
+ RocksDatabase defraggedDb = ((RDBStore) defraggedStore).getDb();
+
+ LOG.info("Created new defragmented DB instance for snapshot: {}", snapshotInfo.getName());
+
+ // 2. & 3. Iterate through tracked column families and copy all key-value pairs
+ long totalKeysCopied = 0;
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ LOG.info("Starting defragmentation of column family: {} for snapshot: {}",
+ cfName, snapshotInfo.getName());
+
+ RocksDatabase.ColumnFamily originalCf = originalDb.getColumnFamily(cfName);
+ RocksDatabase.ColumnFamily defraggedCf = defraggedDb.getColumnFamily(cfName);
+
+ if (originalCf == null) {
+ LOG.warn("Column family {} not found in original DB, skipping", cfName);
+ continue;
+ }
+
+ if (defraggedCf == null) {
+ LOG.warn("Column family {} not found in defragmented DB, skipping", cfName);
+ continue;
+ }
+
+ long cfKeysCopied = 0;
+ try (ManagedWriteBatch writeBatch = new ManagedWriteBatch();
+ ManagedRocksIterator iterator = originalDb.newIterator(originalCf)) {
+
+ iterator.get().seekToFirst();
+
+ while (iterator.get().isValid()) {
+ byte[] key = iterator.get().key();
+ byte[] value = iterator.get().value();
+
+ // Add to batch for efficient writing
+ defraggedCf.batchPut(writeBatch, key, value);
+ cfKeysCopied++;
+
+ // Commit batch every 1000 keys to avoid memory issues
+ if (cfKeysCopied % 1000 == 0) {
+ defraggedDb.batchWrite(writeBatch);
+ writeBatch.clear();
+ LOG.debug("Copied {} keys for column family {} so far", cfKeysCopied, cfName);
+ }
+
+ iterator.get().next();
+ }
+
+ // Commit any remaining keys in the batch
+ if (writeBatch.count() > 0) {
+ defraggedDb.batchWrite(writeBatch);
+ }
+
+ totalKeysCopied += cfKeysCopied;
+ LOG.info("Completed copying {} keys for column family: {} in snapshot: {}",
+ cfKeysCopied, cfName, snapshotInfo.getName());
+ }
+ }
+
+ LOG.info("Copied total of {} keys across all column families for snapshot: {}",
+ totalKeysCopied, snapshotInfo.getName());
+
+ // 4. Perform compaction on the new DB to ensure it's fully defragmented
+ LOG.info("Starting compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+ try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+ compactOptions.setChangeLevel(true);
+ compactOptions.setTargetLevel(1);
+ defraggedDb.compactDB(compactOptions);
+ }
+ LOG.info("Completed compaction of defragmented DB for snapshot: {}", snapshotInfo.getName());
+
+ // 5. Verify data integrity between original and defragmented DBs
+ verifyDbIntegrity(originalDb, defraggedDb, snapshotInfo);
+
+ // Close the defragmented DB
+ defraggedStore.close();
+
+ // TODO: Rename om.db to the om.db-UUID (at least for full defrag)
+
+ LOG.info("Successfully completed full defragmentation for snapshot: {} with {} keys copied",
+ snapshotInfo.getName(), totalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during defragmentation of snapshot: {}", snapshotInfo.getName(), e);
+ throw new IOException("Failed to defragment snapshot: " + snapshotInfo.getName(), e);
+ }
+ }
+
+ /**
+ * Performs incremental defragmentation using diff from previous defragmented snapshot.
+ */
+ private void performIncrementalDefragmentation(SnapshotInfo currentSnapshot,
+ SnapshotInfo previousDefraggedSnapshot, OmSnapshot currentOmSnapshot)
+ throws IOException {
+
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String previousSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), previousDefraggedSnapshot);
+
+ // Fix path construction similar to performFullDefragmentation
+ String previousParentDir = Paths.get(previousSnapshotPath).getParent().getParent().toString();
+ String previousCheckpointDirName = Paths.get(previousSnapshotPath).getFileName().toString();
+ String previousDefraggedDbPath = Paths.get(previousParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ previousCheckpointDirName).toString();
+
+ String currentParentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String currentCheckpointDirName = Paths.get(currentSnapshotPath).getFileName().toString();
+ String currentDefraggedDbPath = Paths.get(currentParentDir, CHECKPOINT_STATE_DEFRAGGED_DIR,
+ currentCheckpointDirName).toString();
+
+ LOG.info("Starting incremental defragmentation for snapshot: {} using previous: {}",
+ currentSnapshot.getName(), previousDefraggedSnapshot.getName());
+ LOG.info("Previous defragmented DB: {}", previousDefraggedDbPath);
+ LOG.info("Current target DB: {}", currentDefraggedDbPath);
+
+ // Note: Don't create target directory - RocksDB createCheckpoint() will create it
+ // and will fail with "Directory exists" if we create it first
+
+ try {
+ // Check if previous defragmented DB exists
+ if (!Files.exists(Paths.get(previousDefraggedDbPath))) {
+ LOG.warn("Previous defragmented DB not found at {}, falling back to full defragmentation",
+ previousDefraggedDbPath);
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ return;
+ }
+
+ // 1. Create a checkpoint from the previous defragmented DB directly at target location
+ LOG.info("Creating checkpoint from previous defragmented DB directly to target location");
+
+ // Open the previous defragmented DB to create checkpoint.
+ // TODO: via SnapshotCache or something equivalent for lock protection
+ DBStoreBuilder previousDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(previousCheckpointDirName)
+ .setPath(Paths.get(previousDefraggedDbPath).getParent())
+ .setOpenReadOnly(true)
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ previousDbBuilder.addTable(cfName);
+ }
+
+ try (DBStore previousDefraggedStore = previousDbBuilder.build()) {
+ RocksDatabase previousDefraggedDb = ((RDBStore) previousDefraggedStore).getDb();
+
+ // Create checkpoint directly at the target location
+ try (RocksDatabase.RocksCheckpoint checkpoint = previousDefraggedDb.createCheckpoint()) {
+ checkpoint.createCheckpoint(Paths.get(currentDefraggedDbPath));
+ LOG.info("Created checkpoint directly at target: {}", currentDefraggedDbPath);
+ }
+ }
+
+ // 2. Open the checkpoint as our working defragmented DB and apply incremental changes
+ DBStoreBuilder currentDbBuilder = DBStoreBuilder.newBuilder(ozoneManager.getConfiguration())
+ .setName(currentCheckpointDirName)
+ .setPath(Paths.get(currentDefraggedDbPath).getParent())
+ .setCreateCheckpointDirs(false);
+
+ // Add tracked column families
+ for (String cfName : COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT) {
+ currentDbBuilder.addTable(cfName);
+ }
+
+ // Build DB from the checkpoint
+ DBStore currentDefraggedStore = currentDbBuilder.build();
+ RocksDatabase currentDefraggedDb = ((RDBStore) currentDefraggedStore).getDb();
+
+ LOG.info("Opened checkpoint as working defragmented DB for incremental update");
+
+ // 3. Apply incremental changes from current snapshot
+ RDBStore currentSnapshotStore = (RDBStore) currentOmSnapshot.getMetadataManager().getStore();
+ RocksDatabase currentSnapshotDb = currentSnapshotStore.getDb();
+
+ long incrementalKeysCopied = applyIncrementalChanges(currentSnapshotDb, currentDefraggedStore,
+ currentSnapshot, previousDefraggedSnapshot);
+
+ LOG.info("Applied {} incremental changes for snapshot: {}",
+ incrementalKeysCopied, currentSnapshot.getName());
+
+ // 4. Perform compaction on the updated DB
+// LOG.info("Starting compaction of incrementally defragmented DB for snapshot: {}",
+// currentSnapshot.getName());
+// try (ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions()) {
+// compactOptions.setChangeLevel(true);
+// compactOptions.setTargetLevel(1);
+// currentDefraggedDb.compactDB(compactOptions);
+// }
+// LOG.info("Completed compaction of incrementally defragmented DB");
+
+ // 5. Verify data integrity
+ verifyDbIntegrity(currentSnapshotDb, currentDefraggedDb, currentSnapshot);
+
+ // Close the defragmented DB. TODO: Close in finally block instead
+ currentDefraggedStore.close();
+
+ LOG.info("Successfully completed incremental defragmentation for snapshot: {} with {} incremental changes",
+ currentSnapshot.getName(), incrementalKeysCopied);
+
+ } catch (RocksDatabaseException e) {
+ LOG.error("RocksDB error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+// LOG.warn("Falling back to full defragmentation due to error");
+// performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ } catch (Exception e) {
+ LOG.error("Unexpected error during incremental defragmentation of snapshot: {}",
+ currentSnapshot.getName(), e);
+ LOG.warn("Falling back to full defragmentation due to error");
+ performFullDefragmentation(currentSnapshot, currentOmSnapshot);
+ }
+ }
+
+ /**
+ * Applies incremental changes by using snapshotDiff to compute the diff list,
+ * then iterating that diff list against the current snapshot checkpoint DB.
+ * Uses RDBSstFileWriter to directly write changes to SST files and then ingests them.
+ */
+ @SuppressWarnings("checkstyle:MethodLength")
+ private long applyIncrementalChanges(RocksDatabase currentSnapshotDb, DBStore targetStore,
+ SnapshotInfo currentSnapshot, SnapshotInfo previousSnapshot) throws RocksDatabaseException {
+
+ LOG.info("Applying incremental changes for snapshot: {} since: {} using snapshotDiff approach",
+ currentSnapshot.getName(), previousSnapshot.getName());
+
+ long totalChanges = 0;
+
+ // Create temporary directory for SST files
+ String currentSnapshotPath = OmSnapshotManager.getSnapshotPath(
+ ozoneManager.getConfiguration(), currentSnapshot);
+ String parentDir = Paths.get(currentSnapshotPath).getParent().getParent().toString();
+ String tempSstDir = Paths.get(parentDir, CHECKPOINT_STATE_DEFRAGGED_DIR, TEMP_DIFF_DIR).toString();
+
+ try {
+ Files.createDirectories(Paths.get(tempSstDir));
+ LOG.info("Created temporary SST directory: {}", tempSstDir);
+
+ // Use snapshotDiff to compute the diff list between previous and current snapshot
+ LOG.info("Computing snapshot diff between {} and {}",
+ previousSnapshot.getName(), currentSnapshot.getName());
+
+ SnapshotDiffResponse diffResponse;
+ try {
+ // Call snapshotDiff to get the diff list
+ diffResponse = ozoneManager.snapshotDiff(
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org