You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2021/11/28 14:24:07 UTC
[hbase] 01/02: HBASE-26485 Introduce a method to clean restore directory after Snapshot Scan (#3877)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4fb3e304e24a728a67d2e277664a6773631f58d8
Author: Ruanhui <32...@users.noreply.github.com>
AuthorDate: Sat Nov 27 20:35:24 2021 +0800
HBASE-26485 Introduce a method to clean restore directory after Snapshot Scan (#3877)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../hbase/mapreduce/TableSnapshotInputFormat.java | 10 ++++++++++
.../mapreduce/TableSnapshotInputFormatImpl.java | 21 +++++++++++++++++++++
.../mapreduce/TestTableSnapshotInputFormat.java | 18 ++++++++++++++++++
3 files changed, 49 insertions(+)
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
index 24cbfcc..6fd0b6e 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
@@ -235,4 +235,14 @@ public class TableSnapshotInputFormat extends InputFormat<ImmutableBytesWritable
TableSnapshotInputFormatImpl.setInput(job.getConfiguration(), snapshotName, restoreDir,
splitAlgo, numSplitsPerRegion);
}
+
+ /**
+ * clean restore directory after snapshot scan job
+ * @param job the snapshot scan job
+ * @param snapshotName the name of the snapshot to read from
+ * @throws IOException if an error occurs
+ */
+ public static void cleanRestoreDir(Job job, String snapshotName) throws IOException {
+ TableSnapshotInputFormatImpl.cleanRestoreDir(job, snapshotName);
+ }
}
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
index f83a9b9..e106b9d 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.RegionSplitter;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -622,4 +623,24 @@ public class TableSnapshotInputFormatImpl {
RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
conf.set(RESTORE_DIR_KEY, restoreDir.toString());
}
+
+ /**
+ * clean restore directory after snapshot scan job
+ * @param job the snapshot scan job
+ * @param snapshotName the name of the snapshot to read from
+ * @throws IOException if an error occurs
+ */
+ public static void cleanRestoreDir(Job job, String snapshotName) throws IOException {
+ Configuration conf = job.getConfiguration();
+ Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY));
+ FileSystem fs = restoreDir.getFileSystem(conf);
+ if (!fs.exists(restoreDir)) {
+ LOG.warn("{} doesn't exist on file system, maybe it's already been cleaned", restoreDir);
+ return;
+ }
+ if (!fs.delete(restoreDir, true)) {
+ LOG.warn("Failed clean restore dir {} for snapshot {}", restoreDir, snapshotName);
+ }
+ LOG.debug("Clean restore directory {} for {}", restoreDir, snapshotName);
+ }
}
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
index b1a07f0..188fc1f 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
@@ -575,4 +575,22 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
public void testWithMapReduceMultipleMappersPerRegion() throws Exception {
testWithMapReduce(UTIL, "testWithMapReduceMultiRegion", 10, 5, 50, false);
}
+
+ @Test
+ public void testCleanRestoreDir() throws Exception {
+ TableName tableName = TableName.valueOf("test_table");
+ String snapshotName = "test_snapshot";
+ createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
+ Job job = Job.getInstance(UTIL.getConfiguration());
+ Path workingDir = UTIL.getDataTestDirOnTestFS(snapshotName);
+ TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
+ new Scan(), TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
+ NullWritable.class, job, false, workingDir);
+ FileSystem fs = workingDir.getFileSystem(job.getConfiguration());
+ Path restorePath = new Path(job.getConfiguration()
+ .get("hbase.TableSnapshotInputFormat.restore.dir"));
+ Assert.assertTrue(fs.exists(restorePath));
+ TableSnapshotInputFormat.cleanRestoreDir(job, snapshotName);
+ Assert.assertFalse(fs.exists(restorePath));
+ }
}