You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by su...@apache.org on 2022/06/21 02:24:51 UTC
[hbase] branch branch-2.5 updated: HBASE-26956 ExportSnapshot tool supports removing TTL (#4538)
This is an automated email from the ASF dual-hosted git repository.
sunxin pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 135348192c2 HBASE-26956 ExportSnapshot tool supports removing TTL (#4538)
135348192c2 is described below
commit 135348192c25e98597ce84fc380b058ee5a23342
Author: XinSun <dd...@gmail.com>
AuthorDate: Tue Jun 21 09:19:08 2022 +0800
HBASE-26956 ExportSnapshot tool supports removing TTL (#4538)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../hadoop/hbase/snapshot/ExportSnapshot.java | 23 +++++++--
.../hadoop/hbase/snapshot/TestExportSnapshot.java | 57 ++++++++++++++++++++--
.../hbase/snapshot/TestExportSnapshotAdjunct.java | 4 +-
.../snapshot/TestExportSnapshotV1NoCluster.java | 2 +-
4 files changed, 74 insertions(+), 12 deletions(-)
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index d37202b0a50..e3a7b805121 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -152,6 +152,8 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
"Number of mappers to use during the copy (mapreduce.job.maps).");
static final Option BANDWIDTH =
new Option(null, "bandwidth", true, "Limit bandwidth to this value in MB/second.");
+ static final Option RESET_TTL =
+ new Option(null, "reset-ttl", false, "Do not copy TTL for the snapshot");
}
// Export Map-Reduce Counters, to keep track of the progress
@@ -917,6 +919,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
private int bandwidthMB = Integer.MAX_VALUE;
private int filesMode = 0;
private int mappers = 0;
+ private boolean resetTtl = false;
@Override
protected void processOptions(CommandLine cmd) {
@@ -938,6 +941,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt());
verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt());
verifySource = !cmd.hasOption(Options.NO_SOURCE_VERIFY.getLongOpt());
+ resetTtl = cmd.hasOption(Options.RESET_TTL.getLongOpt());
}
/**
@@ -1075,11 +1079,19 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
}
}
- // Write a new .snapshotinfo if the target name is different from the source name
- if (!targetName.equals(snapshotName)) {
- SnapshotDescription snapshotDesc = SnapshotDescriptionUtils
- .readSnapshotInfo(inputFs, snapshotDir).toBuilder().setName(targetName).build();
- SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs);
+ // Write a new .snapshotinfo if the target name is different from the source name or we want to
+ // reset TTL for target snapshot.
+ if (!targetName.equals(snapshotName) || resetTtl) {
+ SnapshotDescription.Builder snapshotDescBuilder =
+ SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir).toBuilder();
+ if (!targetName.equals(snapshotName)) {
+ snapshotDescBuilder.setName(targetName);
+ }
+ if (resetTtl) {
+ snapshotDescBuilder.setTtl(HConstants.DEFAULT_SNAPSHOT_TTL);
+ }
+ SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDescBuilder.build(),
+ initialOutputSnapshotDir, outputFs);
if (filesUser != null || filesGroup != null) {
outputFs.setOwner(
new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser,
@@ -1155,6 +1167,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
addOption(Options.CHMOD);
addOption(Options.MAPPERS);
addOption(Options.BANDWIDTH);
+ addOption(Options.RESET_TTL);
}
public static void main(String[] args) {
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index 4a1135b1b10..a9bd94f713b 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -24,9 +24,12 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -188,6 +191,38 @@ public class TestExportSnapshot {
testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
}
+ @Test
+ public void testExportWithResetTtl() throws Exception {
+ String name = "testExportWithResetTtl";
+ TableName tableName = TableName.valueOf(name);
+ String snapshotNameStr = "snaptb-" + name;
+ byte[] snapshotName = Bytes.toBytes(snapshotNameStr);
+ Long ttl = 100000L;
+
+ try {
+ // create Table
+ createTable(tableName);
+ SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
+ int tableNumFiles = admin.getRegions(tableName).size();
+ // take a snapshot with TTL
+ Map<String, Object> props = new HashMap<>();
+ props.put("TTL", ttl);
+ admin.snapshot(snapshotNameStr, tableName, props);
+ Optional<Long> ttlOpt =
+ admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotNameStr))
+ .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny();
+ assertTrue(ttlOpt.isPresent());
+ assertEquals(ttl, ttlOpt.get());
+
+ testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles,
+ getHdfsDestinationDir(), false, true);
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ TEST_UTIL.deleteTable(tableName);
+ }
+ }
+
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
final byte[] targetName, int filesExpected) throws Exception {
testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
@@ -196,8 +231,15 @@ public class TestExportSnapshot {
protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception {
+ testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
+ overwrite, false);
+ }
+
+ protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
+ final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl)
+ throws Exception {
testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
- filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite,
+ filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
getBypassRegionPredicate(), true);
}
@@ -207,7 +249,8 @@ public class TestExportSnapshot {
protected static void testExportFileSystemState(final Configuration conf,
final TableName tableName, final byte[] snapshotName, final byte[] targetName,
final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
- final RegionPredicate bypassregionPredicate, boolean success) throws Exception {
+ final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success)
+ throws Exception {
FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
FileSystem srcFs = srcDir.getFileSystem(conf);
Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
@@ -225,6 +268,9 @@ public class TestExportSnapshot {
if (overwrite) {
opts.add("--overwrite");
}
+ if (resetTtl) {
+ opts.add("--reset-ttl");
+ }
// Export Snapshot
int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
@@ -253,7 +299,7 @@ public class TestExportSnapshot {
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
Set<String> snapshotFiles = verifySnapshot(conf, tgtFs, tgtDir, tableName,
- Bytes.toString(targetName), bypassregionPredicate);
+ Bytes.toString(targetName), resetTtl, bypassregionPredicate);
assertEquals(filesExpected, snapshotFiles.size());
}
@@ -270,7 +316,7 @@ public class TestExportSnapshot {
*/
protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
final Path rootDir, final TableName tableName, final String snapshotName,
- final RegionPredicate bypassregionPredicate) throws IOException {
+ final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
final Path exportedSnapshot =
new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
final Set<String> snapshotFiles = new HashSet<>();
@@ -303,6 +349,9 @@ public class TestExportSnapshot {
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
assertTrue(desc.getName().equals(snapshotName));
assertTrue(desc.getTable().equals(tableName.getNameAsString()));
+ if (resetTtl) {
+ assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
+ }
return snapshotFiles;
}
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
index 55fdbfeec64..c97b6959103 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
@@ -153,7 +153,7 @@ public class TestExportSnapshotAdjunct {
conf.setInt("mapreduce.map.maxattempts", 3);
TestExportSnapshot.testExportFileSystemState(conf, tableName, Bytes.toBytes(snapshotName),
Bytes.toBytes(snapshotName), tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true,
- null, true);
+ false, null, true);
}
/**
@@ -170,6 +170,6 @@ public class TestExportSnapshotAdjunct {
conf.setInt("mapreduce.map.maxattempts", 3);
TestExportSnapshot.testExportFileSystemState(conf, tableName, Bytes.toBytes(snapshotName),
Bytes.toBytes(snapshotName), tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true,
- null, false);
+ false, null, false);
}
}
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java
index 60fb387f8cc..7d27bfea79a 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java
@@ -109,7 +109,7 @@ public class TestExportSnapshotV1NoCluster {
TableName tableName = builder.getTableDescriptor().getTableName();
TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), tableName,
snapshotName, snapshotName, snapshotFilesCount, testDir,
- getDestinationDir(fs, testUtil, testDir), false, null, true);
+ getDestinationDir(fs, testUtil, testDir), false, false, null, true);
}
static Path getDestinationDir(FileSystem fs, HBaseCommonTestingUtility hctu, Path testDir)