You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/07/15 03:53:12 UTC
(hbase) branch branch-3 updated (238160fc1a4 -> f553ed353c4)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
from 238160fc1a4 HBASE-28723 [JDK17] TestSecureIPC fails under JDK17 (#6071)
new bee40d047ca HBASE-28704 The expired snapshot can be read by CopyTable or ExportSnapshot (#6047)
new f553ed353c4 HBASE-28727 SteppingSplitPolicy may not work when table enables region replication (#6077)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hbase/snapshot/ExportSnapshot.java | 48 ++++++-----
.../hadoop/hbase/mapreduce/TestCopyTable.java | 42 ++++++++++
.../hadoop/hbase/snapshot/TestExportSnapshot.java | 96 +++++++++++++++++-----
.../IncreasingToUpperBoundRegionSplitPolicy.java | 6 +-
.../hbase/snapshot/RestoreSnapshotHelper.java | 7 ++
.../hbase/regionserver/TestRegionSplitPolicy.java | 40 +++++++++
.../hbase/snapshot/TestRestoreSnapshotHelper.java | 41 +++++++++
7 files changed, 239 insertions(+), 41 deletions(-)
(hbase) 02/02: HBASE-28727 SteppingSplitPolicy may not work when table enables region replication (#6077)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f553ed353c4b1ae01fae67ef397656e1667ecb7f
Author: lupeng <lu...@qq.com>
AuthorDate: Mon Jul 15 11:45:49 2024 +0800
HBASE-28727 SteppingSplitPolicy may not work when table enables region replication (#6077)
Signed-off-by: Duo Zhang <zh...@apache.org>
(cherry picked from commit 386ecea0a9ceb2f7f455be4f1184e156ecac6431)
---
.../IncreasingToUpperBoundRegionSplitPolicy.java | 6 +++-
.../hbase/regionserver/TestRegionSplitPolicy.java | 40 ++++++++++++++++++++++
2 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
index 7a0393fc0ae..0386bf8245e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.yetus.audience.InterfaceAudience;
@@ -97,7 +98,10 @@ public class IncreasingToUpperBoundRegionSplitPolicy extends ConstantSizeRegionS
int tableRegionsCount = 0;
try {
List<? extends Region> hri = rss.getRegions(tablename);
- tableRegionsCount = hri == null || hri.isEmpty() ? 0 : hri.size();
+ if (hri != null && !hri.isEmpty()) {
+ tableRegionsCount = (int) hri.stream()
+ .filter(r -> r.getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID).count();
+ }
} catch (IOException e) {
LOG.debug("Failed getOnlineRegions " + tablename, e);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
index 0c14c9aa812..404935d3e94 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java
@@ -154,6 +154,46 @@ public class TestRegionSplitPolicy {
assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
}
+ @Test
+ public void testSteppingSplitPolicyWithRegionReplication() throws IOException {
+ conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, SteppingSplitPolicy.class.getName());
+
+ RegionServerServices rss = mock(RegionServerServices.class);
+ doReturn(rss).when(mockRegion).getRegionServerServices();
+
+ long maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE;
+ TableDescriptor td =
+ TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(maxFileSize).build();
+ doReturn(td).when(mockRegion).getTableDescriptor();
+ assertEquals(td.getMaxFileSize(), maxFileSize);
+
+ List<HStore> storefiles = new ArrayList<>();
+ HStore mockStore = mock(HStore.class);
+ long flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
+ TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE);
+ long exceedSize = flushSize * 2 + 1;
+ doReturn(exceedSize).when(mockStore).getSize();
+ doReturn(true).when(mockStore).canSplit();
+ storefiles.add(mockStore);
+ doReturn(storefiles).when(mockRegion).getStores();
+
+ List<HRegion> regions = new ArrayList<>();
+ HRegion r1 = mock(HRegion.class);
+ RegionInfo regionInfo1 = mock(RegionInfo.class);
+ doReturn(regionInfo1).when(r1).getRegionInfo();
+ doReturn(RegionInfo.DEFAULT_REPLICA_ID).when(regionInfo1).getReplicaId();
+ HRegion r2 = mock(HRegion.class);
+ RegionInfo regionInfo2 = mock(RegionInfo.class);
+ doReturn(regionInfo2).when(r2).getRegionInfo();
+ doReturn(1).when(regionInfo2).getReplicaId();
+ regions.add(r1);
+ regions.add(r2);
+ doReturn(regions).when(rss).getRegions(td.getTableName());
+
+ SteppingSplitPolicy policy = (SteppingSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
+ assertTrue(policy.shouldSplit());
+ }
+
@Test
public void testIsExceedSize() throws IOException {
// Configure SteppingAllStoresSizeSplitPolicy as our split policy
(hbase) 01/02: HBASE-28704 The expired snapshot can be read by CopyTable or ExportSnapshot (#6047)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit bee40d047cacdc108ebf4dd717c2784337ca233b
Author: lupeng <lu...@qq.com>
AuthorDate: Mon Jul 15 11:39:07 2024 +0800
HBASE-28704 The expired snapshot can be read by CopyTable or ExportSnapshot (#6047)
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Liangjun He <he...@apache.org>
(cherry picked from commit e9ea1b83796d9cfee524c20c25645291e8cde8d5)
---
.../hadoop/hbase/snapshot/ExportSnapshot.java | 48 ++++++-----
.../hadoop/hbase/mapreduce/TestCopyTable.java | 42 ++++++++++
.../hadoop/hbase/snapshot/TestExportSnapshot.java | 96 +++++++++++++++++-----
.../hbase/snapshot/RestoreSnapshotHelper.java | 7 ++
.../hbase/snapshot/TestRestoreSnapshotHelper.java | 41 +++++++++
5 files changed, 194 insertions(+), 40 deletions(-)
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index 4e0c54b718b..d10ff7f9b3e 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -79,6 +79,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.hbase.thirdparty.org.apache.commons.cli.Option;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
@@ -138,9 +139,9 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
static final Option NO_CHECKSUM_VERIFY = new Option(null, "no-checksum-verify", false,
"Do not verify checksum, use name+length only.");
static final Option NO_TARGET_VERIFY = new Option(null, "no-target-verify", false,
- "Do not verify the integrity of the exported snapshot.");
- static final Option NO_SOURCE_VERIFY =
- new Option(null, "no-source-verify", false, "Do not verify the source of the snapshot.");
+ "Do not verify the exported snapshot's expiration status and integrity.");
+ static final Option NO_SOURCE_VERIFY = new Option(null, "no-source-verify", false,
+ "Do not verify the source snapshot's expiration status and integrity.");
static final Option OVERWRITE =
new Option(null, "overwrite", false, "Rewrite the snapshot manifest if already exists.");
static final Option CHUSER =
@@ -936,13 +937,17 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
}
}
- private void verifySnapshot(final Configuration baseConf, final FileSystem fs, final Path rootDir,
- final Path snapshotDir) throws IOException {
+ private void verifySnapshot(final SnapshotDescription snapshotDesc, final Configuration baseConf,
+ final FileSystem fs, final Path rootDir, final Path snapshotDir) throws IOException {
// Update the conf with the current root dir, since may be a different cluster
Configuration conf = new Configuration(baseConf);
CommonFSUtils.setRootDir(conf, rootDir);
CommonFSUtils.setFsDefault(conf, CommonFSUtils.getRootDir(conf));
- SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ boolean isExpired = SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
+ snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime());
+ if (isExpired) {
+ throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(snapshotDesc));
+ }
SnapshotReferenceUtil.verifySnapshot(conf, fs, snapshotDir, snapshotDesc);
}
@@ -1044,14 +1049,14 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
if (snapshotName == null) {
System.err.println("Snapshot name not provided.");
LOG.error("Use -h or --help for usage instructions.");
- return 0;
+ return EXIT_FAILURE;
}
if (outputRoot == null) {
System.err
.println("Destination file-system (--" + Options.COPY_TO.getLongOpt() + ") not provided.");
LOG.error("Use -h or --help for usage instructions.");
- return 0;
+ return EXIT_FAILURE;
}
if (targetName == null) {
@@ -1079,11 +1084,14 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
LOG.debug("outputFs={}, outputRoot={}, skipTmp={}, initialOutputSnapshotDir={}", outputFs,
outputRoot.toString(), skipTmp, initialOutputSnapshotDir);
+ // throw CorruptedSnapshotException if we can't read the snapshot info.
+ SnapshotDescription sourceSnapshotDesc =
+ SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir);
+
// Verify snapshot source before copying files
if (verifySource) {
- LOG.info("Verify snapshot source, inputFs={}, inputRoot={}, snapshotDir={}.",
- inputFs.getUri(), inputRoot, snapshotDir);
- verifySnapshot(srcConf, inputFs, inputRoot, snapshotDir);
+ LOG.info("Verify the source snapshot's expiration status and integrity.");
+ verifySnapshot(sourceSnapshotDesc, srcConf, inputFs, inputRoot, snapshotDir);
}
// Find the necessary directory which need to change owner and group
@@ -1104,12 +1112,12 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
if (overwrite) {
if (!outputFs.delete(outputSnapshotDir, true)) {
System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
- return 1;
+ return EXIT_FAILURE;
}
} else {
System.err.println("The snapshot '" + targetName + "' already exists in the destination: "
+ outputSnapshotDir);
- return 1;
+ return EXIT_FAILURE;
}
}
@@ -1120,7 +1128,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
if (!outputFs.delete(snapshotTmpDir, true)) {
System.err
.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
- return 1;
+ return EXIT_FAILURE;
}
} else {
System.err
@@ -1129,7 +1137,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
System.err
.println("consider removing " + snapshotTmpDir + " by using the -overwrite option");
- return 1;
+ return EXIT_FAILURE;
}
}
}
@@ -1208,19 +1216,21 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
// Step 4 - Verify snapshot integrity
if (verifyTarget) {
- LOG.info("Verify snapshot integrity");
- verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir);
+ LOG.info("Verify the exported snapshot's expiration status and integrity.");
+ SnapshotDescription targetSnapshotDesc =
+ SnapshotDescriptionUtils.readSnapshotInfo(outputFs, outputSnapshotDir);
+ verifySnapshot(targetSnapshotDesc, destConf, outputFs, outputRoot, outputSnapshotDir);
}
LOG.info("Export Completed: " + targetName);
- return 0;
+ return EXIT_SUCCESS;
} catch (Exception e) {
LOG.error("Snapshot export failed", e);
if (!skipTmp) {
outputFs.delete(snapshotTmpDir, true);
}
outputFs.delete(outputSnapshotDir, true);
- return 1;
+ return EXIT_FAILURE;
}
}
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
index 11e377b199f..5c3e9b65079 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
@@ -19,23 +19,31 @@ package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MapReduceTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.LauncherSecurityManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -210,6 +218,40 @@ public class TestCopyTable extends CopyTableTestBase {
testCopyTableBySnapshot("testLoadingSnapshotToTable", false, false);
}
+ @Test
+ public void testLoadingTtlExpiredSnapshotToTable() throws Exception {
+ String tablePrefix = "testLoadingExpiredSnapshotToTable";
+ TableName table1 = TableName.valueOf(tablePrefix + 1);
+ TableName table2 = TableName.valueOf(tablePrefix + 2);
+ Table t1 = createTable(table1, FAMILY_A, false);
+ createTable(table2, FAMILY_A, false);
+ loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
+ String snapshot = tablePrefix + "_snapshot";
+ Map<String, Object> properties = new HashMap<>();
+ properties.put("TTL", 10);
+ SnapshotDescription snapshotDescription = new SnapshotDescription(snapshot, table1,
+ SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
+ TEST_UTIL.getAdmin().snapshot(snapshotDescription);
+ boolean isExist =
+ TEST_UTIL.getAdmin().listSnapshots().stream().anyMatch(ele -> snapshot.equals(ele.getName()));
+ assertTrue(isExist);
+ int retry = 6;
+ while (
+ !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
+ snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
+ ) {
+ retry--;
+ Thread.sleep(10 * 1000);
+ }
+ boolean isExpiredSnapshot =
+ SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
+ snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime());
+ assertTrue(isExpiredSnapshot);
+ String[] args = new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot };
+ assertThrows(SnapshotTTLExpiredException.class,
+ () -> runCopy(TEST_UTIL.getConfiguration(), args));
+ }
+
@Test
public void tsetLoadingSnapshotToMobTable() throws Exception {
testCopyTableBySnapshot("testLoadingSnapshotToMobTable", false, true);
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index 813da956799..133737bb397 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -44,12 +44,14 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
+import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -281,6 +283,39 @@ public class TestExportSnapshot {
}
}
+ @Test
+ public void testExportExpiredSnapshot() throws Exception {
+ String name = "testExportExpiredSnapshot";
+ TableName tableName = TableName.valueOf(name);
+ String snapshotName = "snapshot-" + name;
+ createTable(tableName);
+ SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
+ Map<String, Object> properties = new HashMap<>();
+ properties.put("TTL", 10);
+ org.apache.hadoop.hbase.client.SnapshotDescription snapshotDescription =
+ new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
+ SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
+ admin.snapshot(snapshotDescription);
+ boolean isExist =
+ admin.listSnapshots().stream().anyMatch(ele -> snapshotName.equals(ele.getName()));
+ assertTrue(isExist);
+ int retry = 6;
+ while (
+ !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
+ snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
+ ) {
+ retry--;
+ Thread.sleep(10 * 1000);
+ }
+ boolean isExpiredSnapshot =
+ SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
+ snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime());
+ assertTrue(isExpiredSnapshot);
+ int res = runExportSnapshot(TEST_UTIL.getConfiguration(), snapshotName, snapshotName,
+ TEST_UTIL.getDefaultRootDirPath(), getHdfsDestinationDir(), false, false, false, true, true);
+ assertTrue(res == AbstractHBaseTool.EXIT_FAILURE);
+ }
+
private void testExportFileSystemState(final TableName tableName, final String snapshotName,
final String targetName, int filesExpected) throws Exception {
testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
@@ -312,29 +347,10 @@ public class TestExportSnapshot {
FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
FileSystem srcFs = srcDir.getFileSystem(conf);
Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
- LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir,
- rawTgtDir, srcFs.getUri(), srcDir);
- List<String> opts = new ArrayList<>();
- opts.add("--snapshot");
- opts.add(snapshotName);
- opts.add("--copy-to");
- opts.add(tgtDir.toString());
- if (!targetName.equals(snapshotName)) {
- opts.add("--target");
- opts.add(targetName);
- }
- if (overwrite) {
- opts.add("--overwrite");
- }
- if (resetTtl) {
- opts.add("--reset-ttl");
- }
- if (!checksumVerify) {
- opts.add("--no-checksum-verify");
- }
// Export Snapshot
- int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
+ int res = runExportSnapshot(conf, snapshotName, targetName, srcDir, rawTgtDir, overwrite,
+ resetTtl, checksumVerify, true, true);
assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res);
if (!success) {
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
@@ -467,4 +483,42 @@ public class TestExportSnapshot {
FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
fs.delete(path, true);
}
+
+ private static int runExportSnapshot(final Configuration conf, final String sourceSnapshotName,
+ final String targetSnapshotName, final Path srcDir, Path rawTgtDir, final boolean overwrite,
+ final boolean resetTtl, final boolean checksumVerify, final boolean noSourceVerify,
+ final boolean noTargetVerify) throws Exception {
+ FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
+ FileSystem srcFs = srcDir.getFileSystem(conf);
+ Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
+ LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir,
+ rawTgtDir, srcFs.getUri(), srcDir);
+ List<String> opts = new ArrayList<>();
+ opts.add("--snapshot");
+ opts.add(sourceSnapshotName);
+ opts.add("--copy-to");
+ opts.add(tgtDir.toString());
+ if (!targetSnapshotName.equals(sourceSnapshotName)) {
+ opts.add("--target");
+ opts.add(targetSnapshotName);
+ }
+ if (overwrite) {
+ opts.add("--overwrite");
+ }
+ if (resetTtl) {
+ opts.add("--reset-ttl");
+ }
+ if (!checksumVerify) {
+ opts.add("--no-checksum-verify");
+ }
+ if (!noSourceVerify) {
+ opts.add("--no-source-verify");
+ }
+ if (!noTargetVerify) {
+ opts.add("--no-target-verify");
+ }
+
+ // Export Snapshot
+ return run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 8395456cd76..377cfda03a7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
import org.apache.hadoop.hbase.security.access.TablePermission;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -877,6 +878,12 @@ public class RestoreSnapshotHelper {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ // check if the snapshot is expired.
+ boolean isExpired = SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
+ snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime());
+ if (isExpired) {
+ throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(snapshotDesc));
+ }
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
MonitoredTask status = TaskMonitor.get()
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
index c18c07fc12e..2b8fa85119b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
@@ -18,10 +18,13 @@
package org.apache.hadoop.hbase.snapshot;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -32,6 +35,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
@@ -45,6 +49,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.wal.WALSplitUtil;
import org.junit.After;
@@ -177,6 +182,42 @@ public class TestRestoreSnapshotHelper {
}
}
+ @Test
+ public void testCopyExpiredSnapshotForScanner() throws IOException, InterruptedException {
+ rootDir = TEST_UTIL.getDefaultRootDirPath();
+ CommonFSUtils.setRootDir(conf, rootDir);
+ TableName tableName = TableName.valueOf("testCopyExpiredSnapshotForScanner");
+ String snapshotName = tableName.getNameAsString() + "-snapshot";
+ Path restoreDir = new Path("/hbase/.tmp-expired-snapshot/copySnapshotDest");
+ // create table and put some data into the table
+ byte[] columnFamily = Bytes.toBytes("A");
+ Table table = TEST_UTIL.createTable(tableName, columnFamily);
+ TEST_UTIL.loadTable(table, columnFamily);
+ // create snapshot with ttl = 10 sec
+ Map<String, Object> properties = new HashMap<>();
+ properties.put("TTL", 10);
+ org.apache.hadoop.hbase.client.SnapshotDescription snapshotDesc =
+ new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
+ SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
+ TEST_UTIL.getAdmin().snapshot(snapshotDesc);
+ boolean isExist = TEST_UTIL.getAdmin().listSnapshots().stream()
+ .anyMatch(ele -> snapshotName.equals(ele.getName()));
+ assertTrue(isExist);
+ int retry = 6;
+ while (
+ !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
+ snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
+ ) {
+ retry--;
+ Thread.sleep(10 * 1000);
+ }
+ boolean isExpiredSnapshot = SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
+ snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime());
+ assertTrue(isExpiredSnapshot);
+ assertThrows(SnapshotTTLExpiredException.class, () -> RestoreSnapshotHelper
+ .copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName));
+ }
+
protected void createTableAndSnapshot(TableName tableName, String snapshotName)
throws IOException {
byte[] column = Bytes.toBytes("A");