You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zy...@apache.org on 2019/05/07 00:18:49 UTC
[hbase] branch branch-2 updated: HBASE-21070 Add Test for
SnapshotFileCache for HBase backed by S3 (#209)
This is an automated email from the ASF dual-hosted git repository.
zyork pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 3cc8b36 HBASE-21070 Add Test for SnapshotFileCache for HBase backed by S3 (#209)
3cc8b36 is described below
commit 3cc8b36fd3e194086516a1f9b5814707ede46ade
Author: z-york <zy...@amazon.com>
AuthorDate: Mon May 6 11:52:08 2019 -0700
HBASE-21070 Add Test for SnapshotFileCache for HBase backed by S3 (#209)
SnapshotFileCache depends on getting the last modified time of the
snapshot directory, however, S3 FileSystem's do not update the
last modified time of the top 'folder' when objects are added/removed.
This commit adds a test for the previously fixed SnapshotFileCache.
---
.../master/snapshot/TestSnapshotFileCache.java | 64 ++++++++++++++--------
1 file changed, 41 insertions(+), 23 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
index 800def6..cb53cf7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
@@ -58,14 +58,18 @@ public class TestSnapshotFileCache {
private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ // don't refresh the cache unless we tell it to
+ private static final long PERIOD = Long.MAX_VALUE;
private static FileSystem fs;
private static Path rootDir;
+ private static Path snapshotDir;
@BeforeClass
public static void startCluster() throws Exception {
UTIL.startMiniDFSCluster(1);
fs = UTIL.getDFSCluster().getFileSystem();
rootDir = UTIL.getDefaultRootDirPath();
+ snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
}
@AfterClass
@@ -76,48 +80,57 @@ public class TestSnapshotFileCache {
@After
public void cleanupFiles() throws Exception {
// cleanup the snapshot directory
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
fs.delete(snapshotDir, true);
}
@Test
public void testLoadAndDelete() throws IOException {
- // don't refresh the cache unless we tell it to
- long period = Long.MAX_VALUE;
- SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+ SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- createAndTestSnapshotV1(cache, "snapshot1a", false, true);
+ createAndTestSnapshotV1(cache, "snapshot1a", false, true, false);
- createAndTestSnapshotV2(cache, "snapshot2a", false, true);
+ createAndTestSnapshotV2(cache, "snapshot2a", false, true, false);
}
@Test
public void testReloadModifiedDirectory() throws IOException {
- // don't refresh the cache unless we tell it to
- long period = Long.MAX_VALUE;
- SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+ SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- createAndTestSnapshotV1(cache, "snapshot1", false, true);
+ createAndTestSnapshotV1(cache, "snapshot1", false, true, false);
// now delete the snapshot and add a file with a different name
- createAndTestSnapshotV1(cache, "snapshot1", false, false);
+ createAndTestSnapshotV1(cache, "snapshot1", false, false, false);
- createAndTestSnapshotV2(cache, "snapshot2", false, true);
+ createAndTestSnapshotV2(cache, "snapshot2", false, true, false);
// now delete the snapshot and add a file with a different name
- createAndTestSnapshotV2(cache, "snapshot2", false, false);
+ createAndTestSnapshotV2(cache, "snapshot2", false, false, false);
}
@Test
public void testSnapshotTempDirReload() throws IOException {
- long period = Long.MAX_VALUE;
- // This doesn't refresh cache until we invoke it explicitly
- SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+ SnapshotFileCache cache =
+ new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+ // Add a new non-tmp snapshot
+ createAndTestSnapshotV1(cache, "snapshot0v1", false, false, false);
+ createAndTestSnapshotV1(cache, "snapshot0v2", false, false, false);
+ }
+
+ @Test
+ public void testCacheUpdatedWhenLastModifiedOfSnapDirNotUpdated() throws IOException {
+ SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
// Add a new non-tmp snapshot
- createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
- createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
+ createAndTestSnapshotV1(cache, "snapshot1v1", false, false, true);
+ createAndTestSnapshotV1(cache, "snapshot1v2", false, false, true);
+
+ // Add a new tmp snapshot
+ createAndTestSnapshotV2(cache, "snapshot2v1", true, false, true);
+
+ // Add another tmp snapshot
+ createAndTestSnapshotV2(cache, "snapshot2v2", true, false, true);
}
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
@@ -130,23 +143,24 @@ public class TestSnapshotFileCache {
};
private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache,
- final String name, final boolean tmp, final boolean removeOnExit) throws IOException {
+ final String name, final boolean tmp, final boolean removeOnExit, boolean setFolderTime)
+ throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name);
- createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+ createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
return builder;
}
private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
- final boolean tmp, final boolean removeOnExit) throws IOException {
+ final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name);
- createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+ createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
}
private void createAndTestSnapshot(final SnapshotFileCache cache,
final SnapshotMock.SnapshotBuilder builder,
- final boolean tmp, final boolean removeOnExit) throws IOException {
+ final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
List<Path> files = new ArrayList<>();
for (int i = 0; i < 3; ++i) {
for (Path filePath: builder.addRegion()) {
@@ -157,6 +171,10 @@ public class TestSnapshotFileCache {
// Finalize the snapshot
builder.commit();
+ if (setFolderTime) {
+ fs.setTimes(snapshotDir, 0, -1);
+ }
+
// Make sure that all files are still present
for (Path path: files) {
assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path));