You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2013/02/13 19:18:36 UTC
svn commit: r1445792 - in /hbase/branches/hbase-7290/hbase-server/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/master/snapshot/
main/java/org/apache/hadoop/hbase/snapshot/
test/java/org/apache/hadoop/hbase/client/ test/java...
Author: jmhsieh
Date: Wed Feb 13 18:18:35 2013
New Revision: 1445792
URL: http://svn.apache.org/r1445792
Log:
HBASE-7174 [snapshots] Refactor snapshot file cleaner cache to use Snapshot FileVisitor (Matteo Bertozzi)
Modified:
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java?rev=1445792&r1=1445791&r2=1445792&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java Wed Feb 13 18:18:35 2013
@@ -92,6 +92,14 @@ public abstract class Chore extends HasT
this.sleeper.skipSleepCycle();
}
+ /*
+ * Exposed for TESTING!
+ * calls directly the chore method, from the current thread.
+ */
+ public void choreForTesting() {
+ chore();
+ }
+
/**
* Override to run a task before we start looping.
* @return true if initial chore was successful
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java?rev=1445792&r1=1445791&r2=1445792&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java Wed Feb 13 18:18:35 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.master.s
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -36,7 +35,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -60,25 +58,10 @@ import org.apache.hadoop.hbase.util.FSUt
* Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are
* also removed from the cache.
* <p>
- * A {@link PathFilter} must be passed when creating <tt>this</tt> to allow filtering of children
+ * A {@link SnapshotFileInspector} must be passed when creating <tt>this</tt> to allow extraction of files
* under the /hbase/.snapshot/[snapshot name] directory, for each snapshot. This allows you to only
* cache files under, for instance, all the logs in the .logs directory or all the files under all
- * the regions. The filter is only applied the children, not to the children of children. For
- * instance, given the layout:
- *
- * <pre>
- * /hbase/.snapshot/SomeSnapshot/
- * .logs/
- * server/
- * server.1234567
- * .regioninfo
- * 1234567890/
- * family/
- * 123456789
- * </pre>
- *
- * would only apply a filter to directories: .logs, .regioninfo and 1234567890, not to their
- * children.
+ * the regions.
* <p>
* <tt>this</tt> also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
* snapshots and will attempt to cache files from those snapshots as well.
@@ -88,11 +71,19 @@ import org.apache.hadoop.hbase.util.FSUt
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class SnapshotFileCache implements Stoppable {
+ public interface SnapshotFileInspector {
+ /**
+ * Returns a collection of file names needed by the snapshot.
+ * @param snapshotDir {@link Path} to the snapshot directory to scan.
+ * @return the collection of file names needed by the snapshot.
+ */
+ Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException;
+ }
private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class);
private volatile boolean stop = false;
private final FileSystem fs;
- private final PathFilter dirFilter;
+ private final SnapshotFileInspector fileInspector;
private final Path snapshotDir;
private final Set<String> cache = new HashSet<String>();
/**
@@ -113,13 +104,13 @@ public class SnapshotFileCache implement
* hbase root directory
* @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
* @param refreshThreadName name of the cache refresh thread
- * @param inspectSnapshotDirectory Filter to apply to the directories under each snapshot.
+ * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
* @throws IOException if the {@link FileSystem} or root directory cannot be loaded
*/
public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName,
- PathFilter inspectSnapshotDirectory) throws IOException {
+ SnapshotFileInspector inspectSnapshotFiles) throws IOException {
this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod,
- refreshThreadName, inspectSnapshotDirectory);
+ refreshThreadName, inspectSnapshotFiles);
}
/**
@@ -130,12 +121,12 @@ public class SnapshotFileCache implement
* @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
* @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
* @param refreshThreadName name of the cache refresh thread
- * @param inspectSnapshotDirectory Filter to apply to the directories under each snapshot.
+ * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
*/
public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
- long cacheRefreshDelay, String refreshThreadName, PathFilter inspectSnapshotDirectory) {
+ long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
this.fs = fs;
- this.dirFilter = inspectSnapshotDirectory;
+ this.fileInspector = inspectSnapshotFiles;
this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
// periodically refresh the file cache to make sure we aren't superfluously saving files.
this.refreshTimer = new Timer(refreshThreadName, true);
@@ -229,23 +220,23 @@ public class SnapshotFileCache implement
FileStatus[] running = FSUtils.listStatus(fs, snapshot.getPath());
if (running == null) continue;
for (FileStatus run : running) {
- this.cache.addAll(getAllFiles(run, dirFilter));
+ this.cache.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
}
+ } else {
+ SnapshotDirectoryInfo files = this.snapshots.remove(name);
+ // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
+ // the latter could occur where I create a snapshot, then delete it, and then make a new
+ // snapshot with the same name. We will need to update the cache the information from that new
+ // snapshot, even though it has the same name as the files referenced have probably changed.
+ if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
+ // get all files for the snapshot and create a new info
+ Collection<String> storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath());
+ files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
+ }
+ // 3.2 add all the files to cache
+ this.cache.addAll(files.getFiles());
+ known.put(name, files);
}
-
- SnapshotDirectoryInfo files = this.snapshots.remove(name);
- // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
- // the latter could occur where I create a snapshot, then delete it, and then make a new
- // snapshot with the same name. We will need to update the cache the information from that new
- // snapshot, even though it has the same name as the files referenced have probably changed.
- if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
- // get all files for the snapshot and create a new info
- Collection<String> storedFiles = getAllFiles(snapshot, dirFilter);
- files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
- }
- // 3.2 add all the files to cache
- this.cache.addAll(files.getFiles());
- known.put(name, files);
}
// 4. set the snapshots we are tracking
@@ -253,22 +244,6 @@ public class SnapshotFileCache implement
this.snapshots.putAll(known);
}
- private Collection<String> getAllFiles(FileStatus file, PathFilter filter) throws IOException {
- if (!file.isDir()) return Collections.singleton(file.getPath().getName());
-
- Set<String> ret = new HashSet<String>();
- // read all the files/directories below the passed directory
- FileStatus[] files = FSUtils.listStatus(fs, file.getPath(), filter);
- if (files == null) return ret;
-
- // get all the files for the children
- for (FileStatus child : files) {
- // children aren't filtered out
- ret.addAll(getAllFiles(child, null));
- }
- return ret;
- }
-
/**
* Simple helper task that just periodically attempts to refresh the cache
*/
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java?rev=1445792&r1=1445791&r2=1445792&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java Wed Feb 13 18:18:35 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.master.snapshot;
import java.io.IOException;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.util.FSUtils;
/**
@@ -67,11 +69,15 @@ public class SnapshotHFileCleaner extend
try {
long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
- FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+ final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
Path rootDir = FSUtils.getRootDir(conf);
cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
- "snapshot-hfile-cleaner-cache-refresher", new FSUtils.RegionDirFilter(fs)
- );
+ "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+ public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+ throws IOException {
+ return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
+ }
+ });
} catch (IOException e) {
LOG.error("Failed to create cleaner util", e);
}
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java?rev=1445792&r1=1445791&r2=1445792&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java Wed Feb 13 18:18:35 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.master.snapshot;
import java.io.IOException;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -26,9 +27,9 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.util.FSUtils;
/**
@@ -77,20 +78,10 @@ public class SnapshotLogCleaner extends
final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
Path rootDir = FSUtils.getRootDir(conf);
cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
- "snapshot-log-cleaner-cache-refresher", new PathFilter() {
- @Override
- public boolean accept(Path path) {
- if (path.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
- try {
- if (!fs.isFile(path)) return true;
- } catch (IOException e) {
- LOG.error("Couldn't reach fs to check:" + path + " is a directory, stopping!");
- SnapshotLogCleaner.this.stop("Couldn't reach FS to check if " + path
- + " was a directory");
- }
- }
-
- return false;
+ "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+ public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+ throws IOException {
+ return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
}
});
} catch (IOException e) {
Modified: hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java?rev=1445792&r1=1445791&r2=1445792&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java Wed Feb 13 18:18:35 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -221,7 +222,11 @@ public final class SnapshotReferenceUtil
visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
public void storeFile (final String region, final String family, final String hfile)
throws IOException {
- names.add(hfile);
+ if (HFileLink.isHFileLink(hfile)) {
+ names.add(HFileLink.getReferencedHFileName(hfile));
+ } else {
+ names.add(hfile);
+ }
}
});
return names;
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java?rev=1445792&r1=1445791&r2=1445792&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java Wed Feb 13 18:18:35 2013
@@ -71,7 +71,11 @@ public class TestRestoreSnapshotFromClie
@BeforeClass
public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.getConfiguration().set("hbase.master.hfilecleaner.plugins",
+ "org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner," +
+ "org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner");
TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
+ TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
@@ -125,10 +129,13 @@ public class TestRestoreSnapshotFromClie
@After
public void tearDown() throws Exception {
- admin.disableTable(tableName);
- admin.deleteTable(tableName);
+ if (admin.tableExists(tableName)) {
+ admin.disableTable(tableName);
+ admin.deleteTable(tableName);
+ }
admin.deleteSnapshot(snapshotName0);
admin.deleteSnapshot(snapshotName1);
+ waitCleanerRun();
}
@Test
@@ -185,6 +192,7 @@ public class TestRestoreSnapshotFromClie
admin.disableTable(clonedTableName);
admin.snapshot(snapshotName2, clonedTableName);
admin.deleteTable(clonedTableName);
+ waitCleanerRun();
admin.cloneSnapshot(snapshotName2, clonedTableName);
table = new HTable(TEST_UTIL.getConfiguration(), clonedTableName);
@@ -193,6 +201,68 @@ public class TestRestoreSnapshotFromClie
admin.deleteTable(clonedTableName);
}
+ /**
+ * Verify that tables created from the snapshot are still alive after source table deletion.
+ */
+ @Test
+ public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
+ // Clone a table from the first snapshot
+ byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis());
+ admin.cloneSnapshot(snapshotName0, clonedTableName);
+ HTable table = new HTable(TEST_UTIL.getConfiguration(), clonedTableName);
+ assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
+
+ // Take a snapshot of this cloned table.
+ admin.disableTable(clonedTableName);
+ admin.snapshot(snapshotName2, clonedTableName);
+
+ // Clone the snapshot of the cloned table
+ byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis());
+ admin.cloneSnapshot(snapshotName2, clonedTableName2);
+ table = new HTable(TEST_UTIL.getConfiguration(), clonedTableName2);
+ assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
+ admin.disableTable(clonedTableName2);
+
+ // Remove the original table
+ admin.disableTable(tableName);
+ admin.deleteTable(tableName);
+ waitCleanerRun();
+
+ // Verify the first cloned table
+ admin.enableTable(clonedTableName);
+ table = new HTable(TEST_UTIL.getConfiguration(), clonedTableName);
+ assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
+
+ // Verify the second cloned table
+ admin.enableTable(clonedTableName2);
+ table = new HTable(TEST_UTIL.getConfiguration(), clonedTableName2);
+ assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
+ admin.disableTable(clonedTableName2);
+
+ // Delete the first cloned table
+ admin.disableTable(clonedTableName);
+ admin.deleteTable(clonedTableName);
+ waitCleanerRun();
+
+ // Verify the second cloned table
+ admin.enableTable(clonedTableName2);
+ table = new HTable(TEST_UTIL.getConfiguration(), clonedTableName2);
+ assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
+
+ // Clone a new table from cloned
+ byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis());
+ admin.cloneSnapshot(snapshotName2, clonedTableName3);
+ table = new HTable(TEST_UTIL.getConfiguration(), clonedTableName3);
+ assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
+
+ // Delete the cloned tables
+ admin.disableTable(clonedTableName2);
+ admin.deleteTable(clonedTableName2);
+ admin.disableTable(clonedTableName3);
+ admin.deleteTable(clonedTableName3);
+ admin.deleteSnapshot(snapshotName2);
+ }
+
// ==========================================================================
// Helpers
// ==========================================================================
@@ -225,4 +295,8 @@ public class TestRestoreSnapshotFromClie
}
table.flushCommits();
}
+
+ private void waitCleanerRun() throws InterruptedException {
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
+ }
}
Modified: hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java?rev=1445792&r1=1445791&r2=1445792&view=diff
==============================================================================
--- hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java (original)
+++ hbase/branches/hbase-7290/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java Wed Feb 13 18:18:35 2013
@@ -21,18 +21,20 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.server.snapshot.TakeSnapshotUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.AfterClass;
@@ -76,10 +78,10 @@ public class TestSnapshotFileCache {
long period = Long.MAX_VALUE;
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
- "test-snapshot-file-cache-refresh", null);
+ "test-snapshot-file-cache-refresh", new SnapshotFiles());
Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "region1");
+ Path region = new Path(snapshot, "7e91021");
Path family = new Path(region, "fam");
Path file1 = new Path(family, "file1");
Path file2 = new Path(family, "file2");
@@ -129,19 +131,19 @@ public class TestSnapshotFileCache {
long period = Long.MAX_VALUE;
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
- "test-snapshot-file-cache-refresh", null);
+ "test-snapshot-file-cache-refresh", new SnapshotFiles());
// create a file in a 'completed' snapshot
Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "region1");
+ Path region = new Path(snapshot, "7e91021");
Path family = new Path(region, "fam");
Path file1 = new Path(family, "file1");
fs.create(file1);
// create an 'in progress' snapshot
SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
- snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, snapshotDir);
- region = new Path(snapshot, "region1");
+ snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+ region = new Path(snapshot, "7e91021");
family = new Path(region, "fam");
Path file2 = new Path(family, "file2");
fs.create(file2);
@@ -158,19 +160,17 @@ public class TestSnapshotFileCache {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
- PathFilter filter = new PathFilter() {
-
- @Override
- public boolean accept(Path path) {
- return path.getName().equals(HConstants.HREGION_LOGDIR_NAME);
- }
- };
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
- "test-snapshot-file-cache-refresh", filter);
+ "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() {
+ public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+ throws IOException {
+ return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
+ }
+ });
// create a file in a 'completed' snapshot
Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "region1");
+ Path region = new Path(snapshot, "7e91021");
Path family = new Path(region, "fam");
Path file1 = new Path(family, "file1");
fs.create(file1);
@@ -194,10 +194,10 @@ public class TestSnapshotFileCache {
long period = Long.MAX_VALUE;
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
- "test-snapshot-file-cache-refresh", null);
+ "test-snapshot-file-cache-refresh", new SnapshotFiles());
Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "region1");
+ Path region = new Path(snapshot, "7e91021");
Path family = new Path(region, "fam");
Path file1 = new Path(family, "file1");
Path file2 = new Path(family, "file2");
@@ -207,7 +207,7 @@ public class TestSnapshotFileCache {
fs.create(file2);
FSUtils.logFileSystemState(fs, rootDir, LOG);
-
+
assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
// now delete the snapshot and add a file with a different name
@@ -219,4 +219,12 @@ public class TestSnapshotFileCache {
assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
}
+ class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
+ public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
+ Collection<String> files = new HashSet<String>();
+ files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
+ files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
+ return files;
+ }
+ };
}