You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/03/17 17:16:15 UTC
svn commit: r1578453 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
Author: mbertozzi
Date: Mon Mar 17 16:16:14 2014
New Revision: 1578453
URL: http://svn.apache.org/r1578453
Log:
HBASE-10766 SnapshotCleaner allows to delete referenced files (Bharath Vissapragada)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java?rev=1578453&r1=1578452&r2=1578453&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java Mon Mar 17 16:16:14 2014
@@ -183,18 +183,29 @@ public class SnapshotFileCache implement
}
private synchronized void refreshCache() throws IOException {
- // get the status of the snapshots directory
- FileStatus status;
+ // get the status of the snapshots directory and <snapshot dir>/.tmp
+ FileStatus dirStatus, tempStatus;
try {
- status = fs.getFileStatus(snapshotDir);
+ dirStatus = fs.getFileStatus(snapshotDir);
} catch (FileNotFoundException e) {
if (this.cache.size() > 0) {
LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
}
return;
}
+
+ try {
+ Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
+ tempStatus = fs.getFileStatus(snapshotTmpDir);
+ } catch (FileNotFoundException e) {
+ tempStatus = dirStatus;
+ }
+
// if the snapshot directory wasn't modified since we last check, we are done
- if (status.getModificationTime() <= lastModifiedTime) return;
+ if (dirStatus.getModificationTime() <= lastModifiedTime &&
+ tempStatus.getModificationTime() <= lastModifiedTime) {
+ return;
+ }
// directory was modified, so we need to reload our cache
// there could be a slight race here where we miss the cache, check the directory modification
@@ -202,7 +213,8 @@ public class SnapshotFileCache implement
// However, snapshot directories are only created once, so this isn't an issue.
// 1. update the modified time
- this.lastModifiedTime = status.getModificationTime();
+ this.lastModifiedTime = Math.min(dirStatus.getModificationTime(),
+ tempStatus.getModificationTime());
// 2.clear the cache
this.cache.clear();
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java?rev=1578453&r1=1578452&r2=1578453&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java Mon Mar 17 16:16:14 2014
@@ -29,7 +29,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
@@ -87,8 +86,8 @@ public class TestSnapshotFileCache {
Path file2 = new Path(family, "file2");
// create two hfiles under the snapshot
- fs.create(file1);
- fs.create(file2);
+ fs.createNewFile(file1);
+ fs.createNewFile(file2);
FSUtils.logFileSystemState(fs, rootDir, LOG);
@@ -138,7 +137,7 @@ public class TestSnapshotFileCache {
Path region = new Path(snapshot, "7e91021");
Path family = new Path(region, "fam");
Path file1 = new Path(family, "file1");
- fs.create(file1);
+ fs.createNewFile(file1);
// create an 'in progress' snapshot
SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
@@ -146,7 +145,7 @@ public class TestSnapshotFileCache {
region = new Path(snapshot, "7e91021");
family = new Path(region, "fam");
Path file2 = new Path(family, "file2");
- fs.create(file2);
+ fs.createNewFile(file2);
FSUtils.logFileSystemState(fs, rootDir, LOG);
@@ -173,12 +172,12 @@ public class TestSnapshotFileCache {
Path region = new Path(snapshot, "7e91021");
Path family = new Path(region, "fam");
Path file1 = new Path(family, "file1");
- fs.create(file1);
+ fs.createNewFile(file1);
// and another file in the logs directory
Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
- fs.create(log);
+ fs.createNewFile(log);
FSUtils.logFileSystemState(fs, rootDir, LOG);
@@ -203,8 +202,8 @@ public class TestSnapshotFileCache {
Path file2 = new Path(family, "file2");
// create two hfiles under the snapshot
- fs.create(file1);
- fs.create(file2);
+ fs.createNewFile(file1);
+ fs.createNewFile(file2);
FSUtils.logFileSystemState(fs, rootDir, LOG);
@@ -213,12 +212,34 @@ public class TestSnapshotFileCache {
// now delete the snapshot and add a file with a different name
fs.delete(snapshot, true);
Path file3 = new Path(family, "new_file");
- fs.create(file3);
+ fs.createNewFile(file3);
FSUtils.logFileSystemState(fs, rootDir, LOG);
assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
}
+ @Test
+ public void testSnapshotTempDirReload() throws IOException {
+ long period = Long.MAX_VALUE;
+ // This doesn't refresh cache until we invoke it explicitly
+ Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
+ SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
+ SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+ "test-snapshot-file-cache-refresh", new SnapshotFiles());
+
+ // Add a new snapshot
+ Path snapshot1 = new Path(snapshotDir, "snapshot1");
+ Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
+ fs.createNewFile(file1);
+ assertTrue(cache.contains(file1.getName()));
+
+ // Add another snapshot
+ Path snapshot2 = new Path(snapshotDir, "snapshot2");
+ Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
+ fs.createNewFile(file2);
+ assertTrue(cache.contains(file2.getName()));
+ }
+
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
Collection<String> files = new HashSet<String>();