You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2013/05/24 00:04:37 UTC
svn commit: r1485870 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/backup/
main/java/org/apache/hadoop/hbase/regionserver/wal/
main/java/org/apache/hadoop/hbase/util/
test/java/org/apache/hadoop/hbase/util/
Author: tedyu
Date: Thu May 23 22:04:36 2013
New Revision: 1485870
URL: http://svn.apache.org/r1485870
Log:
HBASE-8522 Archived hfiles and old hlogs may be deleted immediately by HFileCleaner, LogCleaner in HMaster (Liu Shaohui)
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java Thu May 23 22:04:36 2013
@@ -582,7 +582,7 @@ public class HFileArchiver {
public boolean moveAndClose(Path dest) throws IOException {
this.close();
Path p = this.getPath();
- return fs.rename(p, dest);
+ return FSUtils.renameAndSetModifyTime(fs, p, dest);
}
/**
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java Thu May 23 22:04:36 2013
@@ -701,7 +701,7 @@ class FSHLog implements HLog, Syncable {
i.preLogArchive(p, newPath);
}
}
- if (!this.fs.rename(p, newPath)) {
+ if (!FSUtils.renameAndSetModifyTime(this.fs, p, newPath)) {
throw new IOException("Unable to rename " + p + " to " + newPath);
}
// Tell our listeners that a log has been archived.
@@ -754,7 +754,7 @@ class FSHLog implements HLog, Syncable {
}
}
- if (!fs.rename(file.getPath(),p)) {
+ if (!FSUtils.renameAndSetModifyTime(fs, file.getPath(), p)) {
throw new IOException("Unable to rename " + file.getPath() + " to " + p);
}
// Tell our listeners that a log was archived.
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java Thu May 23 22:04:36 2013
@@ -672,7 +672,7 @@ public class HLogSplitter {
for (Path p : processedLogs) {
Path newPath = FSHLog.getHLogArchivePath(oldLogDir, p);
if (fs.exists(p)) {
- if (!fs.rename(p, newPath)) {
+ if (!FSUtils.renameAndSetModifyTime(fs, p, newPath)) {
LOG.warn("Unable to move " + p + " to " + newPath);
} else {
LOG.debug("Archived processed log " + p + " to " + newPath);
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Thu May 23 22:04:36 2013
@@ -1641,4 +1641,12 @@ public abstract class FSUtils {
}
}
}
+
+ public static boolean renameAndSetModifyTime(final FileSystem fs, Path src, Path dest)
+ throws IOException {
+ if (!fs.rename(src, dest)) return false;
+ // set the modify time for TimeToLive Cleaner
+ fs.setTimes(dest, EnvironmentEdgeManager.currentTimeMillis(), -1);
+ return true;
+ }
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java?rev=1485870&r1=1485869&r2=1485870&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java Thu May 23 22:04:36 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotEquals;
import java.io.File;
import java.io.IOException;
@@ -287,4 +288,39 @@ public class TestFSUtils {
}
}
+ @Test
+ public void testRenameAndSetModifyTime() throws Exception {
+ HBaseTestingUtility htu = new HBaseTestingUtility();
+ Configuration conf = htu.getConfiguration();
+
+ MiniDFSCluster cluster = htu.startMiniDFSCluster(1);
+ assertTrue(FSUtils.isHDFS(conf));
+
+ FileSystem fs = FileSystem.get(conf);
+ Path testDir = htu.getDataTestDir("testArchiveFile");
+
+ String file = UUID.randomUUID().toString();
+ Path p = new Path(testDir, file);
+
+ FSDataOutputStream out = fs.create(p);
+ out.close();
+ assertTrue("The created file should be present", FSUtils.isExists(fs, p));
+
+ long expect = System.currentTimeMillis() + 1000;
+ assertNotEquals(expect, fs.getFileStatus(p).getModificationTime());
+
+ ManualEnvironmentEdge mockEnv = new ManualEnvironmentEdge();
+ mockEnv.setValue(expect);
+ EnvironmentEdgeManager.injectEdge(mockEnv);
+
+ String dstFile = UUID.randomUUID().toString();
+ Path dst = new Path(testDir , dstFile);
+
+ assertTrue(FSUtils.renameAndSetModifyTime(fs, p, dst));
+ assertFalse("The moved file should not be present", FSUtils.isExists(fs, p));
+ assertTrue("The dst file should be present", FSUtils.isExists(fs, dst));
+
+ assertEquals(expect, fs.getFileStatus(dst).getModificationTime());
+ cluster.shutdown();
+ }
}