You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:14:45 UTC
svn commit: r1181508 - in /hbase/branches/0.89/src:
main/java/org/apache/hadoop/hbase/regionserver/Store.java
main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
Author: nspiegelberg
Date: Tue Oct 11 02:14:45 2011
New Revision: 1181508
URL: http://svn.apache.org/viewvc?rev=1181508&view=rev
Log:
HBASE-3693: isMajorCompaction() triggers lots of listStatus DFS RPC calls from HBase
Summary:
Cache the modification time stamp in the storefile.
So every time when adding a new storefile into the store, it will cache the
lastest modification timestamp.
Test Plan:
Add a new unit test function to verify that the cache result in each store file
is exactly the same as the previous one, which get the timestamp from fs
directly.
Passing unit tests
Reviewed By: kannan
Reviewers: jgray, nspiegelberg, dhruba, kannan
Commenters: jgray, nspiegelberg
CC: jgray, nspiegelberg, awolff, kannan, hbase@lists,
Tasks:
#485656: HBASE-3693: isMajorCompaction() triggers lots of listStatus DFS RPC
calls from HBase
Revert Plan:
OK
Differential Revision: 228349
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1181508&r1=1181507&r2=1181508&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Oct 11 02:14:45 2011
@@ -706,23 +706,11 @@ public class Store implements HeapSize {
* @param dir
* @throws IOException
*/
- private static long getLowestTimestamp(FileSystem fs,
- final List<StoreFile> candidates) throws IOException {
+ public static long getLowestTimestamp(final List<StoreFile> candidates)
+ throws IOException {
long minTs = Long.MAX_VALUE;
- if (candidates.isEmpty()) {
- return minTs;
- }
- Path[] p = new Path[candidates.size()];
- for (int i = 0; i < candidates.size(); ++i) {
- p[i] = candidates.get(i).getPath();
- }
-
- FileStatus[] stats = fs.listStatus(p);
- if (stats == null || stats.length == 0) {
- return minTs;
- }
- for (FileStatus s : stats) {
- minTs = Math.min(minTs, s.getModificationTime());
+ for (StoreFile storeFile : candidates) {
+ minTs = Math.min(minTs, storeFile.getModificationTimeStamp());
}
return minTs;
}
@@ -761,7 +749,7 @@ public class Store implements HeapSize {
majorCompactionTime == 0) {
return result;
}
- long lowTimestamp = getLowestTimestamp(fs, filesToCompact);
+ long lowTimestamp = getLowestTimestamp(filesToCompact);
long now = System.currentTimeMillis();
if (lowTimestamp > 0l && lowTimestamp < (now - this.majorCompactionTime)) {
// Major compaction time has elapsed.
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1181508&r1=1181507&r2=1181508&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Tue Oct 11 02:14:45 2011
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.regionse
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
@@ -171,6 +172,8 @@ public class StoreFile {
private final Configuration conf;
private final BloomType bloomType;
+ // the last modification time stamp
+ private long modificationTimeStamp = 0L;
/**
* Constructor, loads a reader and it's indices, etc. May allocate a
@@ -207,6 +210,14 @@ public class StoreFile {
this.bloomType = BloomType.NONE;
LOG.info("Ignoring bloom filter check for file (disabled in config)");
}
+
+ // cache the modification time stamp of this store file
+ FileStatus[] stats = fs.listStatus(p);
+ if (stats != null && stats.length == 1) {
+ this.modificationTimeStamp = stats[0].getModificationTime();
+ } else {
+ this.modificationTimeStamp = 0;
+ }
}
/**
@@ -296,6 +307,14 @@ public class StoreFile {
return this.sequenceid;
}
+ public long getModificationTimeStamp() {
+ return modificationTimeStamp;
+ }
+
+ public void setModificationTimeStamp(long modificationTimeStamp) {
+ this.modificationTimeStamp = modificationTimeStamp;
+ }
+
/**
* Return the highest sequence ID found across all storefiles in
* the given list. Store files that were created by a mapreduce
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=1181508&r1=1181507&r2=1181508&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java Tue Oct 11 02:14:45 2011
@@ -132,6 +132,54 @@ public class TestStore extends TestCase
store = new Store(basedir, region, hcd, fs, conf);
}
+ public void testLowestModificationTime() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ FileSystem fs = FileSystem.get(conf);
+ // Initialize region
+ init(getName(), conf);
+
+ int stroeFileNum = 4;
+ for (int i = 1; i <= stroeFileNum; i++) {
+ LOG.info("Adding some data for the store file #"+i);
+ this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
+ this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
+ this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
+ flush(i);
+ }
+ // after flush; check the lowest time stamp
+ long lowestTimeStampFromStore =
+ Store.getLowestTimestamp(store.getStorefiles());
+ long lowestTimeStampFromFS =
+ getLowestTimeStampFromFS(fs,store.getStorefiles());
+ assertEquals(lowestTimeStampFromStore,lowestTimeStampFromFS);
+
+ // after compact; check the lowest time stamp
+ store.compact();
+ lowestTimeStampFromStore = Store.getLowestTimestamp(store.getStorefiles());
+ lowestTimeStampFromFS = getLowestTimeStampFromFS(fs,store.getStorefiles());
+ assertEquals(lowestTimeStampFromStore,lowestTimeStampFromFS);
+ }
+
+ private static long getLowestTimeStampFromFS(FileSystem fs,
+ final List<StoreFile> candidates) throws IOException {
+ long minTs = Long.MAX_VALUE;
+ if (candidates.isEmpty()) {
+ return minTs;
+ }
+ Path[] p = new Path[candidates.size()];
+ for (int i = 0; i < candidates.size(); ++i) {
+ p[i] = candidates.get(i).getPath();
+ }
+
+ FileStatus[] stats = fs.listStatus(p);
+ if (stats == null || stats.length == 0) {
+ return minTs;
+ }
+ for (FileStatus s : stats) {
+ minTs = Math.min(minTs, s.getModificationTime());
+ }
+ return minTs;
+ }
//////////////////////////////////////////////////////////////////////////////
// Get tests