You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/03/25 22:05:13 UTC
svn commit: r1085560 - in /hbase/trunk: CHANGES.txt
src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
Author: stack
Date: Fri Mar 25 21:05:13 2011
New Revision: 1085560
URL: http://svn.apache.org/viewvc?rev=1085560&view=rev
Log:
HBASE-3696 isMajorCompaction() check triggers lots of listStatus DFS RPC calls from HBase
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1085560&r1=1085559&r2=1085560&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Mar 25 21:05:13 2011
@@ -102,6 +102,8 @@ Release 0.91.0 - Unreleased
HBASE-3541 REST Multi Gets (Elliott Clark via Stack)
HBASE-3052 Add ability to have multiple ZK servers in a quorum in
MiniZooKeeperCluster for test writing (Liyin Tang via Stack)
+ HBASE-3696 isMajorCompaction() check triggers lots of listStatus DFS RPC
+ calls from HBase (Liyin Tang via Stack)
TASK
HBASE-3559 Move report of split to master OFF the heartbeat channel
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1085560&r1=1085559&r2=1085560&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Fri Mar 25 21:05:13 2011
@@ -725,23 +725,11 @@ public class Store implements HeapSize {
* @param dir
* @throws IOException
*/
- private static long getLowestTimestamp(FileSystem fs,
- final List<StoreFile> candidates) throws IOException {
+ public static long getLowestTimestamp(final List<StoreFile> candidates)
+ throws IOException {
long minTs = Long.MAX_VALUE;
- if (candidates.isEmpty()) {
- return minTs;
- }
- Path[] p = new Path[candidates.size()];
- for (int i = 0; i < candidates.size(); ++i) {
- p[i] = candidates.get(i).getPath();
- }
-
- FileStatus[] stats = fs.listStatus(p);
- if (stats == null || stats.length == 0) {
- return minTs;
- }
- for (FileStatus s : stats) {
- minTs = Math.min(minTs, s.getModificationTime());
+ for (StoreFile storeFile : candidates) {
+ minTs = Math.min(minTs, storeFile.getModificationTimeStamp());
}
return minTs;
}
@@ -781,7 +769,7 @@ public class Store implements HeapSize {
return result;
}
// TODO: Use better method for determining stamp of last major (HBASE-2990)
- long lowTimestamp = getLowestTimestamp(fs, filesToCompact);
+ long lowTimestamp = getLowestTimestamp(filesToCompact);
long now = System.currentTimeMillis();
if (lowTimestamp > 0l && lowTimestamp < (now - this.majorCompactionTime)) {
// Major compaction time has elapsed.
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1085560&r1=1085559&r2=1085560&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Fri Mar 25 21:05:13 2011
@@ -40,6 +40,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
@@ -171,6 +172,8 @@ public class StoreFile {
private final Configuration conf;
private final BloomType bloomType;
+ // the last modification time stamp
+ private long modificationTimeStamp = 0L;
/**
* Constructor, loads a reader and it's indices, etc. May allocate a
@@ -207,6 +210,14 @@ public class StoreFile {
this.bloomType = BloomType.NONE;
LOG.info("Ignoring bloom filter check for file (disabled in config)");
}
+
+ // cache the modification time stamp of this store file
+ FileStatus[] stats = fs.listStatus(p);
+ if (stats != null && stats.length == 1) {
+ this.modificationTimeStamp = stats[0].getModificationTime();
+ } else {
+ this.modificationTimeStamp = 0;
+ }
}
/**
@@ -296,6 +307,10 @@ public class StoreFile {
return this.sequenceid;
}
+ public long getModificationTimeStamp() {
+ return modificationTimeStamp;
+ }
+
/**
* Return the highest sequence ID found across all storefiles in
* the given list. Store files that were created by a mapreduce
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=1085560&r1=1085559&r2=1085560&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java Fri Mar 25 21:05:13 2011
@@ -132,6 +132,54 @@ public class TestStore extends TestCase
store = new Store(basedir, region, hcd, fs, conf);
}
+ public void testLowestModificationTime() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ FileSystem fs = FileSystem.get(conf);
+ // Initialize region
+ init(getName(), conf);
+
+ int storeFileNum = 4;
+ for (int i = 1; i <= storeFileNum; i++) {
+ LOG.info("Adding some data for the store file #"+i);
+ this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
+ this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
+ this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
+ flush(i);
+ }
+ // after flush; check the lowest time stamp
+ long lowestTimeStampFromStore =
+ Store.getLowestTimestamp(store.getStorefiles());
+ long lowestTimeStampFromFS =
+ getLowestTimeStampFromFS(fs,store.getStorefiles());
+ assertEquals(lowestTimeStampFromStore,lowestTimeStampFromFS);
+
+ // after compact; check the lowest time stamp
+ store.compact();
+ lowestTimeStampFromStore = Store.getLowestTimestamp(store.getStorefiles());
+ lowestTimeStampFromFS = getLowestTimeStampFromFS(fs,store.getStorefiles());
+ assertEquals(lowestTimeStampFromStore,lowestTimeStampFromFS);
+ }
+
+ private static long getLowestTimeStampFromFS(FileSystem fs,
+ final List<StoreFile> candidates) throws IOException {
+ long minTs = Long.MAX_VALUE;
+ if (candidates.isEmpty()) {
+ return minTs;
+ }
+ Path[] p = new Path[candidates.size()];
+ for (int i = 0; i < candidates.size(); ++i) {
+ p[i] = candidates.get(i).getPath();
+ }
+
+ FileStatus[] stats = fs.listStatus(p);
+ if (stats == null || stats.length == 0) {
+ return minTs;
+ }
+ for (FileStatus s : stats) {
+ minTs = Math.min(minTs, s.getModificationTime());
+ }
+ return minTs;
+ }
//////////////////////////////////////////////////////////////////////////////
// Get tests