You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/11/04 09:28:51 UTC
svn commit: r711207 - in /hadoop/hbase/trunk: ./
src/java/org/apache/hadoop/hbase/io/
src/java/org/apache/hadoop/hbase/regionserver/
src/test/org/apache/hadoop/hbase/regionserver/
Author: stack
Date: Tue Nov 4 00:28:49 2008
New Revision: 711207
URL: http://svn.apache.org/viewvc?rev=711207&view=rev
Log:
HBASE-980 Undo core of HBASE-975, caching of start and end row
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BloomFilterMapFile.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HBaseMapFile.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HalfMapFileReader.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=711207&r1=711206&r2=711207&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue Nov 4 00:28:49 2008
@@ -61,6 +61,7 @@
HBASE-977 Arcane HStoreKey comparator bug
HBASE-979 REST web app is not started automatically
HBASE-964 Startup stuck "waiting for root region"
+ HBASE-980 Undo core of HBASE-975, caching of start and end row
IMPROVEMENTS
HBASE-901 Add a limit to key length, check key and value length on client side
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BloomFilterMapFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BloomFilterMapFile.java?rev=711207&r1=711206&r2=711207&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BloomFilterMapFile.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BloomFilterMapFile.java Tue Nov 4 00:28:49 2008
@@ -30,11 +30,13 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.util.Hash;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.onelab.filter.BloomFilter;
+import org.onelab.filter.Key;
/**
* On write, all keys are added to a bloom filter. On read, all keys are
@@ -228,4 +230,13 @@
}
}
}
+
+ /**
+ * Custom bloom filter key maker.
+ * @param key
+ * @return Key made of bytes of row only.
+ */
+ protected static Key getBloomFilterKey(WritableComparable key) {
+ return new Key(((HStoreKey) key).getRow());
+ }
}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HBaseMapFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HBaseMapFile.java?rev=711207&r1=711206&r2=711207&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HBaseMapFile.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HBaseMapFile.java Tue Nov 4 00:28:49 2008
@@ -41,26 +41,18 @@
*/
public class HBaseMapFile extends MapFile {
private static final Log LOG = LogFactory.getLog(HBaseMapFile.class);
- public static final Class<? extends Writable> VALUE_CLASS =
- ImmutableBytesWritable.class;
-
+
/**
- * Custom bloom filter key maker.
- * @param key
- * @return Key made of bytes of row only.
+ * Values are instances of this class.
*/
- protected static Key getBloomFilterKey(WritableComparable key) {
- return new Key(((HStoreKey) key).getRow());
- }
+ public static final Class<? extends Writable> VALUE_CLASS =
+ ImmutableBytesWritable.class;
/**
* A reader capable of reading and caching blocks of the data file.
*/
public static class HBaseReader extends MapFile.Reader {
private final boolean blockCacheEnabled;
- private final HStoreKey firstKey;
- private final HStoreKey finalKey;
- private final String dirName;
/**
* @param fs
@@ -88,37 +80,17 @@
throws IOException {
super(fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri),
conf, false); // defer opening streams
- this.dirName = dirName;
this.blockCacheEnabled = blockCacheEnabled;
open(fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri), conf);
- // Force reading of the mapfile index by calling midKey.
- // Reading the index will bring the index into memory over
- // here on the client and then close the index file freeing
- // up socket connection and resources in the datanode.
- // Usually, the first access on a MapFile.Reader will load the
- // index force the issue in HStoreFile MapFiles because an
- // access may not happen for some time; meantime we're
- // using up datanode resources. See HADOOP-2341.
- // Of note, midKey just goes to index. Does not seek.
+ // Force reading of the mapfile index by calling midKey. Reading the
+ // index will bring the index into memory over here on the client and
+ // then close the index file freeing up socket connection and resources
+ // in the datanode. Usually, the first access on a MapFile.Reader will
+ // load the index force the issue in HStoreFile MapFiles because an
+ // access may not happen for some time; meantime we're using up datanode
+ // resources (See HADOOP-2341). midKey() goes to index. Does not seek.
midKey();
-
- // Read in the first and last key. Cache them. Make sure we are at start
- // of the file.
- reset();
- HStoreKey key = new HStoreKey();
- super.next(key, new ImmutableBytesWritable());
- key.setHRegionInfo(hri);
- this.firstKey = key;
- // Set us back to start of file. Call to finalKey restores whatever
- // the previous position.
- reset();
-
- // Get final key.
- key = new HStoreKey();
- super.finalKey(key);
- key.setHRegionInfo(hri);
- this.finalKey = key;
}
@Override
@@ -129,71 +101,18 @@
return super.createDataFileReader(fs, dataFile, conf);
}
final int blockSize = conf.getInt("hbase.hstore.blockCache.blockSize",
- 64 * 1024);
+ 64 * 1024);
return new SequenceFile.Reader(fs, dataFile, conf) {
@Override
protected FSDataInputStream openFile(FileSystem fs, Path file,
- int bufferSize, long length) throws IOException {
-
+ int bufferSize, long length)
+ throws IOException {
return new FSDataInputStream(new BlockFSInputStream(
super.openFile(fs, file, bufferSize, length), length,
blockSize));
}
};
}
-
- @Override
- public synchronized void finalKey(final WritableComparable fk)
- throws IOException {
- Writables.copyWritable(this.finalKey, fk);
- }
-
- /**
- * @param hsk
- * @return True if the file *could* contain <code>hsk</code> and false if
- * outside the bounds of this files' start and end keys.
- */
- public boolean containsKey(final HStoreKey hsk) {
- return this.firstKey.compareTo(hsk) <= 0 &&
- this.finalKey.compareTo(hsk) >= 0;
- }
-
- public String toString() {
- HStoreKey midkey = null;
- try {
- midkey = (HStoreKey)midKey();
- } catch (IOException ioe) {
- LOG.warn("Failed get of midkey", ioe);
- }
- return "dirName=" + this.dirName + ", firstKey=" +
- this.firstKey.toString() + ", midKey=" + midkey +
- ", finalKey=" + this.finalKey;
- }
-
- /**
- * @return First key in this file. Can be null around construction time.
- */
- public HStoreKey getFirstKey() {
- return this.firstKey;
- }
-
- /**
- * @return Final key in file. Can be null around construction time.
- */
- public HStoreKey getFinalKey() {
- return this.finalKey;
- }
-
- @Override
- public synchronized WritableComparable getClosest(WritableComparable key,
- Writable value, boolean before)
- throws IOException {
- if ((!before && ((HStoreKey)key).compareTo(this.finalKey) > 0) ||
- (before && ((HStoreKey)key).compareTo(this.firstKey) < 0)) {
- return null;
- }
- return super.getClosest(key, value, before);
- }
}
public static class HBaseWriter extends MapFile.Writer {
@@ -213,7 +132,7 @@
// Default for mapfiles is 128. Makes random reads faster if we
// have more keys indexed and we're not 'next'-ing around in the
// mapfile.
- setIndexInterval(conf.getInt("hbase.io.index.interval", 128));
+ setIndexInterval(conf.getInt("hbase.io.index.interval", 32));
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HalfMapFileReader.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HalfMapFileReader.java?rev=711207&r1=711206&r2=711207&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HalfMapFileReader.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HalfMapFileReader.java Tue Nov 4 00:28:49 2008
@@ -48,8 +48,6 @@
private final boolean top;
private final HStoreKey midkey;
private boolean firstNextCall = true;
- private final WritableComparable<HStoreKey> firstKey;
- private final WritableComparable<HStoreKey> finalKey;
public HalfMapFileReader(final FileSystem fs, final String dirName,
final Configuration conf, final Range r,
@@ -76,18 +74,6 @@
this.midkey.setHRegionInfo(hri);
// Is it top or bottom half?
this.top = Reference.isTopFileRegion(r);
- // Firstkey comes back null if passed midkey is lower than first key in file
- // and its a bottom half HalfMapFileReader, OR, if midkey is higher than
- // the final key in the backing file. In either case, it means this half
- // file is empty.
- this.firstKey = this.top?
- super.getClosest(this.midkey, new ImmutableBytesWritable()):
- super.getFirstKey().compareTo(this.midkey) > 0?
- null: super.getFirstKey();
- this.finalKey = this.top?
- super.getFinalKey():
- super.getClosest(new HStoreKey.BeforeThisStoreKey(this.midkey),
- new ImmutableBytesWritable(), true);
}
/*
@@ -108,12 +94,16 @@
}
}
- @SuppressWarnings("unchecked")
@Override
public synchronized void finalKey(WritableComparable key)
throws IOException {
- Writables.copyWritable(this.finalKey, key);
- return;
+ if (top) {
+ super.finalKey(key);
+ } else {
+ Writable value = new ImmutableBytesWritable();
+ WritableComparable found = super.getClosest(midkey, value, true);
+ Writables.copyWritable(found, key);
+ }
}
@SuppressWarnings("unchecked")
@@ -134,8 +124,8 @@
// If top, the lowest possible key is first key. Do not have to check
// what comes back from super getClosest. Will return exact match or
// greater.
- closest = (key.compareTo(getFirstKey()) < 0)?
- getClosest(getFirstKey(), val): super.getClosest(key, val);
+ closest = (key.compareTo(this.midkey) < 0)?
+ this.midkey: super.getClosest(key, val);
} else {
// We're serving bottom of the file.
if (key.compareTo(this.midkey) < 0) {
@@ -143,8 +133,7 @@
closest = super.getClosest(key, val);
// midkey was made against largest store file at time of split. Smaller
// store files could have anything in them. Check return value is
- // not beyond the midkey (getClosest returns exact match or next
- // after).
+ // not beyond the midkey (getClosest returns exact match or next after)
if (closest != null && closest.compareTo(this.midkey) >= 0) {
// Don't let this value out.
closest = null;
@@ -168,19 +157,17 @@
throws IOException {
if (firstNextCall) {
firstNextCall = false;
- if (isEmpty()) {
- return false;
- }
- // Seek and fill by calling getClosest on first key.
- if (this.firstKey != null) {
- Writables.copyWritable(this.firstKey, key);
- WritableComparable nearest = getClosest(key, val);
- if (!key.equals(nearest)) {
- throw new IOException("Keys don't match and should: " +
- key.toString() + ", " + nearest.toString());
+ if (this.top) {
+ // Seek to midkey. Midkey may not exist in this file. That should be
+ // fine. Then we'll either be positioned at end or start of file.
+ WritableComparable nearest = getClosest(this.midkey, val);
+ // Now copy the midkey into the passed key.
+ if (nearest != null) {
+ Writables.copyWritable(nearest, key);
+ return true;
}
+ return false;
}
- return true;
}
boolean result = super.next(key, val);
if (!top && key.compareTo(midkey) >= 0) {
@@ -189,15 +176,10 @@
return result;
}
- private boolean isEmpty() {
- return this.firstKey == null;
- }
-
@Override
public synchronized void reset() throws IOException {
if (top) {
firstNextCall = true;
- // I don't think this is needed. seek(this.firstKey);
return;
}
super.reset();
@@ -210,19 +192,4 @@
checkKey(key);
return super.seek(key);
}
-
- @Override
- public HStoreKey getFirstKey() {
- return (HStoreKey)this.firstKey;
- }
-
- @Override
- public HStoreKey getFinalKey() {
- return (HStoreKey)this.finalKey;
- }
-
- @Override
- public String toString() {
- return super.toString() + ", half=" + (top? "top": "bottom");
- }
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=711207&r1=711206&r2=711207&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Tue Nov 4 00:28:49 2008
@@ -1371,7 +1371,6 @@
i >= 0 && !hasEnoughVersions(versions, results); i--) {
MapFile.Reader map = maparray[i];
synchronized(map) {
- map.reset();
// Do the priming read
ImmutableBytesWritable readval = new ImmutableBytesWritable();
HStoreKey readkey = (HStoreKey)map.getClosest(key, readval);
@@ -1379,7 +1378,9 @@
// map.getClosest returns null if the passed key is > than the
// last key in the map file. getClosest is a bit of a misnomer
// since it returns exact match or the next closest key AFTER not
- // BEFORE.
+ // BEFORE. We use getClosest because we're usually passed a
+ // key that has a timestamp of maximum long to indicate we want
+ // most recent update.
continue;
}
if (!readkey.matchesRowCol(key)) {
@@ -1868,9 +1869,6 @@
* @throws IOException
*/
private HStoreKey getFinalKey(final MapFile.Reader mf) throws IOException {
- if (mf instanceof HBaseMapFile.HBaseReader) {
- return ((HBaseMapFile.HBaseReader)mf).getFinalKey();
- }
HStoreKey finalKey = new HStoreKey();
mf.finalKey(finalKey);
finalKey.setHRegionInfo(this.info);
@@ -1961,16 +1959,18 @@
// Cast to HbaseReader.
HBaseMapFile.HBaseReader r =
(HBaseMapFile.HBaseReader)this.readers.get(mapIndex);
-
- // get the midkey
+ // Get first, last, and mid keys.
+ r.reset();
+ HStoreKey firstKey = new HStoreKey();
+ HStoreKey lastKey = new HStoreKey();
+ r.next(firstKey, new ImmutableBytesWritable());
+ r.finalKey(lastKey);
HStoreKey mk = (HStoreKey)r.midKey();
if (mk != null) {
// if the midkey is the same as the first and last keys, then we cannot
// (ever) split this region.
- if (HStoreKey.equalsTwoRowKeys(info, mk.getRow(),
- r.getFirstKey().getRow()) &&
- HStoreKey.equalsTwoRowKeys(info, mk.getRow(),
- r.getFinalKey().getRow())) {
+ if (HStoreKey.equalsTwoRowKeys(info, mk.getRow(), firstKey.getRow()) &&
+ HStoreKey.equalsTwoRowKeys(info, mk.getRow(), lastKey.getRow())) {
return null;
}
return new StoreSize(maxSize, mk.getRow());
@@ -2070,4 +2070,4 @@
HRegionInfo getHRegionInfo() {
return this.info;
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java?rev=711207&r1=711206&r2=711207&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java Tue Nov 4 00:28:49 2008
@@ -23,17 +23,16 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HStoreKey;
-import org.apache.hadoop.hbase.io.HBaseMapFile;
import org.apache.hadoop.hbase.io.HalfMapFileReader;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.WritableComparable;
@@ -117,25 +116,6 @@
}
}
- public void testContains() throws Exception {
- HStoreFile hsf = new HStoreFile(this.conf, this.fs, this.dir,
- HRegionInfo.FIRST_META_REGIONINFO,
- Bytes.toBytes("colfamily"), 1234567890L, null);
- MapFile.Writer writer =
- hsf.getWriter(this.fs, SequenceFile.CompressionType.NONE, false, 0);
- writeStoreFile(writer);
- MapFile.Reader r = hsf.getReader(this.fs, false, false);
- HBaseMapFile.HBaseReader reader =
- (HBaseMapFile.HBaseReader)r;
- // Store file should contain 'aa' and 'bb' but not 'AA' nor 'ZZ'.
- assertTrue(reader.containsKey(new HStoreKey("aa", "bb")));
- assertTrue(reader.containsKey(new HStoreKey("bb")));
- assertTrue(reader.containsKey(new HStoreKey("zz")));
- assertFalse(reader.containsKey(new HStoreKey("AA")));
- assertFalse(reader.containsKey(new HStoreKey("{{")));
- assertFalse(reader.containsKey(new HStoreKey()));
- }
-
/**
* Test that our mechanism of writing store files in one region to reference
* store files in other regions works.
@@ -192,13 +172,6 @@
}
}
assertTrue(Bytes.equals(key.getRow(), finalKey));
- // Assert contains works properly.
- HBaseMapFile.HBaseReader hbaseMapfileHalfReader =
- (HBaseMapFile.HBaseReader)halfReader;
- assertTrue(hbaseMapfileHalfReader.containsKey(midkey));
- assertTrue(hbaseMapfileHalfReader.containsKey(new HStoreKey(finalKey)));
- assertFalse(hbaseMapfileHalfReader.containsKey(new HStoreKey("aa")));
- assertFalse(hbaseMapfileHalfReader.containsKey(new HStoreKey("{{")));
}
/**