You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2010/05/20 00:19:25 UTC
svn commit: r946464 [2/2] - in /hbase/trunk: ./
core/src/main/java/org/apache/hadoop/hbase/
core/src/main/java/org/apache/hadoop/hbase/io/hfile/
core/src/main/java/org/apache/hadoop/hbase/mapreduce/
core/src/main/java/org/apache/hadoop/hbase/regionserv...
Modified: hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=946464&r1=946463&r2=946464&view=diff
==============================================================================
--- hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Wed May 19 22:19:24 2010
@@ -21,10 +21,13 @@ package org.apache.hadoop.hbase.regionse
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
@@ -69,11 +72,11 @@ public class TestStoreFile extends HBase
*/
public void testBasicHalfMapFile() throws Exception {
// Make up a directory hierarchy that has a regiondir and familyname.
- HFile.Writer writer = StoreFile.getWriter(this.fs,
- new Path(new Path(this.testDir, "regionname"), "familyname"),
- 2 * 1024, null, null);
+ HFile.Writer writer = StoreFile.createWriter(this.fs,
+ new Path(new Path(this.testDir, "regionname"), "familyname"), 2 * 1024);
writeStoreFile(writer);
- checkHalfHFile(new StoreFile(this.fs, writer.getPath(), true, conf, false));
+ checkHalfHFile(new StoreFile(this.fs, writer.getPath(), true, conf,
+ StoreFile.BloomType.NONE, false));
}
/*
@@ -109,11 +112,11 @@ public class TestStoreFile extends HBase
Path storedir = new Path(new Path(this.testDir, "regionname"), "familyname");
Path dir = new Path(storedir, "1234567890");
// Make a store file and write data to it.
- HFile.Writer writer = StoreFile.getWriter(this.fs, dir, 8 * 1024, null,
- null);
+ HFile.Writer writer = StoreFile.createWriter(this.fs, dir, 8 * 1024);
writeStoreFile(writer);
- StoreFile hsf = new StoreFile(this.fs, writer.getPath(), true, conf, false);
- HFile.Reader reader = hsf.getReader();
+ StoreFile hsf = new StoreFile(this.fs, writer.getPath(), true, conf,
+ StoreFile.BloomType.NONE, false);
+ HFile.Reader reader = hsf.createReader();
// Split on a row, not in middle of row. Midkey returned by reader
// may be in middle of row. Create new one with empty column and
// timestamp.
@@ -123,10 +126,11 @@ public class TestStoreFile extends HBase
byte [] finalRow = kv.getRow();
// Make a reference
Path refPath = StoreFile.split(fs, dir, hsf, midRow, Range.top);
- StoreFile refHsf = new StoreFile(this.fs, refPath, true, conf, false);
+ StoreFile refHsf = new StoreFile(this.fs, refPath, true, conf,
+ StoreFile.BloomType.NONE, false);
// Now confirm that I can read from the reference and that it only gets
// keys from top half of the file.
- HFileScanner s = refHsf.getReader().getScanner(false, false);
+ HFileScanner s = refHsf.createReader().getScanner(false, false);
for(boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
ByteBuffer bb = s.getKey();
kv = KeyValue.createKeyValueFromKey(bb);
@@ -140,7 +144,7 @@ public class TestStoreFile extends HBase
private void checkHalfHFile(final StoreFile f)
throws IOException {
- byte [] midkey = f.getReader().midkey();
+ byte [] midkey = f.createReader().midkey();
KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
byte [] midRow = midKV.getRow();
// Create top split.
@@ -159,8 +163,10 @@ public class TestStoreFile extends HBase
Path bottomPath = StoreFile.split(this.fs, bottomDir,
f, midRow, Range.bottom);
// Make readers on top and bottom.
- HFile.Reader top = new StoreFile(this.fs, topPath, true, conf, false).getReader();
- HFile.Reader bottom = new StoreFile(this.fs, bottomPath, true, conf, false).getReader();
+ HFile.Reader top = new StoreFile(this.fs, topPath, true, conf,
+ StoreFile.BloomType.NONE, false).createReader();
+ HFile.Reader bottom = new StoreFile(this.fs, bottomPath, true, conf,
+ StoreFile.BloomType.NONE, false).createReader();
ByteBuffer previous = null;
LOG.info("Midkey: " + midKV.toString());
ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midkey);
@@ -212,8 +218,10 @@ public class TestStoreFile extends HBase
topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
Range.bottom);
- top = new StoreFile(this.fs, topPath, true, conf, false).getReader();
- bottom = new StoreFile(this.fs, bottomPath, true, conf, false).getReader();
+ top = new StoreFile(this.fs, topPath, true, conf,
+ StoreFile.BloomType.NONE, false).createReader();
+ bottom = new StoreFile(this.fs, bottomPath, true, conf,
+ StoreFile.BloomType.NONE, false).createReader();
bottomScanner = bottom.getScanner(false, false);
int count = 0;
while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
@@ -256,8 +264,10 @@ public class TestStoreFile extends HBase
topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
Range.bottom);
- top = new StoreFile(this.fs, topPath, true, conf, false).getReader();
- bottom = new StoreFile(this.fs, bottomPath, true, conf, false).getReader();
+ top = new StoreFile(this.fs, topPath, true, conf,
+ StoreFile.BloomType.NONE, false).createReader();
+ bottom = new StoreFile(this.fs, bottomPath, true, conf,
+ StoreFile.BloomType.NONE, false).createReader();
first = true;
bottomScanner = bottom.getScanner(false, false);
while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
@@ -296,4 +306,138 @@ public class TestStoreFile extends HBase
fs.delete(f.getPath(), true);
}
}
-}
\ No newline at end of file
+
+ private static String ROOT_DIR =
+ System.getProperty("test.build.data", "/tmp/TestStoreFile");
+ private static String localFormatter = "%010d";
+
+ public void testBloomFilter() throws Exception {
+ FileSystem fs = FileSystem.getLocal(conf);
+ conf.setFloat("io.hfile.bloom.error.rate", (float)0.01);
+ conf.setBoolean("io.hfile.bloom.enabled", true);
+
+ // write the file
+ Path f = new Path(ROOT_DIR, getName());
+ StoreFile.Writer writer = new StoreFile.Writer(fs, f,
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
+ conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
+
+ long now = System.currentTimeMillis();
+ for (int i = 0; i < 2000; i += 2) {
+ String row = String.format(localFormatter, Integer.valueOf(i));
+ KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(),
+ "col".getBytes(), now, "value".getBytes());
+ writer.append(kv);
+ }
+ writer.close();
+
+ StoreFile.Reader reader = new StoreFile.Reader(fs, f, null, false);
+ reader.loadFileInfo();
+ reader.loadBloomfilter();
+ HFileScanner scanner = reader.getScanner(false, false);
+
+ // check false positives rate
+ int falsePos = 0;
+ int falseNeg = 0;
+ for (int i = 0; i < 2000; i++) {
+ String row = String.format(localFormatter, Integer.valueOf(i));
+ TreeSet<byte[]> columns = new TreeSet<byte[]>();
+ columns.add("family:col".getBytes());
+
+ boolean exists = scanner.shouldSeek(row.getBytes(), columns);
+ if (i % 2 == 0) {
+ if (!exists) falseNeg++;
+ } else {
+ if (exists) falsePos++;
+ }
+ }
+ reader.close();
+ fs.delete(f, true);
+ System.out.println("False negatives: " + falseNeg);
+ assertEquals(0, falseNeg);
+ System.out.println("False positives: " + falsePos);
+ assertTrue(falsePos < 2);
+ }
+
+ public void testBloomTypes() throws Exception {
+ float err = (float) 0.01;
+ FileSystem fs = FileSystem.getLocal(conf);
+ conf.setFloat("io.hfile.bloom.error.rate", err);
+ conf.setBoolean("io.hfile.bloom.enabled", true);
+
+ int rowCount = 50;
+ int colCount = 10;
+ int versions = 2;
+
+ // run once using columns and once using rows
+ StoreFile.BloomType[] bt =
+ {StoreFile.BloomType.ROWCOL, StoreFile.BloomType.ROW};
+ int[] expKeys = {rowCount*colCount, rowCount};
+ // below line deserves commentary. it is expected bloom false positives
+ // column = rowCount*2*colCount inserts
+ // row-level = only rowCount*2 inserts, but failures will be magnified by
+ // 2nd for loop for every column (2*colCount)
+ float[] expErr = {2*rowCount*colCount*err, 2*rowCount*2*colCount*err};
+
+ for (int x : new int[]{0,1}) {
+ // write the file
+ Path f = new Path(ROOT_DIR, getName());
+ StoreFile.Writer writer = new StoreFile.Writer(fs, f,
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL,
+ HFile.DEFAULT_COMPRESSION_ALGORITHM,
+ conf, KeyValue.COMPARATOR, bt[x], expKeys[x]);
+
+ long now = System.currentTimeMillis();
+ for (int i = 0; i < rowCount*2; i += 2) { // rows
+ for (int j = 0; j < colCount*2; j += 2) { // column qualifiers
+ String row = String.format(localFormatter, Integer.valueOf(i));
+ String col = String.format(localFormatter, Integer.valueOf(j));
+ for (int k= 0; k < versions; ++k) { // versions
+ KeyValue kv = new KeyValue(row.getBytes(),
+ "family".getBytes(), ("col" + col).getBytes(),
+ now-k, Bytes.toBytes((long)-1));
+ writer.append(kv);
+ }
+ }
+ }
+ writer.close();
+
+ StoreFile.Reader reader = new StoreFile.Reader(fs, f, null, false);
+ reader.loadFileInfo();
+ reader.loadBloomfilter();
+ HFileScanner scanner = reader.getScanner(false, false);
+ assertEquals(expKeys[x], reader.getBloomFilter().getKeyCount());
+
+ // check false positives rate
+ int falsePos = 0;
+ int falseNeg = 0;
+ for (int i = 0; i < rowCount*2; ++i) { // rows
+ for (int j = 0; j < colCount*2; ++j) { // column qualifiers
+ String row = String.format(localFormatter, Integer.valueOf(i));
+ String col = String.format(localFormatter, Integer.valueOf(j));
+ TreeSet<byte[]> columns = new TreeSet<byte[]>();
+ columns.add(("col" + col).getBytes());
+
+ boolean exists = scanner.shouldSeek(row.getBytes(), columns);
+ boolean shouldRowExist = i % 2 == 0;
+ boolean shouldColExist = j % 2 == 0;
+ shouldColExist = shouldColExist || bt[x] == StoreFile.BloomType.ROW;
+ if (shouldRowExist && shouldColExist) {
+ if (!exists) falseNeg++;
+ } else {
+ if (exists) falsePos++;
+ }
+ }
+ }
+ reader.close();
+ fs.delete(f, true);
+ System.out.println(bt[x].toString());
+ System.out.println(" False negatives: " + falseNeg);
+ System.out.println(" False positives: " + falsePos);
+ assertEquals(0, falseNeg);
+ assertTrue(falsePos < 2*expErr[x]);
+ }
+
+ }
+
+}
Modified: hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java?rev=946464&r1=946463&r2=946464&view=diff
==============================================================================
--- hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java (original)
+++ hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java Wed May 19 22:19:24 2010
@@ -52,7 +52,8 @@ public class TestWideScanner extends HBa
TESTTABLEDESC.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY,
10, // Ten is arbitrary number. Keep versions to help debuggging.
Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
- HConstants.FOREVER, false, HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
+ HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
+ HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
}
/** HRegionInfo for root region */
public static final HRegionInfo REGION_INFO =
Modified: hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java
URL: http://svn.apache.org/viewvc/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java?rev=946464&r1=946463&r2=946464&view=diff
==============================================================================
--- hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java (original)
+++ hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java Wed May 19 22:19:24 2010
@@ -33,7 +33,7 @@ public class TestColumnSchemaModel exten
protected static final String COLUMN_NAME = "testcolumn";
protected static final boolean BLOCKCACHE = true;
protected static final int BLOCKSIZE = 16384;
- protected static final boolean BLOOMFILTER = false;
+ protected static final String BLOOMFILTER = "none";
protected static final String COMPRESSION = "GZ";
protected static final boolean IN_MEMORY = false;
protected static final int TTL = 86400;
@@ -42,7 +42,7 @@ public class TestColumnSchemaModel exten
protected static final String AS_XML =
"<ColumnSchema name=\"testcolumn\"" +
" BLOCKSIZE=\"16384\"" +
- " BLOOMFILTER=\"false\"" +
+ " BLOOMFILTER=\"none\"" +
" BLOCKCACHE=\"true\"" +
" COMPRESSION=\"GZ\"" +
" VERSIONS=\"1\"" +