You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/02/28 00:37:54 UTC
svn commit: r1294395 [2/2] - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/mapreduce/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/util/ test/java/org/...
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1294395&r1=1294394&r2=1294395&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Mon Feb 27 23:37:53 2012
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
@@ -101,9 +100,12 @@ public class TestStoreFile extends HBase
*/
public void testBasicHalfMapFile() throws Exception {
// Make up a directory hierarchy that has a regiondir and familyname.
- StoreFile.Writer writer = StoreFile.createWriter(this.fs,
- new Path(new Path(this.testDir, "regionname"), "familyname"), 2 * 1024,
- conf, cacheConf);
+ Path outputDir = new Path(new Path(this.testDir, "regionname"),
+ "familyname");
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
+ this.fs, 2 * 1024)
+ .withOutputDir(outputDir)
+ .build();
writeStoreFile(writer);
checkHalfHFile(new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE));
@@ -143,8 +145,10 @@ public class TestStoreFile extends HBase
Path storedir = new Path(new Path(this.testDir, "regionname"), "familyname");
Path dir = new Path(storedir, "1234567890");
// Make a store file and write data to it.
- StoreFile.Writer writer = StoreFile.createWriter(this.fs, dir, 8 * 1024,
- conf, cacheConf);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
+ this.fs, 8 * 1024)
+ .withOutputDir(dir)
+ .build();
writeStoreFile(writer);
StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
@@ -404,10 +408,12 @@ public class TestStoreFile extends HBase
// write the file
Path f = new Path(ROOT_DIR, getName());
- StoreFile.Writer writer =
- new StoreFile.Writer(fs, f, StoreFile.DEFAULT_BLOCKSIZE_SMALL,
- HFile.DEFAULT_COMPRESSION_ALGORITHM, conf, cacheConf,
- KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL)
+ .withFilePath(f)
+ .withBloomType(StoreFile.BloomType.ROW)
+ .withMaxKeyCount(2000)
+ .build();
bloomWriteRead(writer, fs);
}
@@ -421,10 +427,11 @@ public class TestStoreFile extends HBase
// write the file
Path f = new Path(ROOT_DIR, getName());
- StoreFile.Writer writer = new StoreFile.Writer(fs, f,
- StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE,
- 2000);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
+ fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
+ .withFilePath(f)
+ .withMaxKeyCount(2000)
+ .build();
// add delete family
long now = System.currentTimeMillis();
@@ -489,10 +496,12 @@ public class TestStoreFile extends HBase
for (int x : new int[]{0,1}) {
// write the file
Path f = new Path(ROOT_DIR, getName() + x);
- StoreFile.Writer writer = new StoreFile.Writer(fs, f,
- StoreFile.DEFAULT_BLOCKSIZE_SMALL,
- HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, bt[x], expKeys[x]);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
+ fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
+ .withFilePath(f)
+ .withBloomType(bt[x])
+ .withMaxKeyCount(expKeys[x])
+ .build();
long now = System.currentTimeMillis();
for (int i = 0; i < rowCount*2; i += 2) { // rows
@@ -562,10 +571,12 @@ public class TestStoreFile extends HBase
conf.setInt(HFile.FORMAT_VERSION_KEY, 1);
// this should not create a bloom because the max keys is too small
- StoreFile.Writer writer = new StoreFile.Writer(fs, f,
- StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW,
- 2000);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL)
+ .withFilePath(f)
+ .withBloomType(StoreFile.BloomType.ROW)
+ .withMaxKeyCount(2000)
+ .build();
assertFalse(writer.hasGeneralBloom());
writer.close();
fs.delete(f, true);
@@ -573,23 +584,23 @@ public class TestStoreFile extends HBase
conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_MAX_KEYS,
Integer.MAX_VALUE);
- // TODO: commented out because we ran out of java heap space on trunk
- /*
- // the below config caused IllegalArgumentException in our production cluster
- // however, the resulting byteSize is < MAX_INT, so this should work properly
- writer = new StoreFile.Writer(fs, f,
- StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 272446963);
- assertTrue(writer.hasBloom());
+ writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL)
+ .withFilePath(f)
+ .withBloomType(StoreFile.BloomType.ROW)
+ .withMaxKeyCount(27244696)
+ .build();
+ assertTrue(writer.hasGeneralBloom());
bloomWriteRead(writer, fs);
- */
// this, however, is too large and should not create a bloom
// because Java can't create a contiguous array > MAX_INT
- writer = new StoreFile.Writer(fs, f,
- StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW,
- Integer.MAX_VALUE);
+ writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL)
+ .withFilePath(f)
+ .withBloomType(StoreFile.BloomType.ROW)
+ .withMaxKeyCount(Integer.MAX_VALUE)
+ .build();
assertFalse(writer.hasGeneralBloom());
writer.close();
fs.delete(f, true);
@@ -680,8 +691,10 @@ public class TestStoreFile extends HBase
Path storedir = new Path(new Path(this.testDir, "regionname"),
"familyname");
Path dir = new Path(storedir, "1234567890");
- StoreFile.Writer writer = StoreFile.createWriter(this.fs, dir, 8 * 1024,
- conf, cacheConf);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
+ this.fs, 8 * 1024)
+ .withOutputDir(dir)
+ .build();
List<KeyValue> kvList = getKeyValueSet(timestamps,numRows,
family, qualifier);
@@ -853,10 +866,11 @@ public class TestStoreFile extends HBase
totalSize += kv.getLength() + 1;
}
int blockSize = totalSize / numBlocks;
- StoreFile.Writer writer = new StoreFile.Writer(fs, path, blockSize,
- HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE,
- 2000);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
+ blockSize)
+ .withFilePath(path)
+ .withMaxKeyCount(2000)
+ .build();
// We'll write N-1 KVs to ensure we don't write an extra block
kvs.remove(kvs.size()-1);
for (KeyValue kv : kvs) {
@@ -882,15 +896,12 @@ public class TestStoreFile extends HBase
dataBlockEncoderAlgo,
dataBlockEncoderAlgo);
cacheConf = new CacheConfig(conf);
- StoreFile.Writer writer = new StoreFile.Writer(fs,
- path, HFile.DEFAULT_BLOCKSIZE,
- HFile.DEFAULT_COMPRESSION_ALGORITHM,
- dataBlockEncoder,
- conf,
- cacheConf,
- KeyValue.COMPARATOR,
- StoreFile.BloomType.NONE,
- HColumnDescriptor.DEFAULT_BLOOMFILTER_ERROR_RATE, 2000, null);
+ StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
+ HFile.DEFAULT_BLOCKSIZE)
+ .withFilePath(path)
+ .withDataBlockEncoder(dataBlockEncoder)
+ .withMaxKeyCount(2000)
+ .build();
writer.close();
StoreFile storeFile = new StoreFile(fs, writer.getPath(), conf,
@@ -902,4 +913,4 @@ public class TestStoreFile extends HBase
assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
}
-}
+}
\ No newline at end of file
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java?rev=1294395&r1=1294394&r2=1294395&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java Mon Feb 27 23:37:53 2012
@@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -188,7 +187,8 @@ public class TestWALReplay {
HLog wal = createWAL(this.conf);
HRegion region = HRegion.openHRegion(hri, basedir, wal, this.conf);
Path f = new Path(basedir, "hfile");
- HFile.Writer writer = HFile.getWriterFactory(conf).createWriter(this.fs, f);
+ HFile.Writer writer =
+ HFile.getWriterFactoryNoCache(conf).withPath(fs, f).create();
byte [] family = hri.getTableDesc().getFamilies().iterator().next().getName();
byte [] row = Bytes.toBytes(tableNameStr);
writer.append(new KeyValue(row, family, family, row));
@@ -550,4 +550,4 @@ public class TestWALReplay {
HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
return wal;
}
-}
+}
\ No newline at end of file