You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2012/05/31 06:27:11 UTC
svn commit: r1344561 -
/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
Author: stack
Date: Thu May 31 04:27:11 2012
New Revision: 1344561
URL: http://svn.apache.org/viewvc?rev=1344561&view=rev
Log:
HBASE-6040 Use block encoding and HBase handled checksum verification in bulk loading using HFileOutputFormat
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java?rev=1344561&r1=1344560&r2=1344561&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java Thu May 31 04:27:11 2012
@@ -46,9 +46,14 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
import org.apache.hadoop.hbase.util.Bytes;
@@ -75,6 +80,8 @@ public class HFileOutputFormat extends F
static Log LOG = LogFactory.getLog(HFileOutputFormat.class);
static final String COMPRESSION_CONF_KEY = "hbase.hfileoutputformat.families.compression";
TimeRangeTracker trt = new TimeRangeTracker();
+ private static final String DATABLOCK_ENCODING_CONF_KEY =
+ "hbase.mapreduce.hfileoutputformat.datablock.encoding";
public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
throws IOException, InterruptedException {
@@ -96,6 +103,22 @@ public class HFileOutputFormat extends F
// create a map from column family to the compression algorithm
final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf);
+
+ String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY);
+ final HFileDataBlockEncoder encoder;
+ if (dataBlockEncodingStr == null) {
+ encoder = NoOpDataBlockEncoder.INSTANCE;
+ } else {
+ try {
+ encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding
+ .valueOf(dataBlockEncodingStr));
+ } catch (IllegalArgumentException ex) {
+ throw new RuntimeException(
+ "Invalid data block encoding type configured for the param "
+ + DATABLOCK_ENCODING_CONF_KEY + " : "
+ + dataBlockEncodingStr);
+ }
+ }
return new RecordWriter<ImmutableBytesWritable, KeyValue>() {
// Map of families to writers and how much has been output on the writer.
@@ -178,6 +201,9 @@ public class HFileOutputFormat extends F
.withBlockSize(blocksize)
.withCompression(compression)
.withComparator(KeyValue.KEY_COMPARATOR)
+ .withDataBlockEncoder(encoder)
+ .withChecksumType(Store.getChecksumType(conf))
+ .withBytesPerChecksum(Store.getBytesPerChecksum(conf))
.create();
this.writers.put(family, wl);
return wl;