You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2012/06/20 19:33:20 UTC
svn commit: r1352220 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/regionserver/
Author: ramkrishna
Date: Wed Jun 20 17:33:19 2012
New Revision: 1352220
URL: http://svn.apache.org/viewvc?rev=1352220&view=rev
Log:
HBASE=6164 Correct the bug in block encoding usage in bulkload
Submitted by: Anoop
Reviewed by: Ted
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java Wed Jun 20 17:33:19 2012
@@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.io.encodi
import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* Controls what kind of data block encoding is used. If data block encoding is
@@ -33,7 +33,9 @@ import org.apache.hadoop.hbase.regionser
*/
@InterfaceAudience.Private
public interface HFileDataBlockEncoder {
-
+ /** Type of encoding used for data blocks in HFile. Stored in file info. */
+ byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
+
/**
* Converts a block from the on-disk format to the in-cache format. Called in
* the following cases:
@@ -71,11 +73,11 @@ public interface HFileDataBlockEncoder {
public boolean useEncodedScanner(boolean isCompaction);
/**
- * Save metadata in StoreFile which will be written to disk
- * @param storeFileWriter writer for a given StoreFile
+ * Save metadata in HFile which will be written to disk
+ * @param writer writer for a given HFile
* @exception IOException on disk problems
*/
- public void saveMetadata(StoreFile.Writer storeFileWriter)
+ public void saveMetadata(HFile.Writer writer)
throws IOException;
/** @return the on-disk data block encoding */
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java Wed Jun 20 17:33:19 2012
@@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.io.encodi
import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.Bytes;
import com.google.common.base.Preconditions;
@@ -96,8 +95,7 @@ public class HFileDataBlockEncoderImpl i
public static HFileDataBlockEncoder createFromFileInfo(
FileInfo fileInfo, DataBlockEncoding preferredEncodingInCache)
throws IOException {
- byte[] dataBlockEncodingType =
- fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+ byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
if (dataBlockEncodingType == null) {
return NoOpDataBlockEncoder.INSTANCE;
}
@@ -128,10 +126,8 @@ public class HFileDataBlockEncoderImpl i
}
@Override
- public void saveMetadata(StoreFile.Writer storeFileWriter)
- throws IOException {
- storeFileWriter.appendFileInfo(StoreFile.DATA_BLOCK_ENCODING,
- onDisk.getNameInBytes());
+ public void saveMetadata(HFile.Writer writer) throws IOException {
+ writer.appendFileInfo(DATA_BLOCK_ENCODING, onDisk.getNameInBytes());
}
@Override
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java Wed Jun 20 17:33:19 2012
@@ -334,6 +334,8 @@ public class HFileWriterV1 extends Abstr
if (this.outputStream == null) {
return;
}
+ // Save data block encoder metadata in the file info.
+ blockEncoder.saveMetadata(this);
// Write out the end of the data blocks, then write meta data blocks.
// followed by fileinfo, data block index and meta block index.
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Wed Jun 20 17:33:19 2012
@@ -358,6 +358,8 @@ public class HFileWriterV2 extends Abstr
if (outputStream == null) {
return;
}
+ // Save data block encoder metadata in the file info.
+ blockEncoder.saveMetadata(this);
// Write out the end of the data blocks, then write meta data blocks.
// followed by fileinfo, data block index and meta block index.
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java Wed Jun 20 17:33:19 2012
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.io.encodi
import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
/**
* Does not perform any kind of encoding/decoding.
@@ -71,7 +70,7 @@ public class NoOpDataBlockEncoder implem
}
@Override
- public void saveMetadata(StoreFile.Writer storeFileWriter) {
+ public void saveMetadata(HFile.Writer writer) {
}
@Override
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Wed Jun 20 17:33:19 2012
@@ -137,10 +137,6 @@ public class StoreFile extends SchemaCon
/** Key for timestamp of earliest-put in metadata*/
public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
- /** Type of encoding used for data blocks in HFile. Stored in file info. */
- public static final byte[] DATA_BLOCK_ENCODING =
- Bytes.toBytes("DATA_BLOCK_ENCODING");
-
// Make default block size for StoreFiles 8k while testing. TODO: FIX!
// Need to make it 8k for testing.
public static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
@@ -1208,9 +1204,6 @@ public class StoreFile extends SchemaCon
}
public void close() throws IOException {
- // Save data block encoder metadata in the file info.
- dataBlockEncoder.saveMetadata(this);
-
boolean hasGeneralBloom = this.closeGeneralBloomFilter();
boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Wed Jun 20 17:33:19 2012
@@ -913,7 +913,7 @@ public class TestStoreFile extends HBase
StoreFile.Reader reader = storeFile.createReader();
Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
- byte[] value = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+ byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
}