You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2012/06/20 19:34:52 UTC
svn commit: r1352221 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/regionserver/
Author: ramkrishna
Date: Wed Jun 20 17:34:51 2012
New Revision: 1352221
URL: http://svn.apache.org/viewvc?rev=1352221&view=rev
Log:
HBASE-6164 Correct the bug in block encoding usage in bulkload
Submitted by:Anoop
Reviewed by:Ted
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java Wed Jun 20 17:34:51 2012
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
/**
@@ -29,6 +29,9 @@ import org.apache.hadoop.hbase.util.Pair
* should just return the unmodified block.
*/
public interface HFileDataBlockEncoder {
+ /** Type of encoding used for data blocks in HFile. Stored in file info. */
+ public static final byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
+
/**
* Converts a block from the on-disk format to the in-cache format. Called in
* the following cases:
@@ -64,12 +67,11 @@ public interface HFileDataBlockEncoder {
public boolean useEncodedScanner(boolean isCompaction);
/**
- * Save metadata in StoreFile which will be written to disk
- * @param storeFileWriter writer for a given StoreFile
+ * Save metadata in HFile which will be written to disk
+ * @param writer writer for a given HFile
* @exception IOException on disk problems
*/
- public void saveMetadata(StoreFile.Writer storeFileWriter)
- throws IOException;
+ public void saveMetadata(HFile.Writer writer) throws IOException;
/** @return the on-disk data block encoding */
public DataBlockEncoding getEncodingOnDisk();
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java Wed Jun 20 17:34:51 2012
@@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.io.encodi
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -70,7 +69,7 @@ public class HFileDataBlockEncoderImpl i
boolean hasPreferredCacheEncoding = preferredEncodingInCache != null
&& preferredEncodingInCache != DataBlockEncoding.NONE;
- byte[] dataBlockEncodingType = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+ byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
if (dataBlockEncodingType == null && !hasPreferredCacheEncoding) {
return NoOpDataBlockEncoder.INSTANCE;
}
@@ -105,10 +104,8 @@ public class HFileDataBlockEncoderImpl i
}
@Override
- public void saveMetadata(StoreFile.Writer storeFileWriter)
- throws IOException {
- storeFileWriter.appendFileInfo(StoreFile.DATA_BLOCK_ENCODING,
- onDisk.getNameInBytes());
+ public void saveMetadata(HFile.Writer writer) throws IOException {
+ writer.appendFileInfo(DATA_BLOCK_ENCODING, onDisk.getNameInBytes());
}
@Override
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java Wed Jun 20 17:34:51 2012
@@ -332,6 +332,8 @@ public class HFileWriterV1 extends Abstr
if (this.outputStream == null) {
return;
}
+ // Save data block encoder metadata in the file info.
+ blockEncoder.saveMetadata(this);
// Write out the end of the data blocks, then write meta data blocks.
// followed by fileinfo, data block index and meta block index.
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Wed Jun 20 17:34:51 2012
@@ -356,6 +356,8 @@ public class HFileWriterV2 extends Abstr
if (outputStream == null) {
return;
}
+ // Save data block encoder metadata in the file info.
+ blockEncoder.saveMetadata(this);
// Write out the end of the data blocks, then write meta data blocks.
// followed by fileinfo, data block index and meta block index.
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java Wed Jun 20 17:34:51 2012
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.io.hfile
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.Pair;
/**
@@ -54,7 +53,7 @@ public class NoOpDataBlockEncoder implem
}
@Override
- public void saveMetadata(StoreFile.Writer storeFileWriter) {
+ public void saveMetadata(HFile.Writer writer) {
}
@Override
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Wed Jun 20 17:34:51 2012
@@ -135,10 +135,6 @@ public class StoreFile extends SchemaCon
/** Key for timestamp of earliest-put in metadata*/
public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
- /** Type of encoding used for data blocks in HFile. Stored in file info. */
- public static final byte[] DATA_BLOCK_ENCODING =
- Bytes.toBytes("DATA_BLOCK_ENCODING");
-
// Make default block size for StoreFiles 8k while testing. TODO: FIX!
// Need to make it 8k for testing.
public static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
@@ -1205,9 +1201,6 @@ public class StoreFile extends SchemaCon
}
public void close() throws IOException {
- // Save data block encoder metadata in the file info.
- dataBlockEncoder.saveMetadata(this);
-
boolean hasGeneralBloom = this.closeGeneralBloomFilter();
boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Wed Jun 20 17:34:51 2012
@@ -916,7 +916,7 @@ public class TestStoreFile extends HBase
StoreFile.Reader reader = storeFile.createReader();
Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
- byte[] value = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+ byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
}