svn commit: r1181387 - /hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java

[ns...@apache.org]

Author: nspiegelberg
Date: Tue Oct 11 02:04:21 2011
New Revision: 1181387

URL: http://svn.apache.org/viewvc?rev=1181387&view=rev
Log:
apply HBASE 3040: BlockIndex readIndex too slowly in heavy write scenario

Summary:
Applied patches from HBASE 3040.

Test Plan:
going to fire unit tests

DiffCamp Revision: 163501
Reviewed By: kannan
CC: kannan, subbu
Revert Plan:
OK

Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1181387&r1=1181386&r2=1181387&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Oct 11 02:04:21 2011
@@ -23,6 +23,7 @@ import java.io.BufferedInputStream;
 import java.io.Closeable;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -832,6 +833,16 @@ public class HFile {
       return this.inMemory;
     }
 
+	private byte[] readAllIndex(final FSDataInputStream in, final long indexOffset,
+			final int indexSize) throws IOException
+	{
+		byte[] allIndex = new byte[indexSize];
+		in.seek(indexOffset);
+		IOUtils.readFully(in, allIndex, 0, allIndex.length);
+
+		return allIndex;
+	}
+
     /**
      * Read in the index and file info.
      * @return A map of fileinfo data.
@@ -852,16 +863,28 @@ public class HFile {
       String clazzName = Bytes.toString(fi.get(FileInfo.COMPARATOR));
       this.comparator = getComparator(clazzName);
 
+	  int allIndexSize = (int)(this.fileSize - this.trailer.dataIndexOffset - FixedFileTrailer.trailerSize());
+	  byte[] dataAndMetaIndex = readAllIndex(this.istream, this.trailer.dataIndexOffset, allIndexSize);
+
+	  ByteArrayInputStream bis = new ByteArrayInputStream(dataAndMetaIndex);
+	  DataInputStream dis = new DataInputStream(bis);
+
       // Read in the data index.
-      this.blockIndex = BlockIndex.readIndex(this.comparator, this.istream,
-        this.trailer.dataIndexOffset, this.trailer.dataIndexCount);
+    this.blockIndex =
+      BlockIndex.readIndex(this.comparator, dis, this.trailer.dataIndexCount);
 
       // Read in the metadata index.
       if (trailer.metaIndexCount > 0) {
-        this.metaIndex = BlockIndex.readIndex(Bytes.BYTES_RAWCOMPARATOR,
-          this.istream, this.trailer.metaIndexOffset, trailer.metaIndexCount);
+		  this.metaIndex = BlockIndex.readIndex(Bytes.BYTES_RAWCOMPARATOR,
+				  dis, this.trailer.metaIndexCount);
       }
       this.fileInfoLoaded = true;
+
+	  if (null != dis)
+	  {
+		  dis.close();
+	  }
+
       return fi;
     }
 
@@ -1684,12 +1707,13 @@ public class HFile {
     /*
      * Read in the index that is at <code>indexOffset</code>
      * Must match what was written by writeIndex in the Writer.close.
+     * @param c Comparator to use.
      * @param in
-     * @param indexOffset
+     * @param indexSize
      * @throws IOException
      */
     static BlockIndex readIndex(final RawComparator<byte []> c,
-        final FSDataInputStream in, final long indexOffset, final int indexSize)
+        DataInputStream in, final int indexSize)
     throws IOException {
       BlockIndex bi = new BlockIndex(c);
       bi.blockOffsets = new long[indexSize];
@@ -1697,9 +1721,8 @@ public class HFile {
       bi.blockDataSizes = new int[indexSize];
       // If index size is zero, no index was written.
       if (indexSize > 0) {
-        in.seek(indexOffset);
         byte [] magic = new byte[INDEXBLOCKMAGIC.length];
-        IOUtils.readFully(in, magic, 0, magic.length);
+        in.readFully(magic);
         if (!Arrays.equals(magic, INDEXBLOCKMAGIC)) {
           throw new IOException("Index block magic is wrong: " +
             Arrays.toString(magic));