You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/03/06 20:57:34 UTC
svn commit: r751053 - in /hadoop/hbase/branches/0.19: CHANGES.txt
src/java/org/apache/hadoop/hbase/io/MapFile.java
Author: stack
Date: Fri Mar 6 19:57:33 2009
New Revision: 751053
URL: http://svn.apache.org/viewvc?rev=751053&view=rev
Log:
HBASE-1229 Apply HADOOP-5369 to HBase MapFile
Modified:
hadoop/hbase/branches/0.19/CHANGES.txt
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java
Modified: hadoop/hbase/branches/0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/CHANGES.txt?rev=751053&r1=751052&r2=751053&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19/CHANGES.txt Fri Mar 6 19:57:33 2009
@@ -38,6 +38,7 @@
(Erik Holstad via Stack)
HBASE-1240 Would be nice if RowResult could be comparable
(Erik Holstad via Stack)
+ HBASE-1229 Apply HADOOP-5369 to HBase MapFile (Ben Maurer via Stack)
Release 0.19.0
INCOMPATIBLE CHANGES
Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java?rev=751053&r1=751052&r2=751053&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/io/MapFile.java Fri Mar 6 19:57:33 2009
@@ -18,6 +18,8 @@
package org.apache.hadoop.hbase.io;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.io.*;
import org.apache.commons.logging.Log;
@@ -82,6 +84,12 @@
private DataOutputBuffer outBuf = new DataOutputBuffer();
private WritableComparable lastKey;
+ /** What's the position (in bytes) we wrote when we got the last index */
+ private long lastIndexPos = 0;
+
+ /** What was size when we last wrote an index */
+ private long lastIndexKeyCount = 0;
+
/** Create the named map for keys of the named class. */
public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -198,10 +206,15 @@
throws IOException {
checkKey(key);
-
- if (size % indexInterval == 0) { // add an index entry
- position.set(data.getLength()); // point to current eof
+
+ long pos = data.getLength();
+ // Only write an index if we've changed positions. In a block compressed
+ // file, this means we write an entry at the start of each block
+ if (size >= lastIndexKeyCount + indexInterval && pos > lastIndexPos) {
+ position.set(pos); // point to current eof
index.append(key, position);
+ lastIndexPos = pos;
+ lastIndexKeyCount = size;
}
data.append(key, val); // append key/value to data
@@ -315,12 +328,14 @@
if (this.keys != null)
return;
this.count = 0;
- this.keys = new WritableComparable[1024];
this.positions = new long[1024];
+
try {
int skip = INDEX_SKIP;
LongWritable position = new LongWritable();
WritableComparable lastKey = null;
+ long lastIndex = -1;
+ ArrayList<WritableComparable> keyBuilder = new ArrayList<WritableComparable>(1024);
while (true) {
WritableComparable k = comparator.newKey();
@@ -331,7 +346,6 @@
if (lastKey != null && comparator.compare(lastKey, k) > 0)
throw new IOException("key out of order: "+k+" after "+lastKey);
lastKey = k;
-
if (skip > 0) {
skip--;
continue; // skip this entry
@@ -339,20 +353,23 @@
skip = INDEX_SKIP; // reset skip
}
- if (count == keys.length) { // time to grow arrays
- int newLength = (keys.length*3)/2;
- WritableComparable[] newKeys = new WritableComparable[newLength];
- long[] newPositions = new long[newLength];
- System.arraycopy(keys, 0, newKeys, 0, count);
- System.arraycopy(positions, 0, newPositions, 0, count);
- keys = newKeys;
- positions = newPositions;
+ // don't read an index that is the same as the previous one. Block
+ // compressed map files used to do this (multiple entries would point
+ // at the same block)
+ if (position.get() == lastIndex)
+ continue;
+
+ if (count == positions.length) {
+ positions = Arrays.copyOf(positions, positions.length * 2);
}
- keys[count] = k;
+ keyBuilder.add(k);
positions[count] = position.get();
count++;
}
+
+ this.keys = keyBuilder.toArray(new WritableComparable[count]);
+ positions = Arrays.copyOf(positions, count);
} catch (EOFException e) {
LOG.warn("Unexpected EOF reading " + index +
" at entry #" + count + ". Ignoring.");