You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/09/08 02:14:11 UTC
svn commit: r692963 [2/2] - in /hadoop/hbase: branches/0.2/
branches/0.2/src/java/org/apache/hadoop/hbase/
branches/0.2/src/java/org/apache/hadoop/hbase/client/
branches/0.2/src/java/org/apache/hadoop/hbase/master/
branches/0.2/src/java/org/apache/hado...
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java Sun Sep 7 17:14:10 2008
@@ -119,25 +119,28 @@
private final HBaseConfiguration conf;
private final FileSystem fs;
private final Reference reference;
+ private final HRegionInfo hri;
/**
* Constructor that fully initializes the object
* @param conf Configuration object
* @param basedir qualified path that is parent of region directory
- * @param encodedRegionName file name friendly name of the region
* @param colFamily name of the column family
* @param fileId file identifier
* @param ref Reference to another HStoreFile.
+ * @param hri The region info for this file (HACK HBASE-868). TODO: Fix.
* @throws IOException
*/
HStoreFile(HBaseConfiguration conf, FileSystem fs, Path basedir,
- int encodedRegionName, byte [] colFamily, long fileId,
- final Reference ref) throws IOException {
+ final HRegionInfo hri, byte [] colFamily, long fileId,
+ final Reference ref)
+ throws IOException {
this.conf = conf;
this.fs = fs;
this.basedir = basedir;
- this.encodedRegionName = encodedRegionName;
+ this.encodedRegionName = hri.getEncodedName();
this.colFamily = colFamily;
+ this.hri = hri;
long id = fileId;
if (id == -1) {
@@ -431,7 +434,7 @@
"HStoreFile reference");
}
return new BloomFilterMapFile.Writer(conf, fs,
- getMapFilePath().toString(), compression, bloomFilter, nrows);
+ getMapFilePath().toString(), compression, bloomFilter, nrows, this.hri);
}
/**
@@ -584,7 +587,6 @@
* Hbase customizations of MapFile.
*/
static class HbaseMapFile extends MapFile {
- static final Class<? extends Writable> KEY_CLASS = HStoreKey.class;
static final Class<? extends Writable> VALUE_CLASS =
ImmutableBytesWritable.class;
@@ -672,9 +674,10 @@
* @throws IOException
*/
public HbaseWriter(Configuration conf, FileSystem fs, String dirName,
- SequenceFile.CompressionType compression)
+ SequenceFile.CompressionType compression, final HRegionInfo hri)
throws IOException {
- super(conf, fs, dirName, KEY_CLASS, VALUE_CLASS, compression);
+ super(conf, fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri),
+ VALUE_CLASS, compression);
// Default for mapfiles is 128. Makes random reads faster if we
// have more keys indexed and we're not 'next'-ing around in the
// mapfile.
@@ -788,14 +791,15 @@
* @param compression
* @param filter
* @param nrows
+ * @param hri
* @throws IOException
*/
@SuppressWarnings("unchecked")
public Writer(Configuration conf, FileSystem fs, String dirName,
SequenceFile.CompressionType compression, final boolean filter,
- int nrows)
+ int nrows, final HRegionInfo hri)
throws IOException {
- super(conf, fs, dirName, compression);
+ super(conf, fs, dirName, compression, hri);
this.dirName = dirName;
this.fs = fs;
if (filter) {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java Sun Sep 7 17:14:10 2008
@@ -119,8 +119,10 @@
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
(chosenRow == null ||
- (Bytes.compareTo(keys[i].getRow(), chosenRow) < 0) ||
- ((Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) &&
+ (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) < 0) ||
+ ((HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = keys[i].getRow();
chosenTimestamp = keys[i].getTimestamp();
@@ -150,7 +152,8 @@
while ((scanners[i] != null
&& !filtered
&& moreToFollow)
- && (Bytes.compareTo(keys[i].getRow(), chosenRow) == 0)) {
+ && (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) == 0)) {
// If we are doing a wild card match or there are multiple
// matchers per column, we need to scan all the older versions of
// this row to pick up the rest of the family members
@@ -165,7 +168,7 @@
// values with older ones. So now we only insert
// a result if the map does not contain the key.
HStoreKey hsk = new HStoreKey(key.getRow(), HConstants.EMPTY_BYTE_ARRAY,
- key.getTimestamp());
+ key.getTimestamp(), this.store.getHRegionInfo());
for (Map.Entry<byte [], Cell> e : resultSets[i].entrySet()) {
hsk.setColumn(e.getKey());
if (HLogEdit.isDeleted(e.getValue().getValue())) {
@@ -202,7 +205,8 @@
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
- (Bytes.compareTo(keys[i].getRow(), chosenRow) <= 0)) {
+ (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java Sun Sep 7 17:14:10 2008
@@ -61,12 +61,10 @@
// so no additional synchronization is required.
// The currently active sorted map of edits.
- private volatile SortedMap<HStoreKey, byte[]> memcache =
- createSynchronizedSortedMap();
+ private volatile SortedMap<HStoreKey, byte[]> memcache;
// Snapshot of memcache. Made for flusher.
- private volatile SortedMap<HStoreKey, byte[]> snapshot =
- createSynchronizedSortedMap();
+ private volatile SortedMap<HStoreKey, byte[]> snapshot;
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@@ -75,7 +73,10 @@
*/
public Memcache() {
this.ttl = HConstants.FOREVER;
- this.regionInfo = null;
+ // Set default to be the first meta region.
+ this.regionInfo = HRegionInfo.FIRST_META_REGIONINFO;
+ this.memcache = createSynchronizedSortedMap();
+ this.snapshot = createSynchronizedSortedMap();
}
/**
@@ -86,14 +87,18 @@
public Memcache(final long ttl, HRegionInfo regionInfo) {
this.ttl = ttl;
this.regionInfo = regionInfo;
+ this.memcache = createSynchronizedSortedMap();
+ this.snapshot = createSynchronizedSortedMap();
}
/*
- * Utility method.
+ * Utility method using HSKWritableComparator
* @return sycnhronized sorted map of HStoreKey to byte arrays.
*/
- private static SortedMap<HStoreKey, byte[]> createSynchronizedSortedMap() {
- return Collections.synchronizedSortedMap(new TreeMap<HStoreKey, byte []>());
+ private SortedMap<HStoreKey, byte[]> createSynchronizedSortedMap() {
+ return Collections.synchronizedSortedMap(
+ new TreeMap<HStoreKey, byte []>(
+ new HStoreKey.HStoreKeyWritableComparator(this.regionInfo)));
}
/**
@@ -229,7 +234,7 @@
if (b == null) {
return a;
}
- return Bytes.compareTo(a, b) <= 0? a: b;
+ return HStoreKey.compareTwoRowKeys(regionInfo, a, b) <= 0? a: b;
}
/**
@@ -259,14 +264,13 @@
synchronized (map) {
// Make an HSK with maximum timestamp so we get past most of the current
// rows cell entries.
- HStoreKey hsk = new HStoreKey(row, HConstants.LATEST_TIMESTAMP);
+ HStoreKey hsk = new HStoreKey(row, HConstants.LATEST_TIMESTAMP, this.regionInfo);
SortedMap<HStoreKey, byte []> tailMap = map.tailMap(hsk);
// Iterate until we fall into the next row; i.e. move off current row
for (Map.Entry<HStoreKey, byte []> es: tailMap.entrySet()) {
HStoreKey itKey = es.getKey();
- if (Bytes.compareTo(itKey.getRow(), row) <= 0) {
+ if (HStoreKey.compareTwoRowKeys(regionInfo, itKey.getRow(), row) <= 0)
continue;
- }
// Note: Not suppressing deletes or expired cells.
result = itKey.getRow();
break;
@@ -330,13 +334,15 @@
}
}
}
- } else if (Bytes.compareTo(key.getRow(), itKey.getRow()) < 0) {
+ } else if (HStoreKey.compareTwoRowKeys(regionInfo, key.getRow(),
+ itKey.getRow()) < 0) {
break;
}
}
// Remove expired victims from the map.
- for (HStoreKey v: victims)
+ for (HStoreKey v: victims) {
map.remove(v);
+ }
}
/**
@@ -377,8 +383,8 @@
final Set<HStoreKey> deletes) {
// We want the earliest possible to start searching from. Start before
// the candidate key in case it turns out a delete came in later.
- HStoreKey search_key = candidateKeys.isEmpty()? new HStoreKey(row):
- new HStoreKey(candidateKeys.firstKey().getRow());
+ HStoreKey search_key = candidateKeys.isEmpty()? new HStoreKey(row, this.regionInfo):
+ new HStoreKey(candidateKeys.firstKey().getRow(), this.regionInfo);
List<HStoreKey> victims = new ArrayList<HStoreKey>();
long now = System.currentTimeMillis();
@@ -409,7 +415,8 @@
deletedOrExpiredRow = found_key;
}
} else {
- if (HStore.notExpiredAndNotInDeletes(this.ttl, found_key, now, deletes)) {
+ if (HStore.notExpiredAndNotInDeletes(this.ttl,
+ found_key, now, deletes)) {
candidateKeys.put(stripTimestamp(found_key),
new Long(found_key.getTimestamp()));
} else {
@@ -469,7 +476,8 @@
// not a delete record.
boolean deleted = HLogEdit.isDeleted(headMap.get(found_key));
if (lastRowFound != null &&
- !Bytes.equals(lastRowFound, found_key.getRow()) && !deleted) {
+ !HStoreKey.equalsTwoRowKeys(regionInfo, lastRowFound,
+ found_key.getRow()) && !deleted) {
break;
}
// If this isn't a delete, record it as a candidate key. Also
@@ -496,7 +504,7 @@
// smaller acceptable candidate keys would have caused us to start
// our search earlier in the list, and we wouldn't be searching here.
SortedMap<HStoreKey, byte[]> thisRowTailMap =
- headMap.tailMap(new HStoreKey(headMap.lastKey().getRow()));
+ headMap.tailMap(new HStoreKey(headMap.lastKey().getRow(), this.regionInfo));
Iterator<HStoreKey> key_iterator = thisRowTailMap.keySet().iterator();
do {
HStoreKey found_key = key_iterator.next();
@@ -521,7 +529,7 @@
}
static HStoreKey stripTimestamp(HStoreKey key) {
- return new HStoreKey(key.getRow(), key.getColumn());
+ return new HStoreKey(key.getRow(), key.getColumn(), key.getHRegionInfo());
}
/*
@@ -636,7 +644,8 @@
if (origin.getColumn() != null && origin.getColumn().length == 0) {
// if the current and origin row don't match, then we can jump
// out of the loop entirely.
- if (!Bytes.equals(key.getRow(), origin.getRow())) {
+ if (!HStoreKey.equalsTwoRowKeys(regionInfo, key.getRow(),
+ origin.getRow())) {
break;
}
// if the rows match but the timestamp is newer, skip it so we can
@@ -788,7 +797,6 @@
results.put(column, c);
}
this.currentRow = getNextRow(this.currentRow);
-
}
// Set the timestamp to the largest one for the row if we would otherwise
// return HConstants.LATEST_TIMESTAMP
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java Sun Sep 7 17:14:10 2008
@@ -101,7 +101,7 @@
// Advance the readers to the first pos.
for (i = 0; i < readers.length; i++) {
- keys[i] = new HStoreKey();
+ keys[i] = new HStoreKey(HConstants.EMPTY_BYTE_ARRAY, this.store.getHRegionInfo());
if (firstRow != null && firstRow.length != 0) {
if (findFirstRow(i, firstRow)) {
continue;
@@ -159,7 +159,8 @@
for (int i = 0; i < keys.length; i++) {
// Fetch the data
while ((keys[i] != null)
- && (Bytes.compareTo(keys[i].getRow(), viableRow.getRow()) == 0)) {
+ && (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow.getRow()) == 0)) {
// If we are doing a wild card match or there are multiple matchers
// per column, we need to scan all the older versions of this row
@@ -187,7 +188,8 @@
// Advance the current scanner beyond the chosen row, to
// a valid timestamp, so we're ready next time.
while ((keys[i] != null)
- && ((Bytes.compareTo(keys[i].getRow(), viableRow.getRow()) <= 0)
+ && ((HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow.getRow()) <= 0)
|| (keys[i].getTimestamp() > this.timestamp)
|| (! columnMatch(i)))) {
getNext(i);
@@ -246,8 +248,10 @@
// column matches and the timestamp of the row is less than or equal
// to this.timestamp, so we do not need to test that here
&& ((viableRow == null)
- || (Bytes.compareTo(keys[i].getRow(), viableRow) < 0)
- || ((Bytes.compareTo(keys[i].getRow(), viableRow) == 0)
+ || (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow) < 0)
+ || ((HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow) == 0)
&& (keys[i].getTimestamp() > viableTimestamp)))) {
if (ttl == HConstants.FOREVER || now < keys[i].getTimestamp() + ttl) {
viableRow = keys[i].getRow();
@@ -273,7 +277,7 @@
private boolean findFirstRow(int i, final byte [] firstRow) throws IOException {
ImmutableBytesWritable ibw = new ImmutableBytesWritable();
HStoreKey firstKey
- = (HStoreKey)readers[i].getClosest(new HStoreKey(firstRow), ibw);
+ = (HStoreKey)readers[i].getClosest(new HStoreKey(firstRow, this.store.getHRegionInfo()), ibw);
if (firstKey == null) {
// Didn't find it. Close the scanner and return TRUE
closeSubScanner(i);
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java Sun Sep 7 17:14:10 2008
@@ -34,6 +34,7 @@
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HStoreKey;
/**
* Test HStoreFile
@@ -126,7 +127,7 @@
throws IOException {
// Make a store file and write data to it.
HStoreFile hsf = new HStoreFile(this.conf, this.fs, this.dir,
- JenkinsHash.hash(Bytes.toBytes(getName())),
+ HRegionInfo.FIRST_META_REGIONINFO,
Bytes.toBytes("colfamily"), 1234567890L, null);
MapFile.Writer writer =
hsf.getWriter(this.fs, SequenceFile.CompressionType.NONE, false, 0);
@@ -145,7 +146,7 @@
midkey, HStoreFile.Range.top);
HStoreFile refHsf = new HStoreFile(this.conf, this.fs,
new Path(DIR, getName()),
- JenkinsHash.hash(Bytes.toBytes(getName() + "_reference")),
+ HRegionInfo.FIRST_META_REGIONINFO,
hsf.getColFamily(), 456, reference);
// Assert that reference files are written and that we can write and
// read the info reference file at least.