You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/09/08 02:14:11 UTC
svn commit: r692963 [1/2] - in /hadoop/hbase: branches/0.2/
branches/0.2/src/java/org/apache/hadoop/hbase/
branches/0.2/src/java/org/apache/hadoop/hbase/client/
branches/0.2/src/java/org/apache/hadoop/hbase/master/
branches/0.2/src/java/org/apache/hado...
Author: stack
Date: Sun Sep 7 17:14:10 2008
New Revision: 692963
URL: http://svn.apache.org/viewvc?rev=692963&view=rev
Log:
HBASE-868 and HBASE-871 Incrementing binary rows cause strange behavior once table splits AND Major compaction periodicity should be specifyable at the column family level, not cluster wide
Modified:
hadoop/hbase/branches/0.2/CHANGES.txt
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HRegionInfo.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/MetaScanner.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionInfo.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreKey.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
Modified: hadoop/hbase/branches/0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/CHANGES.txt?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.2/CHANGES.txt Sun Sep 7 17:14:10 2008
@@ -56,6 +56,8 @@
HBASE-864 Deadlock in regionserver
HBASE-865 Fix javadoc warnings (Rong-En Fan via Jim Kellerman)
HBASE-872 Getting exceptions in shell when creating/disabling tables
+ HBASE-868 Incrementing binary rows cause strange behavior once table
+ splits (Jonathan Gray via Stack)
IMPROVEMENTS
HBASE-801 When a table haven't disable, shell could response in a "user
@@ -70,6 +72,8 @@
(Jean-Daniel Cryans via Jim Kellerman)
HBASE-849 Speed improvement in JenkinsHash (Andrzej Bialecki via Stack)
HBASE-752 HBase 0.2.1 should be based on Hadoop 0.17.2.1
+ HBASE-871 Major compaction periodicity should be specifyable at the column
+ family level, not cluster wide (Jonathan Gray via Stack)
NEW FEATURES
OPTIMIZATIONS
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java Sun Sep 7 17:14:10 2008
@@ -92,6 +92,9 @@
/** Parameter name for how often threads should wake up */
static final String THREAD_WAKE_FREQUENCY = "hbase.server.thread.wakefrequency";
+
+ /** Parameter name for how often a region should should perform a major compaction */
+ static final String MAJOR_COMPACTION_PERIOD = "hbase.hregion.majorcompaction";
/** Parameter name for HBase instance root directory */
static final String HBASE_DIR = "hbase.rootdir";
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HRegionInfo.java Sun Sep 7 17:14:10 2008
@@ -23,6 +23,7 @@
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JenkinsHash;
import org.apache.hadoop.io.VersionedWritable;
@@ -397,12 +398,12 @@
}
// Compare start keys.
- result = Bytes.compareTo(this.startKey, other.startKey);
+ result = HStoreKey.compareTwoRowKeys(other, this.startKey, other.startKey);
if (result != 0) {
return result;
}
// Compare end keys.
- return Bytes.compareTo(this.endKey, other.endKey);
+ return HStoreKey.compareTwoRowKeys(other, this.endKey, other.endKey);
}
}
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java Sun Sep 7 17:14:10 2008
@@ -26,6 +26,7 @@
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
/**
* A Key for a stored row.
@@ -78,6 +79,29 @@
* The column and table names default to the empty string
*
* @param row row key
+ * @param hri
+ */
+ public HStoreKey(final byte [] row, final HRegionInfo hri) {
+ this(row, HConstants.EMPTY_BYTE_ARRAY, hri);
+ }
+
+ /**
+ * Create an HStoreKey specifying the row and timestamp
+ * The column and table names default to the empty string
+ *
+ * @param row row key
+ * @param timestamp timestamp value
+ * @param hri HRegionInfo
+ */
+ public HStoreKey(final byte [] row, long timestamp, final HRegionInfo hri) {
+ this(row, HConstants.EMPTY_BYTE_ARRAY, timestamp, hri);
+ }
+
+ /**
+ * Create an HStoreKey specifying the row and timestamp
+ * The column and table names default to the empty string
+ *
+ * @param row row key
* @param timestamp timestamp value
*/
public HStoreKey(final byte [] row, long timestamp) {
@@ -188,7 +212,7 @@
* @param other the source key
*/
public HStoreKey(HStoreKey other) {
- this(other.row, other.column, other.timestamp);
+ this(other.row, other.column, other.timestamp, other.regionInfo);
}
/**
@@ -257,7 +281,7 @@
* @see #matchesRowFamily(HStoreKey)
*/
public boolean matchesRowCol(HStoreKey other) {
- return Bytes.equals(this.row, other.row) &&
+ return HStoreKey.equalsTwoRowKeys(this.regionInfo, this.row, other.row) &&
Bytes.equals(column, other.column);
}
@@ -271,7 +295,7 @@
* @see #matchesRowFamily(HStoreKey)
*/
public boolean matchesWithoutColumn(HStoreKey other) {
- return Bytes.equals(this.row, other.row) &&
+ return equalsTwoRowKeys(this.regionInfo, this.row, other.row) &&
this.timestamp >= other.getTimestamp();
}
@@ -286,7 +310,7 @@
*/
public boolean matchesRowFamily(HStoreKey that) {
int delimiterIndex = getFamilyDelimiterIndex(this.column);
- return Bytes.equals(this.row, that.row) &&
+ return equalsTwoRowKeys(this.regionInfo, this.row, that.row) &&
Bytes.compareTo(this.column, 0, delimiterIndex, that.column, 0,
delimiterIndex) == 0;
}
@@ -317,15 +341,19 @@
/** {@inheritDoc} */
public int compareTo(Object o) {
- HStoreKey other = (HStoreKey)o;
- int result = compareTwoRowKeys(this.regionInfo, this.row, other.row);
+ return compareTo(this.regionInfo, this, (HStoreKey)o);
+ }
+
+ static int compareTo(final HRegionInfo hri, final HStoreKey left,
+ final HStoreKey right) {
+ int result = compareTwoRowKeys(hri, left.getRow(), right.getRow());
if (result != 0) {
return result;
}
- result = this.column == null && other.column == null? 0:
- this.column == null && other.column != null? -1:
- this.column != null && other.column == null? 1:
- Bytes.compareTo(this.column, other.column);
+ result = left.getColumn() == null && right.getColumn() == null? 0:
+ left.getColumn() == null && right.getColumn() != null? -1:
+ left.getColumn() != null && right.getColumn() == null? 1:
+ Bytes.compareTo(left.getColumn(), right.getColumn());
if (result != 0) {
return result;
}
@@ -333,9 +361,9 @@
// wrong but it is intentional. This way, newer timestamps are first
// found when we iterate over a memcache and newer versions are the
// first we trip over when reading from a store file.
- if (this.timestamp < other.timestamp) {
+ if (left.getTimestamp() < right.getTimestamp()) {
result = 1;
- } else if (this.timestamp > other.timestamp) {
+ } else if (left.getTimestamp() > right.getTimestamp()) {
result = -1;
}
return result;
@@ -482,9 +510,8 @@
if(rowCompare == 0)
rowCompare = Bytes.compareTo(keysA[1], KeysB[1]);
return rowCompare;
- } else {
- return Bytes.compareTo(rowA, rowB);
}
+ return Bytes.compareTo(rowA, rowB);
}
/**
@@ -513,10 +540,14 @@
break;
}
}
- byte [] row = new byte[offset];
- System.arraycopy(rowKey, 0, row, 0,offset);
- byte [] timestamp = new byte[rowKey.length - offset - 1];
- System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1);
+ byte [] row = rowKey;
+ byte [] timestamp = HConstants.EMPTY_BYTE_ARRAY;
+ if (offset != -1) {
+ row = new byte[offset];
+ System.arraycopy(rowKey, 0, row, 0, offset);
+ timestamp = new byte[rowKey.length - offset - 1];
+ System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1);
+ }
byte[][] elements = new byte[2][];
elements[0] = row;
elements[1] = timestamp;
@@ -538,4 +569,20 @@
this.column = Bytes.readByteArray(in);
this.timestamp = in.readLong();
}
+
+ /**
+ * Passed as comparator for memcache and for store files. See HBASE-868.
+ */
+ public static class HStoreKeyWritableComparator extends WritableComparator {
+ private final HRegionInfo hri;
+
+ public HStoreKeyWritableComparator(final HRegionInfo hri) {
+ super(HStoreKey.class);
+ this.hri = hri;
+ }
+
+ public int compare(final WritableComparable left, final WritableComparable right) {
+ return compareTo(this.hri, (HStoreKey)left, (HStoreKey)right);
+ }
+ }
}
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Sun Sep 7 17:14:10 2008
@@ -37,6 +37,7 @@
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.MasterNotRunningException;
@@ -328,7 +329,7 @@
if (currentRegion != null) {
byte[] endKey = currentRegion.getEndKey();
if (endKey == null ||
- Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)) {
+ HStoreKey.equalsTwoRowKeys(currentRegion, endKey, HConstants.EMPTY_BYTE_ARRAY)) {
// We have reached the end of the table and we're done
break;
}
@@ -641,8 +642,10 @@
// this one. the exception case is when the endkey is EMPTY_START_ROW,
// signifying that the region we're checking is actually the last
// region in the table.
- if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
- Bytes.compareTo(endKey, row) > 0) {
+ if (HStoreKey.equalsTwoRowKeys(possibleRegion.getRegionInfo(),
+ endKey, HConstants.EMPTY_END_ROW) ||
+ HStoreKey.compareTwoRowKeys(possibleRegion.getRegionInfo(),
+ endKey, row) > 0) {
return possibleRegion;
}
}
@@ -689,7 +692,8 @@
// by nature of the map, we know that the start key has to be <
// otherwise it wouldn't be in the headMap.
- if (Bytes.compareTo(endKey, row) <= 0) {
+ if (HStoreKey.compareTwoRowKeys(possibleRegion.getRegionInfo(),
+ endKey, row) <= 0) {
// delete any matching entry
HRegionLocation rl =
tableLocations.remove(matchingRegions.lastKey());
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/MetaScanner.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/MetaScanner.java Sun Sep 7 17:14:10 2008
@@ -5,6 +5,7 @@
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.util.Bytes;
@@ -47,9 +48,10 @@
HRegionInfo.createRegionName(tableName, null, ZEROES);
// Scan over each meta region
+ ScannerCallable callable = null;
do {
- ScannerCallable callable = new ScannerCallable(connection,
- META_TABLE_NAME, COLUMN_FAMILY_ARRAY, startRow, LATEST_TIMESTAMP, null);
+ callable = new ScannerCallable(connection, META_TABLE_NAME,
+ COLUMN_FAMILY_ARRAY, startRow, LATEST_TIMESTAMP, null);
// Open scanner
connection.getRegionServerWithRetries(callable);
try {
@@ -67,7 +69,7 @@
callable.setClose();
connection.getRegionServerWithRetries(callable);
}
- } while (Bytes.compareTo(startRow, LAST_ROW) != 0);
+ } while (HStoreKey.compareTwoRowKeys(callable.getHRegionInfo(), startRow, LAST_ROW) != 0);
}
/**
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/master/MetaRegion.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/master/MetaRegion.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/master/MetaRegion.java Sun Sep 7 17:14:10 2008
@@ -20,7 +20,9 @@
package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.util.Bytes;
@@ -90,7 +92,8 @@
public int compareTo(MetaRegion other) {
int result = Bytes.compareTo(this.regionName, other.getRegionName());
if(result == 0) {
- result = Bytes.compareTo(this.startKey, other.getStartKey());
+ result = HStoreKey.compareTwoRowKeys(HRegionInfo.FIRST_META_REGIONINFO,
+ this.startKey, other.getStartKey());
if (result == 0) {
// Might be on different host?
result = this.server.compareTo(other.server);
@@ -98,4 +101,4 @@
}
return result;
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sun Sep 7 17:14:10 2008
@@ -109,7 +109,7 @@
static final Log LOG = LogFactory.getLog(HRegion.class);
final AtomicBoolean closed = new AtomicBoolean(false);
private final RegionHistorian historian;
-
+
/**
* Merge two HRegions. The regions must be adjacent andmust not overlap.
*
@@ -132,12 +132,14 @@
}
// A's start key is null but B's isn't. Assume A comes before B
} else if ((srcB.getStartKey() == null) // A is not null but B is
- || (Bytes.compareTo(srcA.getStartKey(), srcB.getStartKey()) > 0)) { // A > B
+ || (HStoreKey.compareTwoRowKeys(srcA.getRegionInfo(),
+ srcA.getStartKey(), srcB.getStartKey()) > 0)) { // A > B
a = srcB;
b = srcA;
}
- if (!Bytes.equals(a.getEndKey(), b.getStartKey())) {
+ if (!HStoreKey.equalsTwoRowKeys(srcA.getRegionInfo(),
+ a.getEndKey(), b.getStartKey())) {
throw new IOException("Cannot merge non-adjacent regions");
}
return merge(a, b);
@@ -181,13 +183,19 @@
HTableDescriptor tabledesc = a.getTableDesc();
HLog log = a.getLog();
Path basedir = a.getBaseDir();
- final byte [] startKey = Bytes.equals(a.getStartKey(), EMPTY_BYTE_ARRAY) ||
- Bytes.equals(b.getStartKey(), EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
- Bytes.compareTo(a.getStartKey(), b.getStartKey()) <= 0 ?
+ final byte [] startKey = HStoreKey.equalsTwoRowKeys(a.getRegionInfo(),
+ a.getStartKey(), EMPTY_BYTE_ARRAY) ||
+ HStoreKey.equalsTwoRowKeys(a.getRegionInfo(),
+ b.getStartKey(), EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
+ HStoreKey.compareTwoRowKeys(a.getRegionInfo(), a.getStartKey(),
+ b.getStartKey()) <= 0 ?
a.getStartKey() : b.getStartKey();
- final byte [] endKey = Bytes.equals(a.getEndKey(), EMPTY_BYTE_ARRAY) ||
- Bytes.equals(b.getEndKey(), EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
- Bytes.compareTo(a.getEndKey(), b.getEndKey()) <= 0 ?
+ final byte [] endKey = HStoreKey.equalsTwoRowKeys(a.getRegionInfo(),
+ a.getEndKey(), EMPTY_BYTE_ARRAY) ||
+ HStoreKey.equalsTwoRowKeys(b.getRegionInfo(), b.getEndKey(),
+ EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
+ HStoreKey.compareTwoRowKeys(a.getRegionInfo(), a.getEndKey(),
+ b.getEndKey()) <= 0 ?
b.getEndKey() : a.getEndKey();
HRegionInfo newRegionInfo = new HRegionInfo(tabledesc, startKey, endKey);
@@ -232,7 +240,7 @@
}
for (HStoreFile hsf: srcFiles) {
HStoreFile dst = new HStoreFile(conf, fs, basedir,
- newRegionInfo.getEncodedName(), colFamily, -1, null);
+ newRegionInfo, colFamily, -1, null);
if (LOG.isDebugEnabled()) {
LOG.debug("Renaming " + hsf + " to " + dst);
}
@@ -738,12 +746,12 @@
// Add start/end key checking: hbase-428.
byte [] startKey = this.regionInfo.getStartKey();
byte [] endKey = this.regionInfo.getEndKey();
- if (Bytes.equals(startKey, midKey)) {
+ if (HStoreKey.equalsTwoRowKeys(this.regionInfo,startKey, midKey)) {
LOG.debug("Startkey (" + startKey + ") and midkey + (" +
midKey + ") are same, not splitting");
return null;
}
- if (Bytes.equals(midKey, endKey)) {
+ if (HStoreKey.equalsTwoRowKeys(this.regionInfo,midKey, endKey)) {
LOG.debug("Endkey and midkey are same, not splitting");
return null;
}
@@ -789,15 +797,15 @@
// A reference to the bottom half of the hsf store file.
HStoreFile.Reference aReference = new HStoreFile.Reference(
this.regionInfo.getEncodedName(), h.getFileId(),
- new HStoreKey(midKey), HStoreFile.Range.bottom);
+ new HStoreKey(midKey, this.regionInfo), HStoreFile.Range.bottom);
HStoreFile a = new HStoreFile(this.conf, fs, splits,
- regionAInfo.getEncodedName(), h.getColFamily(), -1, aReference);
+ regionAInfo, h.getColFamily(), -1, aReference);
// Reference to top half of the hsf store file.
HStoreFile.Reference bReference = new HStoreFile.Reference(
this.regionInfo.getEncodedName(), h.getFileId(),
new HStoreKey(midKey), HStoreFile.Range.top);
HStoreFile b = new HStoreFile(this.conf, fs, splits,
- regionBInfo.getEncodedName(), h.getColFamily(), -1, bReference);
+ regionBInfo, h.getColFamily(), -1, bReference);
h.splitStoreFile(a, b, this.fs);
}
@@ -1163,7 +1171,7 @@
checkRow(row);
checkColumn(column);
// Don't need a row lock for a simple get
- HStoreKey key = new HStoreKey(row, column, timestamp);
+ HStoreKey key = new HStoreKey(row, column, timestamp, this.regionInfo);
return getStore(column).get(key, numVersions);
} finally {
splitsAndClosesLock.readLock().unlock();
@@ -1195,7 +1203,7 @@
checkColumn(column);
}
}
- HStoreKey key = new HStoreKey(row, ts);
+ HStoreKey key = new HStoreKey(row, ts, this.regionInfo);
Integer lid = obtainRowLock(row);
HashSet<HStore> storeSet = new HashSet<HStore>();
try {
@@ -1259,14 +1267,14 @@
byte [] closestKey = store.getRowKeyAtOrBefore(row);
// if it happens to be an exact match, we can stop looping
if (HStoreKey.equalsTwoRowKeys(regionInfo,row, closestKey)) {
- key = new HStoreKey(closestKey);
+ key = new HStoreKey(closestKey, this.regionInfo);
break;
}
// otherwise, we need to check if it's the max and move to the next
if (closestKey != null
&& (key == null || HStoreKey.compareTwoRowKeys(
regionInfo,closestKey, key.getRow()) > 0) ) {
- key = new HStoreKey(closestKey);
+ key = new HStoreKey(closestKey, this.regionInfo);
}
}
if (key == null) {
@@ -1388,7 +1396,8 @@
try {
List<byte []> deletes = null;
for (BatchOperation op: b) {
- HStoreKey key = new HStoreKey(row, op.getColumn(), commitTime);
+ HStoreKey key = new HStoreKey(row, op.getColumn(), commitTime,
+ this.regionInfo);
byte[] val = null;
if (op.isPut()) {
val = op.getValue();
@@ -1507,7 +1516,7 @@
long now = System.currentTimeMillis();
try {
for (HStore store : stores.values()) {
- List<HStoreKey> keys = store.getKeys(new HStoreKey(row, ts),
+ List<HStoreKey> keys = store.getKeys(new HStoreKey(row, ts, this.regionInfo),
ALL_VERSIONS, now);
TreeMap<HStoreKey, byte []> edits = new TreeMap<HStoreKey, byte []>();
for (HStoreKey key: keys) {
@@ -1538,7 +1547,7 @@
// find the HStore for the column family
HStore store = getStore(family);
// find all the keys that match our criteria
- List<HStoreKey> keys = store.getKeys(new HStoreKey(row, timestamp),
+ List<HStoreKey> keys = store.getKeys(new HStoreKey(row, timestamp, this.regionInfo),
ALL_VERSIONS, now);
// delete all the cells
TreeMap<HStoreKey, byte []> edits = new TreeMap<HStoreKey, byte []>();
@@ -1566,7 +1575,7 @@
final long ts, final int versions)
throws IOException {
checkReadOnly();
- HStoreKey origin = new HStoreKey(row, column, ts);
+ HStoreKey origin = new HStoreKey(row, column, ts, this.regionInfo);
Set<HStoreKey> keys = getKeys(origin, versions);
if (keys.size() > 0) {
TreeMap<HStoreKey, byte []> edits = new TreeMap<HStoreKey, byte []>();
@@ -1816,7 +1825,7 @@
}
}
}
-
+
/** {@inheritDoc} */
@Override
public boolean equals(Object o) {
@@ -1893,7 +1902,7 @@
this.resultSets = new TreeMap[scanners.length];
this.keys = new HStoreKey[scanners.length];
for (int i = 0; i < scanners.length; i++) {
- keys[i] = new HStoreKey();
+ keys[i] = new HStoreKey(HConstants.EMPTY_BYTE_ARRAY,regionInfo);
resultSets[i] = new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
if(scanners[i] != null && !scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
@@ -1918,9 +1927,11 @@
long chosenTimestamp = -1;
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
- (chosenRow == null ||
- (Bytes.compareTo(keys[i].getRow(), chosenRow) < 0) ||
- ((Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) &&
+ (chosenRow == null ||
+ (HStoreKey.compareTwoRowKeys(regionInfo,
+ keys[i].getRow(), chosenRow) < 0) ||
+ ((HStoreKey.compareTwoRowKeys(regionInfo, keys[i].getRow(),
+ chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = keys[i].getRow();
chosenTimestamp = keys[i].getTimestamp();
@@ -1937,7 +1948,7 @@
for (int i = 0; i < scanners.length; i++) {
if (scanners[i] != null &&
- Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) {
+ HStoreKey.compareTwoRowKeys(regionInfo,keys[i].getRow(), chosenRow) == 0) {
// NOTE: We used to do results.putAll(resultSets[i]);
// but this had the effect of overwriting newer
// values with older ones. So now we only insert
@@ -1959,7 +1970,7 @@
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
- (Bytes.compareTo(keys[i].getRow(), chosenRow) <= 0)) {
+ (HStoreKey.compareTwoRowKeys(regionInfo,keys[i].getRow(), chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
@@ -2138,7 +2149,8 @@
byte [] row = r.getRegionName();
Integer lid = meta.obtainRowLock(row);
try {
- HStoreKey key = new HStoreKey(row, COL_REGIONINFO, System.currentTimeMillis());
+ HStoreKey key = new HStoreKey(row, COL_REGIONINFO,
+ System.currentTimeMillis(), r.getRegionInfo());
TreeMap<HStoreKey, byte[]> edits = new TreeMap<HStoreKey, byte[]>();
edits.put(key, Writables.getBytes(r.getRegionInfo()));
meta.update(edits);
@@ -2239,9 +2251,9 @@
*/
public static boolean rowIsInRange(HRegionInfo info, final byte [] row) {
return ((info.getStartKey().length == 0) ||
- (Bytes.compareTo(info.getStartKey(), row) <= 0)) &&
+ (HStoreKey.compareTwoRowKeys(info,info.getStartKey(), row) <= 0)) &&
((info.getEndKey().length == 0) ||
- (Bytes.compareTo(info.getEndKey(), row) > 0));
+ (HStoreKey.compareTwoRowKeys(info,info.getEndKey(), row) > 0));
}
/**
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Sun Sep 7 17:14:10 2008
@@ -1305,8 +1305,7 @@
long timestamp) throws IOException{
getRegion(regionName).deleteFamily(row, family, timestamp);
}
-
-
+
/**
* @return Info on this server.
*/
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Sun Sep 7 17:14:10 2008
@@ -188,7 +188,12 @@
}
this.desiredMaxFileSize = maxFileSize;
- this.majorCompactionTime = conf.getLong("hbase.hregion.majorcompaction", 86400000);
+ this.majorCompactionTime = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 86400000);
+ if (family.getValue(HConstants.MAJOR_COMPACTION_PERIOD) != null) {
+ String strCompactionTime = family.getValue(HConstants.MAJOR_COMPACTION_PERIOD);
+ this.majorCompactionTime = (new Long(strCompactionTime)).longValue();
+ }
+
this.maxFilesToCompact = conf.getInt("hbase.hstore.compaction.max", 10);
this.storeSize = 0L;
@@ -319,7 +324,8 @@
|| !HStoreKey.matchingFamily(family.getName(), column)) {
continue;
}
- HStoreKey k = new HStoreKey(key.getRow(), column, val.getTimestamp());
+ HStoreKey k = new HStoreKey(key.getRow(), column, val.getTimestamp(),
+ this.info);
reconstructedCache.put(k, val.getVal());
editsCount++;
// Every 2k edits, tell the reporter we're making progress.
@@ -390,7 +396,7 @@
if (isReference) {
reference = HStoreFile.readSplitInfo(p, fs);
}
- curfile = new HStoreFile(conf, fs, basedir, info.getEncodedName(),
+ curfile = new HStoreFile(conf, fs, basedir, this.info,
family.getName(), fid, reference);
long storeSeqId = -1;
try {
@@ -424,7 +430,9 @@
// Try fixing this file.. if we can. Use the hbase version of fix.
// Need to remove the old index file first else fix won't go ahead.
this.fs.delete(new Path(mapfile, MapFile.INDEX_FILE_NAME), false);
- long count = MapFile.fix(this.fs, mapfile, HStoreFile.HbaseMapFile.KEY_CLASS,
+ // TODO: This is going to fail if we are to rebuild a file from
+ // meta because it won't have right comparator: HBASE-848.
+ long count = MapFile.fix(this.fs, mapfile, HStoreKey.class,
HStoreFile.HbaseMapFile.VALUE_CLASS, false, this.conf);
if (LOG.isDebugEnabled()) {
LOG.debug("Fixed index on " + mapfile.toString() + "; had " +
@@ -589,7 +597,7 @@
long now = System.currentTimeMillis();
// A. Write the Maps out to the disk
HStoreFile flushedFile = new HStoreFile(conf, fs, basedir,
- info.getEncodedName(), family.getName(), -1L, null);
+ this.info, family.getName(), -1L, null);
MapFile.Writer out = flushedFile.getWriter(this.fs, this.compression,
this.family.isBloomfilter(), cache.size());
out.setIndexInterval(family.getMapFileIndexInterval());
@@ -873,8 +881,7 @@
// Step through them, writing to the brand-new MapFile
HStoreFile compactedOutputFile = new HStoreFile(conf, fs,
- this.compactionDir, info.getEncodedName(), family.getName(),
- -1L, null);
+ this.compactionDir, this.info, family.getName(), -1L, null);
if (LOG.isDebugEnabled()) {
LOG.debug("started compaction of " + rdrs.size() + " files into " +
FSUtils.getPath(compactedOutputFile.getMapFilePath()));
@@ -962,7 +969,7 @@
}
}
HStoreKey sk = keys[smallestKey];
- if (Bytes.equals(lastRow, sk.getRow())
+ if (HStoreKey.equalsTwoRowKeys(info,lastRow, sk.getRow())
&& Bytes.equals(lastColumn, sk.getColumn())) {
timesSeen++;
} else {
@@ -1045,7 +1052,7 @@
try {
// 1. Moving the new MapFile into place.
HStoreFile finalCompactedFile = new HStoreFile(conf, fs, basedir,
- info.getEncodedName(), family.getName(), -1, null);
+ this.info, family.getName(), -1, null);
if (LOG.isDebugEnabled()) {
LOG.debug("moving " + FSUtils.getPath(compactedFile.getMapFilePath()) +
" to " + FSUtils.getPath(finalCompactedFile.getMapFilePath()));
@@ -1207,7 +1214,7 @@
}
}
}
- } else if (Bytes.compareTo(key.getRow(), readkey.getRow()) < 0) {
+ } else if (HStoreKey.compareTwoRowKeys(info,key.getRow(), readkey.getRow()) < 0) {
// if we've crossed into the next row, then we can just stop
// iterating
break;
@@ -1498,7 +1505,7 @@
}
// If start row for this file is beyond passed in row, return; nothing
// in here is of use to us.
- if (Bytes.compareTo(startKey.getRow(), row) > 0) {
+ if (HStoreKey.compareTwoRowKeys(info,startKey.getRow(), row) > 0) {
return;
}
long now = System.currentTimeMillis();
@@ -1531,7 +1538,7 @@
// up to the row before and return that.
HStoreKey finalKey = getFinalKey(map);
HStoreKey searchKey = null;
- if (Bytes.compareTo(finalKey.getRow(), row) < 0) {
+ if (HStoreKey.compareTwoRowKeys(info,finalKey.getRow(), row) < 0) {
searchKey = finalKey;
} else {
searchKey = new HStoreKey(row);
@@ -1592,7 +1599,7 @@
do {
// If we have an exact match on row, and it's not a delete, save this
// as a candidate key
- if (Bytes.equals(readkey.getRow(), searchKey.getRow())) {
+ if (HStoreKey.equalsTwoRowKeys(info,readkey.getRow(), searchKey.getRow())) {
if (!HLogEdit.isDeleted(readval.get())) {
if (handleNonDelete(readkey, now, deletes, candidateKeys)) {
foundCandidate = true;
@@ -1604,7 +1611,7 @@
if (deletedOrExpiredRow == null) {
deletedOrExpiredRow = copy;
}
- } else if (Bytes.compareTo(readkey.getRow(), searchKey.getRow()) > 0) {
+ } else if (HStoreKey.compareTwoRowKeys(info,readkey.getRow(), searchKey.getRow()) > 0) {
// if the row key we just read is beyond the key we're searching for,
// then we're done.
break;
@@ -1685,9 +1692,9 @@
do {
// if we have an exact match on row, and it's not a delete, save this
// as a candidate key
- if (Bytes.equals(readkey.getRow(), row)) {
+ if (HStoreKey.equalsTwoRowKeys(info,readkey.getRow(), row)) {
handleKey(readkey, readval.get(), now, deletes, candidateKeys);
- } else if (Bytes.compareTo(readkey.getRow(), row) > 0 ) {
+ } else if (HStoreKey.compareTwoRowKeys(info,readkey.getRow(), row) > 0 ) {
// if the row key we just read is beyond the key we're searching for,
// then we're done.
break;
@@ -1769,7 +1776,7 @@
}
static HStoreKey stripTimestamp(HStoreKey key) {
- return new HStoreKey(key.getRow(), key.getColumn());
+ return new HStoreKey(key.getRow(), key.getColumn(), key.getHRegionInfo());
}
/*
@@ -1784,7 +1791,7 @@
// if the origin's column is empty, then we're matching any column
if (Bytes.equals(origin.getColumn(), HConstants.EMPTY_BYTE_ARRAY)) {
// if the row matches, then...
- if (Bytes.equals(target.getRow(), origin.getRow())) {
+ if (HStoreKey.equalsTwoRowKeys(info, target.getRow(), origin.getRow())) {
// check the timestamp
return target.getTimestamp() <= origin.getTimestamp();
}
@@ -1805,7 +1812,7 @@
// if the origin's column is empty, then we're matching any column
if (Bytes.equals(origin.getColumn(), HConstants.EMPTY_BYTE_ARRAY)) {
// if the row matches, then...
- return Bytes.equals(target.getRow(), origin.getRow());
+ return HStoreKey.equalsTwoRowKeys(info, target.getRow(), origin.getRow());
}
// otherwise, we want to match on row and column
return target.matchesRowCol(origin);
@@ -1864,8 +1871,8 @@
if (mk != null) {
// if the midkey is the same as the first and last keys, then we cannot
// (ever) split this region.
- if (Bytes.equals(mk.getRow(), firstKey.getRow()) &&
- Bytes.equals(mk.getRow(), lastKey.getRow())) {
+ if (HStoreKey.equalsTwoRowKeys(info, mk.getRow(), firstKey.getRow()) &&
+ HStoreKey.equalsTwoRowKeys(info, mk.getRow(), lastKey.getRow())) {
return null;
}
return new StoreSize(maxSize, mk.getRow());
@@ -1952,4 +1959,8 @@
return key;
}
}
+
+ HRegionInfo getHRegionInfo() {
+ return this.info;
+ }
}
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java Sun Sep 7 17:14:10 2008
@@ -119,25 +119,28 @@
private final HBaseConfiguration conf;
private final FileSystem fs;
private final Reference reference;
+ private final HRegionInfo hri;
/**
* Constructor that fully initializes the object
* @param conf Configuration object
* @param basedir qualified path that is parent of region directory
- * @param encodedRegionName file name friendly name of the region
* @param colFamily name of the column family
* @param fileId file identifier
* @param ref Reference to another HStoreFile.
+ * @param hri The region info for this file (HACK HBASE-868). TODO: Fix.
* @throws IOException
*/
HStoreFile(HBaseConfiguration conf, FileSystem fs, Path basedir,
- int encodedRegionName, byte [] colFamily, long fileId,
- final Reference ref) throws IOException {
+ final HRegionInfo hri, byte [] colFamily, long fileId,
+ final Reference ref)
+ throws IOException {
this.conf = conf;
this.fs = fs;
this.basedir = basedir;
- this.encodedRegionName = encodedRegionName;
+ this.encodedRegionName = hri.getEncodedName();
this.colFamily = colFamily;
+ this.hri = hri;
long id = fileId;
if (id == -1) {
@@ -431,7 +434,7 @@
"HStoreFile reference");
}
return new BloomFilterMapFile.Writer(conf, fs,
- getMapFilePath().toString(), compression, bloomFilter, nrows);
+ getMapFilePath().toString(), compression, bloomFilter, nrows, this.hri);
}
/**
@@ -584,7 +587,6 @@
* Hbase customizations of MapFile.
*/
static class HbaseMapFile extends MapFile {
- static final Class<? extends Writable> KEY_CLASS = HStoreKey.class;
static final Class<? extends Writable> VALUE_CLASS =
ImmutableBytesWritable.class;
@@ -672,9 +674,10 @@
* @throws IOException
*/
public HbaseWriter(Configuration conf, FileSystem fs, String dirName,
- SequenceFile.CompressionType compression)
+ SequenceFile.CompressionType compression, final HRegionInfo hri)
throws IOException {
- super(conf, fs, dirName, KEY_CLASS, VALUE_CLASS, compression);
+ super(conf, fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri),
+ VALUE_CLASS, compression);
// Default for mapfiles is 128. Makes random reads faster if we
// have more keys indexed and we're not 'next'-ing around in the
// mapfile.
@@ -788,14 +791,15 @@
* @param compression
* @param filter
* @param nrows
+ * @param hri
* @throws IOException
*/
@SuppressWarnings("unchecked")
public Writer(Configuration conf, FileSystem fs, String dirName,
SequenceFile.CompressionType compression, final boolean filter,
- int nrows)
+ int nrows, final HRegionInfo hri)
throws IOException {
- super(conf, fs, dirName, compression);
+ super(conf, fs, dirName, compression, hri);
this.dirName = dirName;
this.fs = fs;
if (filter) {
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java Sun Sep 7 17:14:10 2008
@@ -119,8 +119,10 @@
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
(chosenRow == null ||
- (Bytes.compareTo(keys[i].getRow(), chosenRow) < 0) ||
- ((Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) &&
+ (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) < 0) ||
+ ((HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = keys[i].getRow();
chosenTimestamp = keys[i].getTimestamp();
@@ -150,7 +152,8 @@
while ((scanners[i] != null
&& !filtered
&& moreToFollow)
- && (Bytes.compareTo(keys[i].getRow(), chosenRow) == 0)) {
+ && (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) == 0)) {
// If we are doing a wild card match or there are multiple
// matchers per column, we need to scan all the older versions of
// this row to pick up the rest of the family members
@@ -165,7 +168,7 @@
// values with older ones. So now we only insert
// a result if the map does not contain the key.
HStoreKey hsk = new HStoreKey(key.getRow(), HConstants.EMPTY_BYTE_ARRAY,
- key.getTimestamp());
+ key.getTimestamp(), this.store.getHRegionInfo());
for (Map.Entry<byte [], Cell> e : resultSets[i].entrySet()) {
hsk.setColumn(e.getKey());
if (HLogEdit.isDeleted(e.getValue().getValue())) {
@@ -202,7 +205,8 @@
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
- (Bytes.compareTo(keys[i].getRow(), chosenRow) <= 0)) {
+ (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java Sun Sep 7 17:14:10 2008
@@ -61,12 +61,10 @@
// so no additional synchronization is required.
// The currently active sorted map of edits.
- private volatile SortedMap<HStoreKey, byte[]> memcache =
- createSynchronizedSortedMap();
+ private volatile SortedMap<HStoreKey, byte[]> memcache;
// Snapshot of memcache. Made for flusher.
- private volatile SortedMap<HStoreKey, byte[]> snapshot =
- createSynchronizedSortedMap();
+ private volatile SortedMap<HStoreKey, byte[]> snapshot;
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@@ -75,7 +73,10 @@
*/
public Memcache() {
this.ttl = HConstants.FOREVER;
- this.regionInfo = null;
+ // Set default to be the first meta region.
+ this.regionInfo = HRegionInfo.FIRST_META_REGIONINFO;
+ this.memcache = createSynchronizedSortedMap();
+ this.snapshot = createSynchronizedSortedMap();
}
/**
@@ -86,14 +87,18 @@
public Memcache(final long ttl, HRegionInfo regionInfo) {
this.ttl = ttl;
this.regionInfo = regionInfo;
+ this.memcache = createSynchronizedSortedMap();
+ this.snapshot = createSynchronizedSortedMap();
}
/*
- * Utility method.
+ * Utility method using HSKWritableComparator
* @return sycnhronized sorted map of HStoreKey to byte arrays.
*/
- private static SortedMap<HStoreKey, byte[]> createSynchronizedSortedMap() {
- return Collections.synchronizedSortedMap(new TreeMap<HStoreKey, byte []>());
+ private SortedMap<HStoreKey, byte[]> createSynchronizedSortedMap() {
+ return Collections.synchronizedSortedMap(
+ new TreeMap<HStoreKey, byte []>(
+ new HStoreKey.HStoreKeyWritableComparator(this.regionInfo)));
}
/**
@@ -229,7 +234,7 @@
if (b == null) {
return a;
}
- return Bytes.compareTo(a, b) <= 0? a: b;
+ return HStoreKey.compareTwoRowKeys(regionInfo, a, b) <= 0? a: b;
}
/**
@@ -259,14 +264,13 @@
synchronized (map) {
// Make an HSK with maximum timestamp so we get past most of the current
// rows cell entries.
- HStoreKey hsk = new HStoreKey(row, HConstants.LATEST_TIMESTAMP);
+ HStoreKey hsk = new HStoreKey(row, HConstants.LATEST_TIMESTAMP, this.regionInfo);
SortedMap<HStoreKey, byte []> tailMap = map.tailMap(hsk);
// Iterate until we fall into the next row; i.e. move off current row
for (Map.Entry<HStoreKey, byte []> es: tailMap.entrySet()) {
HStoreKey itKey = es.getKey();
- if (Bytes.compareTo(itKey.getRow(), row) <= 0) {
+ if (HStoreKey.compareTwoRowKeys(regionInfo, itKey.getRow(), row) <= 0)
continue;
- }
// Note: Not suppressing deletes or expired cells.
result = itKey.getRow();
break;
@@ -330,13 +334,15 @@
}
}
}
- } else if (Bytes.compareTo(key.getRow(), itKey.getRow()) < 0) {
+ } else if (HStoreKey.compareTwoRowKeys(regionInfo, key.getRow(),
+ itKey.getRow()) < 0) {
break;
}
}
// Remove expired victims from the map.
- for (HStoreKey v: victims)
+ for (HStoreKey v: victims) {
map.remove(v);
+ }
}
/**
@@ -377,8 +383,8 @@
final Set<HStoreKey> deletes) {
// We want the earliest possible to start searching from. Start before
// the candidate key in case it turns out a delete came in later.
- HStoreKey search_key = candidateKeys.isEmpty()? new HStoreKey(row):
- new HStoreKey(candidateKeys.firstKey().getRow());
+ HStoreKey search_key = candidateKeys.isEmpty()? new HStoreKey(row, this.regionInfo):
+ new HStoreKey(candidateKeys.firstKey().getRow(), this.regionInfo);
List<HStoreKey> victims = new ArrayList<HStoreKey>();
long now = System.currentTimeMillis();
@@ -409,7 +415,8 @@
deletedOrExpiredRow = found_key;
}
} else {
- if (HStore.notExpiredAndNotInDeletes(this.ttl, found_key, now, deletes)) {
+ if (HStore.notExpiredAndNotInDeletes(this.ttl,
+ found_key, now, deletes)) {
candidateKeys.put(stripTimestamp(found_key),
new Long(found_key.getTimestamp()));
} else {
@@ -469,7 +476,8 @@
// not a delete record.
boolean deleted = HLogEdit.isDeleted(headMap.get(found_key));
if (lastRowFound != null &&
- !Bytes.equals(lastRowFound, found_key.getRow()) && !deleted) {
+ !HStoreKey.equalsTwoRowKeys(regionInfo, lastRowFound,
+ found_key.getRow()) && !deleted) {
break;
}
// If this isn't a delete, record it as a candidate key. Also
@@ -496,7 +504,7 @@
// smaller acceptable candidate keys would have caused us to start
// our search earlier in the list, and we wouldn't be searching here.
SortedMap<HStoreKey, byte[]> thisRowTailMap =
- headMap.tailMap(new HStoreKey(headMap.lastKey().getRow()));
+ headMap.tailMap(new HStoreKey(headMap.lastKey().getRow(), this.regionInfo));
Iterator<HStoreKey> key_iterator = thisRowTailMap.keySet().iterator();
do {
HStoreKey found_key = key_iterator.next();
@@ -521,7 +529,7 @@
}
static HStoreKey stripTimestamp(HStoreKey key) {
- return new HStoreKey(key.getRow(), key.getColumn());
+ return new HStoreKey(key.getRow(), key.getColumn(), key.getHRegionInfo());
}
/*
@@ -636,7 +644,8 @@
if (origin.getColumn() != null && origin.getColumn().length == 0) {
// if the current and origin row don't match, then we can jump
// out of the loop entirely.
- if (!Bytes.equals(key.getRow(), origin.getRow())) {
+ if (!HStoreKey.equalsTwoRowKeys(regionInfo, key.getRow(),
+ origin.getRow())) {
break;
}
// if the rows match but the timestamp is newer, skip it so we can
@@ -788,7 +797,6 @@
results.put(column, c);
}
this.currentRow = getNextRow(this.currentRow);
-
}
// Set the timestamp to the largest one for the row if we would otherwise
// return HConstants.LATEST_TIMESTAMP
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java Sun Sep 7 17:14:10 2008
@@ -101,7 +101,7 @@
// Advance the readers to the first pos.
for (i = 0; i < readers.length; i++) {
- keys[i] = new HStoreKey();
+ keys[i] = new HStoreKey(HConstants.EMPTY_BYTE_ARRAY, this.store.getHRegionInfo());
if (firstRow != null && firstRow.length != 0) {
if (findFirstRow(i, firstRow)) {
continue;
@@ -159,7 +159,8 @@
for (int i = 0; i < keys.length; i++) {
// Fetch the data
while ((keys[i] != null)
- && (Bytes.compareTo(keys[i].getRow(), viableRow.getRow()) == 0)) {
+ && (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow.getRow()) == 0)) {
// If we are doing a wild card match or there are multiple matchers
// per column, we need to scan all the older versions of this row
@@ -187,7 +188,8 @@
// Advance the current scanner beyond the chosen row, to
// a valid timestamp, so we're ready next time.
while ((keys[i] != null)
- && ((Bytes.compareTo(keys[i].getRow(), viableRow.getRow()) <= 0)
+ && ((HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow.getRow()) <= 0)
|| (keys[i].getTimestamp() > this.timestamp)
|| (! columnMatch(i)))) {
getNext(i);
@@ -246,8 +248,10 @@
// column matches and the timestamp of the row is less than or equal
// to this.timestamp, so we do not need to test that here
&& ((viableRow == null)
- || (Bytes.compareTo(keys[i].getRow(), viableRow) < 0)
- || ((Bytes.compareTo(keys[i].getRow(), viableRow) == 0)
+ || (HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow) < 0)
+ || ((HStoreKey.compareTwoRowKeys(store.getHRegionInfo(),
+ keys[i].getRow(), viableRow) == 0)
&& (keys[i].getTimestamp() > viableTimestamp)))) {
if (ttl == HConstants.FOREVER || now < keys[i].getTimestamp() + ttl) {
viableRow = keys[i].getRow();
@@ -273,7 +277,7 @@
private boolean findFirstRow(int i, final byte [] firstRow) throws IOException {
ImmutableBytesWritable ibw = new ImmutableBytesWritable();
HStoreKey firstKey
- = (HStoreKey)readers[i].getClosest(new HStoreKey(firstRow), ibw);
+ = (HStoreKey)readers[i].getClosest(new HStoreKey(firstRow, this.store.getHRegionInfo()), ibw);
if (firstKey == null) {
// Didn't find it. Close the scanner and return TRUE
closeSubScanner(i);
Modified: hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java (original)
+++ hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java Sun Sep 7 17:14:10 2008
@@ -34,6 +34,7 @@
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HStoreKey;
/**
* Test HStoreFile
@@ -126,7 +127,7 @@
throws IOException {
// Make a store file and write data to it.
HStoreFile hsf = new HStoreFile(this.conf, this.fs, this.dir,
- JenkinsHash.hash(Bytes.toBytes(getName())),
+ HRegionInfo.FIRST_META_REGIONINFO,
Bytes.toBytes("colfamily"), 1234567890L, null);
MapFile.Writer writer =
hsf.getWriter(this.fs, SequenceFile.CompressionType.NONE, false, 0);
@@ -145,7 +146,7 @@
midkey, HStoreFile.Range.top);
HStoreFile refHsf = new HStoreFile(this.conf, this.fs,
new Path(DIR, getName()),
- JenkinsHash.hash(Bytes.toBytes(getName() + "_reference")),
+ HRegionInfo.FIRST_META_REGIONINFO,
hsf.getColFamily(), 456, reference);
// Assert that reference files are written and that we can write and
// read the info reference file at least.
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Sun Sep 7 17:14:10 2008
@@ -58,6 +58,8 @@
HBASE-864 Deadlock in regionserver
HBASE-865 Fix javadoc warnings (Rong-En Fan via Jim Kellerman)
HBASE-872 Getting exceptions in shell when creating/disabling tables
+ HBASE-868 Incrementing binary rows cause strange behavior once table
+ splits (Jonathan Gray via Stack)
IMPROVEMENTS
HBASE-801 When a table haven't disable, shell could response in a "user
@@ -78,6 +80,8 @@
(Sishen Freecity via Stack)
HBASE-874 deleting a table kills client rpc; no subsequent communication if
shell or thrift server, etc. (Jonathan Gray via Jim Kellerman)
+ HBASE-871 Major compaction periodicity should be specifyable at the column
+ family level, not cluster wide (Jonathan Gray via Stack)
NEW FEATURES
HBASE-787 Postgresql to HBase table replication example (Tim Sell via Stack)
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java Sun Sep 7 17:14:10 2008
@@ -92,6 +92,9 @@
/** Parameter name for how often threads should wake up */
static final String THREAD_WAKE_FREQUENCY = "hbase.server.thread.wakefrequency";
+
+ /** Parameter name for how often a region should should perform a major compaction */
+ static final String MAJOR_COMPACTION_PERIOD = "hbase.hregion.majorcompaction";
/** Parameter name for HBase instance root directory */
static final String HBASE_DIR = "hbase.rootdir";
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionInfo.java Sun Sep 7 17:14:10 2008
@@ -23,6 +23,7 @@
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JenkinsHash;
import org.apache.hadoop.io.VersionedWritable;
@@ -397,12 +398,12 @@
}
// Compare start keys.
- result = Bytes.compareTo(this.startKey, other.startKey);
+ result = HStoreKey.compareTwoRowKeys(other, this.startKey, other.startKey);
if (result != 0) {
return result;
}
// Compare end keys.
- return Bytes.compareTo(this.endKey, other.endKey);
+ return HStoreKey.compareTwoRowKeys(other, this.endKey, other.endKey);
}
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreKey.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreKey.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreKey.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreKey.java Sun Sep 7 17:14:10 2008
@@ -26,6 +26,7 @@
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
/**
* A Key for a stored row.
@@ -78,6 +79,29 @@
* The column and table names default to the empty string
*
* @param row row key
+ * @param hri
+ */
+ public HStoreKey(final byte [] row, final HRegionInfo hri) {
+ this(row, HConstants.EMPTY_BYTE_ARRAY, hri);
+ }
+
+ /**
+ * Create an HStoreKey specifying the row and timestamp
+ * The column and table names default to the empty string
+ *
+ * @param row row key
+ * @param timestamp timestamp value
+ * @param hri HRegionInfo
+ */
+ public HStoreKey(final byte [] row, long timestamp, final HRegionInfo hri) {
+ this(row, HConstants.EMPTY_BYTE_ARRAY, timestamp, hri);
+ }
+
+ /**
+ * Create an HStoreKey specifying the row and timestamp
+ * The column and table names default to the empty string
+ *
+ * @param row row key
* @param timestamp timestamp value
*/
public HStoreKey(final byte [] row, long timestamp) {
@@ -188,7 +212,7 @@
* @param other the source key
*/
public HStoreKey(HStoreKey other) {
- this(other.row, other.column, other.timestamp);
+ this(other.row, other.column, other.timestamp, other.regionInfo);
}
/**
@@ -257,7 +281,7 @@
* @see #matchesRowFamily(HStoreKey)
*/
public boolean matchesRowCol(HStoreKey other) {
- return Bytes.equals(this.row, other.row) &&
+ return HStoreKey.equalsTwoRowKeys(this.regionInfo, this.row, other.row) &&
Bytes.equals(column, other.column);
}
@@ -271,7 +295,7 @@
* @see #matchesRowFamily(HStoreKey)
*/
public boolean matchesWithoutColumn(HStoreKey other) {
- return Bytes.equals(this.row, other.row) &&
+ return equalsTwoRowKeys(this.regionInfo, this.row, other.row) &&
this.timestamp >= other.getTimestamp();
}
@@ -286,7 +310,7 @@
*/
public boolean matchesRowFamily(HStoreKey that) {
int delimiterIndex = getFamilyDelimiterIndex(this.column);
- return Bytes.equals(this.row, that.row) &&
+ return equalsTwoRowKeys(this.regionInfo, this.row, that.row) &&
Bytes.compareTo(this.column, 0, delimiterIndex, that.column, 0,
delimiterIndex) == 0;
}
@@ -317,15 +341,19 @@
/** {@inheritDoc} */
public int compareTo(Object o) {
- HStoreKey other = (HStoreKey)o;
- int result = compareTwoRowKeys(this.regionInfo, this.row, other.row);
+ return compareTo(this.regionInfo, this, (HStoreKey)o);
+ }
+
+ static int compareTo(final HRegionInfo hri, final HStoreKey left,
+ final HStoreKey right) {
+ int result = compareTwoRowKeys(hri, left.getRow(), right.getRow());
if (result != 0) {
return result;
}
- result = this.column == null && other.column == null? 0:
- this.column == null && other.column != null? -1:
- this.column != null && other.column == null? 1:
- Bytes.compareTo(this.column, other.column);
+ result = left.getColumn() == null && right.getColumn() == null? 0:
+ left.getColumn() == null && right.getColumn() != null? -1:
+ left.getColumn() != null && right.getColumn() == null? 1:
+ Bytes.compareTo(left.getColumn(), right.getColumn());
if (result != 0) {
return result;
}
@@ -333,9 +361,9 @@
// wrong but it is intentional. This way, newer timestamps are first
// found when we iterate over a memcache and newer versions are the
// first we trip over when reading from a store file.
- if (this.timestamp < other.timestamp) {
+ if (left.getTimestamp() < right.getTimestamp()) {
result = 1;
- } else if (this.timestamp > other.timestamp) {
+ } else if (left.getTimestamp() > right.getTimestamp()) {
result = -1;
}
return result;
@@ -482,9 +510,8 @@
if(rowCompare == 0)
rowCompare = Bytes.compareTo(keysA[1], KeysB[1]);
return rowCompare;
- } else {
- return Bytes.compareTo(rowA, rowB);
}
+ return Bytes.compareTo(rowA, rowB);
}
/**
@@ -513,10 +540,14 @@
break;
}
}
- byte [] row = new byte[offset];
- System.arraycopy(rowKey, 0, row, 0,offset);
- byte [] timestamp = new byte[rowKey.length - offset - 1];
- System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1);
+ byte [] row = rowKey;
+ byte [] timestamp = HConstants.EMPTY_BYTE_ARRAY;
+ if (offset != -1) {
+ row = new byte[offset];
+ System.arraycopy(rowKey, 0, row, 0, offset);
+ timestamp = new byte[rowKey.length - offset - 1];
+ System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1);
+ }
byte[][] elements = new byte[2][];
elements[0] = row;
elements[1] = timestamp;
@@ -538,4 +569,20 @@
this.column = Bytes.readByteArray(in);
this.timestamp = in.readLong();
}
+
+ /**
+ * Passed as comparator for memcache and for store files. See HBASE-868.
+ */
+ public static class HStoreKeyWritableComparator extends WritableComparator {
+ private final HRegionInfo hri;
+
+ public HStoreKeyWritableComparator(final HRegionInfo hri) {
+ super(HStoreKey.class);
+ this.hri = hri;
+ }
+
+ public int compare(final WritableComparable left, final WritableComparable right) {
+ return compareTo(this.hri, (HStoreKey)left, (HStoreKey)right);
+ }
+ }
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Sun Sep 7 17:14:10 2008
@@ -37,6 +37,7 @@
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.MasterNotRunningException;
@@ -323,7 +324,7 @@
if (currentRegion != null) {
byte[] endKey = currentRegion.getEndKey();
if (endKey == null ||
- Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)) {
+ HStoreKey.equalsTwoRowKeys(currentRegion, endKey, HConstants.EMPTY_BYTE_ARRAY)) {
// We have reached the end of the table and we're done
break;
}
@@ -636,8 +637,10 @@
// this one. the exception case is when the endkey is EMPTY_START_ROW,
// signifying that the region we're checking is actually the last
// region in the table.
- if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
- Bytes.compareTo(endKey, row) > 0) {
+ if (HStoreKey.equalsTwoRowKeys(possibleRegion.getRegionInfo(),
+ endKey, HConstants.EMPTY_END_ROW) ||
+ HStoreKey.compareTwoRowKeys(possibleRegion.getRegionInfo(),
+ endKey, row) > 0) {
return possibleRegion;
}
}
@@ -684,7 +687,8 @@
// by nature of the map, we know that the start key has to be <
// otherwise it wouldn't be in the headMap.
- if (Bytes.compareTo(endKey, row) <= 0) {
+ if (HStoreKey.compareTwoRowKeys(possibleRegion.getRegionInfo(),
+ endKey, row) <= 0) {
// delete any matching entry
HRegionLocation rl =
tableLocations.remove(matchingRegions.lastKey());
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java Sun Sep 7 17:14:10 2008
@@ -5,6 +5,7 @@
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.util.Bytes;
@@ -47,9 +48,10 @@
HRegionInfo.createRegionName(tableName, null, ZEROES);
// Scan over each meta region
+ ScannerCallable callable = null;
do {
- ScannerCallable callable = new ScannerCallable(connection,
- META_TABLE_NAME, COLUMN_FAMILY_ARRAY, startRow, LATEST_TIMESTAMP, null);
+ callable = new ScannerCallable(connection, META_TABLE_NAME,
+ COLUMN_FAMILY_ARRAY, startRow, LATEST_TIMESTAMP, null);
// Open scanner
connection.getRegionServerWithRetries(callable);
try {
@@ -67,7 +69,7 @@
callable.setClose();
connection.getRegionServerWithRetries(callable);
}
- } while (Bytes.compareTo(startRow, LAST_ROW) != 0);
+ } while (HStoreKey.compareTwoRowKeys(callable.getHRegionInfo(), startRow, LAST_ROW) != 0);
}
/**
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java Sun Sep 7 17:14:10 2008
@@ -20,7 +20,9 @@
package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.util.Bytes;
@@ -90,7 +92,8 @@
public int compareTo(MetaRegion other) {
int result = Bytes.compareTo(this.regionName, other.getRegionName());
if(result == 0) {
- result = Bytes.compareTo(this.startKey, other.getStartKey());
+ result = HStoreKey.compareTwoRowKeys(HRegionInfo.FIRST_META_REGIONINFO,
+ this.startKey, other.getStartKey());
if (result == 0) {
// Might be on different host?
result = this.server.compareTo(other.server);
@@ -98,4 +101,4 @@
}
return result;
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sun Sep 7 17:14:10 2008
@@ -109,7 +109,7 @@
static final Log LOG = LogFactory.getLog(HRegion.class);
final AtomicBoolean closed = new AtomicBoolean(false);
private final RegionHistorian historian;
-
+
/**
* Merge two HRegions. The regions must be adjacent andmust not overlap.
*
@@ -132,12 +132,14 @@
}
// A's start key is null but B's isn't. Assume A comes before B
} else if ((srcB.getStartKey() == null) // A is not null but B is
- || (Bytes.compareTo(srcA.getStartKey(), srcB.getStartKey()) > 0)) { // A > B
+ || (HStoreKey.compareTwoRowKeys(srcA.getRegionInfo(),
+ srcA.getStartKey(), srcB.getStartKey()) > 0)) { // A > B
a = srcB;
b = srcA;
}
- if (!Bytes.equals(a.getEndKey(), b.getStartKey())) {
+ if (!HStoreKey.equalsTwoRowKeys(srcA.getRegionInfo(),
+ a.getEndKey(), b.getStartKey())) {
throw new IOException("Cannot merge non-adjacent regions");
}
return merge(a, b);
@@ -181,13 +183,19 @@
HTableDescriptor tabledesc = a.getTableDesc();
HLog log = a.getLog();
Path basedir = a.getBaseDir();
- final byte [] startKey = Bytes.equals(a.getStartKey(), EMPTY_BYTE_ARRAY) ||
- Bytes.equals(b.getStartKey(), EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
- Bytes.compareTo(a.getStartKey(), b.getStartKey()) <= 0 ?
+ final byte [] startKey = HStoreKey.equalsTwoRowKeys(a.getRegionInfo(),
+ a.getStartKey(), EMPTY_BYTE_ARRAY) ||
+ HStoreKey.equalsTwoRowKeys(a.getRegionInfo(),
+ b.getStartKey(), EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
+ HStoreKey.compareTwoRowKeys(a.getRegionInfo(), a.getStartKey(),
+ b.getStartKey()) <= 0 ?
a.getStartKey() : b.getStartKey();
- final byte [] endKey = Bytes.equals(a.getEndKey(), EMPTY_BYTE_ARRAY) ||
- Bytes.equals(b.getEndKey(), EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
- Bytes.compareTo(a.getEndKey(), b.getEndKey()) <= 0 ?
+ final byte [] endKey = HStoreKey.equalsTwoRowKeys(a.getRegionInfo(),
+ a.getEndKey(), EMPTY_BYTE_ARRAY) ||
+ HStoreKey.equalsTwoRowKeys(b.getRegionInfo(), b.getEndKey(),
+ EMPTY_BYTE_ARRAY) ? EMPTY_BYTE_ARRAY :
+ HStoreKey.compareTwoRowKeys(a.getRegionInfo(), a.getEndKey(),
+ b.getEndKey()) <= 0 ?
b.getEndKey() : a.getEndKey();
HRegionInfo newRegionInfo = new HRegionInfo(tabledesc, startKey, endKey);
@@ -232,7 +240,7 @@
}
for (HStoreFile hsf: srcFiles) {
HStoreFile dst = new HStoreFile(conf, fs, basedir,
- newRegionInfo.getEncodedName(), colFamily, -1, null);
+ newRegionInfo, colFamily, -1, null);
if (LOG.isDebugEnabled()) {
LOG.debug("Renaming " + hsf + " to " + dst);
}
@@ -718,12 +726,12 @@
// Add start/end key checking: hbase-428.
byte [] startKey = this.regionInfo.getStartKey();
byte [] endKey = this.regionInfo.getEndKey();
- if (Bytes.equals(startKey, midKey)) {
+ if (HStoreKey.equalsTwoRowKeys(this.regionInfo,startKey, midKey)) {
LOG.debug("Startkey (" + startKey + ") and midkey + (" +
midKey + ") are same, not splitting");
return null;
}
- if (Bytes.equals(midKey, endKey)) {
+ if (HStoreKey.equalsTwoRowKeys(this.regionInfo,midKey, endKey)) {
LOG.debug("Endkey and midkey are same, not splitting");
return null;
}
@@ -769,15 +777,15 @@
// A reference to the bottom half of the hsf store file.
HStoreFile.Reference aReference = new HStoreFile.Reference(
this.regionInfo.getEncodedName(), h.getFileId(),
- new HStoreKey(midKey), HStoreFile.Range.bottom);
+ new HStoreKey(midKey, this.regionInfo), HStoreFile.Range.bottom);
HStoreFile a = new HStoreFile(this.conf, fs, splits,
- regionAInfo.getEncodedName(), h.getColFamily(), -1, aReference);
+ regionAInfo, h.getColFamily(), -1, aReference);
// Reference to top half of the hsf store file.
HStoreFile.Reference bReference = new HStoreFile.Reference(
this.regionInfo.getEncodedName(), h.getFileId(),
new HStoreKey(midKey), HStoreFile.Range.top);
HStoreFile b = new HStoreFile(this.conf, fs, splits,
- regionBInfo.getEncodedName(), h.getColFamily(), -1, bReference);
+ regionBInfo, h.getColFamily(), -1, bReference);
h.splitStoreFile(a, b, this.fs);
}
@@ -1142,7 +1150,7 @@
checkRow(row);
checkColumn(column);
// Don't need a row lock for a simple get
- HStoreKey key = new HStoreKey(row, column, timestamp);
+ HStoreKey key = new HStoreKey(row, column, timestamp, this.regionInfo);
Cell[] result = getStore(column).get(key, numVersions);
// Guarantee that we return null instead of a zero-length array,
// if there are no results to return.
@@ -1178,7 +1186,7 @@
checkColumn(column);
}
}
- HStoreKey key = new HStoreKey(row, ts);
+ HStoreKey key = new HStoreKey(row, ts, this.regionInfo);
Integer lid = getLock(lockid,row);
HashSet<HStore> storeSet = new HashSet<HStore>();
try {
@@ -1242,14 +1250,14 @@
byte [] closestKey = store.getRowKeyAtOrBefore(row);
// if it happens to be an exact match, we can stop looping
if (HStoreKey.equalsTwoRowKeys(regionInfo,row, closestKey)) {
- key = new HStoreKey(closestKey);
+ key = new HStoreKey(closestKey, this.regionInfo);
break;
}
// otherwise, we need to check if it's the max and move to the next
if (closestKey != null
&& (key == null || HStoreKey.compareTwoRowKeys(
regionInfo,closestKey, key.getRow()) > 0) ) {
- key = new HStoreKey(closestKey);
+ key = new HStoreKey(closestKey, this.regionInfo);
}
}
if (key == null) {
@@ -1401,7 +1409,8 @@
try {
List<byte []> deletes = null;
for (BatchOperation op: b) {
- HStoreKey key = new HStoreKey(row, op.getColumn(), commitTime);
+ HStoreKey key = new HStoreKey(row, op.getColumn(), commitTime,
+ this.regionInfo);
byte[] val = null;
if (op.isPut()) {
val = op.getValue();
@@ -1524,7 +1533,7 @@
long now = System.currentTimeMillis();
try {
for (HStore store : stores.values()) {
- List<HStoreKey> keys = store.getKeys(new HStoreKey(row, ts),
+ List<HStoreKey> keys = store.getKeys(new HStoreKey(row, ts, this.regionInfo),
ALL_VERSIONS, now);
TreeMap<HStoreKey, byte []> edits = new TreeMap<HStoreKey, byte []>();
for (HStoreKey key: keys) {
@@ -1557,8 +1566,8 @@
// find the HStore for the column family
HStore store = getStore(family);
// find all the keys that match our criteria
- List<HStoreKey> keys = store.getKeys(new HStoreKey(row, timestamp),
- ALL_VERSIONS, now);
+ List<HStoreKey> keys = store.getKeys(new HStoreKey(row, timestamp,
+ this.regionInfo), ALL_VERSIONS, now);
// delete all the cells
TreeMap<HStoreKey, byte []> edits = new TreeMap<HStoreKey, byte []>();
for (HStoreKey key: keys) {
@@ -1585,7 +1594,7 @@
final long ts, final int versions)
throws IOException {
checkReadOnly();
- HStoreKey origin = new HStoreKey(row, column, ts);
+ HStoreKey origin = new HStoreKey(row, column, ts, this.regionInfo);
Set<HStoreKey> keys = getKeys(origin, versions);
if (keys.size() > 0) {
TreeMap<HStoreKey, byte []> edits = new TreeMap<HStoreKey, byte []>();
@@ -1881,7 +1890,7 @@
}
}
}
-
+
/** {@inheritDoc} */
@Override
public boolean equals(Object o) {
@@ -1958,7 +1967,7 @@
this.resultSets = new TreeMap[scanners.length];
this.keys = new HStoreKey[scanners.length];
for (int i = 0; i < scanners.length; i++) {
- keys[i] = new HStoreKey();
+ keys[i] = new HStoreKey(HConstants.EMPTY_BYTE_ARRAY,regionInfo);
resultSets[i] = new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
if(scanners[i] != null && !scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
@@ -1983,9 +1992,11 @@
long chosenTimestamp = -1;
for (int i = 0; i < this.keys.length; i++) {
if (scanners[i] != null &&
- (chosenRow == null ||
- (Bytes.compareTo(keys[i].getRow(), chosenRow) < 0) ||
- ((Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) &&
+ (chosenRow == null ||
+ (HStoreKey.compareTwoRowKeys(regionInfo,
+ keys[i].getRow(), chosenRow) < 0) ||
+ ((HStoreKey.compareTwoRowKeys(regionInfo, keys[i].getRow(),
+ chosenRow) == 0) &&
(keys[i].getTimestamp() > chosenTimestamp)))) {
chosenRow = keys[i].getRow();
chosenTimestamp = keys[i].getTimestamp();
@@ -2002,7 +2013,7 @@
for (int i = 0; i < scanners.length; i++) {
if (scanners[i] != null &&
- Bytes.compareTo(keys[i].getRow(), chosenRow) == 0) {
+ HStoreKey.compareTwoRowKeys(regionInfo,keys[i].getRow(), chosenRow) == 0) {
// NOTE: We used to do results.putAll(resultSets[i]);
// but this had the effect of overwriting newer
// values with older ones. So now we only insert
@@ -2024,7 +2035,7 @@
// If the current scanner is non-null AND has a lower-or-equal
// row label, then its timestamp is bad. We need to advance it.
while ((scanners[i] != null) &&
- (Bytes.compareTo(keys[i].getRow(), chosenRow) <= 0)) {
+ (HStoreKey.compareTwoRowKeys(regionInfo,keys[i].getRow(), chosenRow) <= 0)) {
resultSets[i].clear();
if (!scanners[i].next(keys[i], resultSets[i])) {
closeScanner(i);
@@ -2206,7 +2217,8 @@
byte [] row = r.getRegionName();
Integer lid = meta.obtainRowLock(row);
try {
- HStoreKey key = new HStoreKey(row, COL_REGIONINFO, System.currentTimeMillis());
+ HStoreKey key = new HStoreKey(row, COL_REGIONINFO,
+ System.currentTimeMillis(), r.getRegionInfo());
TreeMap<HStoreKey, byte[]> edits = new TreeMap<HStoreKey, byte[]>();
edits.put(key, Writables.getBytes(r.getRegionInfo()));
meta.update(edits);
@@ -2307,9 +2319,9 @@
*/
public static boolean rowIsInRange(HRegionInfo info, final byte [] row) {
return ((info.getStartKey().length == 0) ||
- (Bytes.compareTo(info.getStartKey(), row) <= 0)) &&
+ (HStoreKey.compareTwoRowKeys(info,info.getStartKey(), row) <= 0)) &&
((info.getEndKey().length == 0) ||
- (Bytes.compareTo(info.getEndKey(), row) > 0));
+ (HStoreKey.compareTwoRowKeys(info,info.getEndKey(), row) > 0));
}
/**
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=692963&r1=692962&r2=692963&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Sun Sep 7 17:14:10 2008
@@ -188,7 +188,12 @@
}
this.desiredMaxFileSize = maxFileSize;
- this.majorCompactionTime = conf.getLong("hbase.hregion.majorcompaction", 86400000);
+ this.majorCompactionTime = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 86400000);
+ if (family.getValue(HConstants.MAJOR_COMPACTION_PERIOD) != null) {
+ String strCompactionTime = family.getValue(HConstants.MAJOR_COMPACTION_PERIOD);
+ this.majorCompactionTime = (new Long(strCompactionTime)).longValue();
+ }
+
this.maxFilesToCompact = conf.getInt("hbase.hstore.compaction.max", 10);
this.storeSize = 0L;
@@ -319,7 +324,8 @@
|| !HStoreKey.matchingFamily(family.getName(), column)) {
continue;
}
- HStoreKey k = new HStoreKey(key.getRow(), column, val.getTimestamp());
+ HStoreKey k = new HStoreKey(key.getRow(), column, val.getTimestamp(),
+ this.info);
reconstructedCache.put(k, val.getVal());
editsCount++;
// Every 2k edits, tell the reporter we're making progress.
@@ -390,7 +396,7 @@
if (isReference) {
reference = HStoreFile.readSplitInfo(p, fs);
}
- curfile = new HStoreFile(conf, fs, basedir, info.getEncodedName(),
+ curfile = new HStoreFile(conf, fs, basedir, this.info,
family.getName(), fid, reference);
long storeSeqId = -1;
try {
@@ -424,7 +430,9 @@
// Try fixing this file.. if we can. Use the hbase version of fix.
// Need to remove the old index file first else fix won't go ahead.
this.fs.delete(new Path(mapfile, MapFile.INDEX_FILE_NAME), false);
- long count = MapFile.fix(this.fs, mapfile, HStoreFile.HbaseMapFile.KEY_CLASS,
+ // TODO: This is going to fail if we are to rebuild a file from
+ // meta because it won't have right comparator: HBASE-848.
+ long count = MapFile.fix(this.fs, mapfile, HStoreKey.class,
HStoreFile.HbaseMapFile.VALUE_CLASS, false, this.conf);
if (LOG.isDebugEnabled()) {
LOG.debug("Fixed index on " + mapfile.toString() + "; had " +
@@ -589,7 +597,7 @@
long now = System.currentTimeMillis();
// A. Write the Maps out to the disk
HStoreFile flushedFile = new HStoreFile(conf, fs, basedir,
- info.getEncodedName(), family.getName(), -1L, null);
+ this.info, family.getName(), -1L, null);
MapFile.Writer out = flushedFile.getWriter(this.fs, this.compression,
this.family.isBloomfilter(), cache.size());
out.setIndexInterval(family.getMapFileIndexInterval());
@@ -873,8 +881,7 @@
// Step through them, writing to the brand-new MapFile
HStoreFile compactedOutputFile = new HStoreFile(conf, fs,
- this.compactionDir, info.getEncodedName(), family.getName(),
- -1L, null);
+ this.compactionDir, this.info, family.getName(), -1L, null);
if (LOG.isDebugEnabled()) {
LOG.debug("started compaction of " + rdrs.size() + " files into " +
FSUtils.getPath(compactedOutputFile.getMapFilePath()));
@@ -962,7 +969,7 @@
}
}
HStoreKey sk = keys[smallestKey];
- if (Bytes.equals(lastRow, sk.getRow())
+ if (HStoreKey.equalsTwoRowKeys(info,lastRow, sk.getRow())
&& Bytes.equals(lastColumn, sk.getColumn())) {
timesSeen++;
} else {
@@ -1045,7 +1052,7 @@
try {
// 1. Moving the new MapFile into place.
HStoreFile finalCompactedFile = new HStoreFile(conf, fs, basedir,
- info.getEncodedName(), family.getName(), -1, null);
+ this.info, family.getName(), -1, null);
if (LOG.isDebugEnabled()) {
LOG.debug("moving " + FSUtils.getPath(compactedFile.getMapFilePath()) +
" to " + FSUtils.getPath(finalCompactedFile.getMapFilePath()));
@@ -1207,7 +1214,7 @@
}
}
}
- } else if (Bytes.compareTo(key.getRow(), readkey.getRow()) < 0) {
+ } else if (HStoreKey.compareTwoRowKeys(info,key.getRow(), readkey.getRow()) < 0) {
// if we've crossed into the next row, then we can just stop
// iterating
break;
@@ -1774,7 +1781,7 @@
}
static HStoreKey stripTimestamp(HStoreKey key) {
- return new HStoreKey(key.getRow(), key.getColumn());
+ return new HStoreKey(key.getRow(), key.getColumn(), key.getHRegionInfo());
}
/*
@@ -1789,7 +1796,7 @@
// if the origin's column is empty, then we're matching any column
if (Bytes.equals(origin.getColumn(), HConstants.EMPTY_BYTE_ARRAY)) {
// if the row matches, then...
- if (Bytes.equals(target.getRow(), origin.getRow())) {
+ if (HStoreKey.equalsTwoRowKeys(info, target.getRow(), origin.getRow())) {
// check the timestamp
return target.getTimestamp() <= origin.getTimestamp();
}
@@ -1810,7 +1817,7 @@
// if the origin's column is empty, then we're matching any column
if (Bytes.equals(origin.getColumn(), HConstants.EMPTY_BYTE_ARRAY)) {
// if the row matches, then...
- return Bytes.equals(target.getRow(), origin.getRow());
+ return HStoreKey.equalsTwoRowKeys(info, target.getRow(), origin.getRow());
}
// otherwise, we want to match on row and column
return target.matchesRowCol(origin);
@@ -1869,8 +1876,8 @@
if (mk != null) {
// if the midkey is the same as the first and last keys, then we cannot
// (ever) split this region.
- if (Bytes.equals(mk.getRow(), firstKey.getRow()) &&
- Bytes.equals(mk.getRow(), lastKey.getRow())) {
+ if (HStoreKey.equalsTwoRowKeys(info, mk.getRow(), firstKey.getRow()) &&
+ HStoreKey.equalsTwoRowKeys(info, mk.getRow(), lastKey.getRow())) {
return null;
}
return new StoreSize(maxSize, mk.getRow());
@@ -1957,4 +1964,8 @@
return key;
}
}
+
+ HRegionInfo getHRegionInfo() {
+ return this.info;
+ }
}