You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/02/19 20:04:03 UTC
svn commit: r629192 - in /hadoop/hbase: branches/0.1/
branches/0.1/src/java/org/apache/hadoop/hbase/ trunk/
trunk/src/java/org/apache/hadoop/hbase/
Author: stack
Date: Tue Feb 19 11:03:57 2008
New Revision: 629192
URL: http://svn.apache.org/viewvc?rev=629192&view=rev
Log:
HBASE-428 Under continuous upload of rows, WrongRegionExceptions are thrown
that reach the client even after retries
Applied to TRUNK and branch.
Modified:
hadoop/hbase/branches/0.1/CHANGES.txt
hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java
hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegionServer.java
hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java
hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionServer.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStore.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreFile.java
Modified: hadoop/hbase/branches/0.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/CHANGES.txt?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.1/CHANGES.txt Tue Feb 19 11:03:57 2008
@@ -1,7 +1,7 @@
HBase Change Log
-Release 0.1.0
+Unreleased
INCOMPATIBLE CHANGES
HADOOP-2750 Deprecated methods startBatchUpdate, commitBatch, abortBatch,
@@ -26,6 +26,8 @@
HBase-421 TestRegionServerExit broken
HBASE-426 hbase can't find remote filesystem
HBASE-446 Fully qualified hbase.rootdir doesn't work
+ HBASE-428 Under continuous upload of rows, WrongRegionExceptions are
+ thrown that reach the client even after retries
IMPROVEMENTS
HADOOP-2555 Refactor the HTable#get and HTable#getRow methods to avoid
Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java Tue Feb 19 11:03:57 2008
@@ -536,7 +536,7 @@
HStore.HStoreSize largestHStore(Text midkey) {
HStore.HStoreSize biggest = null;
boolean splitable = true;
- for(HStore h: stores.values()) {
+ for (HStore h: stores.values()) {
HStore.HStoreSize size = h.size(midkey);
// If we came across a reference down in the store, then propagate
// fact that region is not splitable.
@@ -577,14 +577,25 @@
if(!this.fs.exists(splits)) {
this.fs.mkdirs(splits);
}
+ // Make copies just in case and add start/end key checking: hbase-428.
+ Text startKey = new Text(this.regionInfo.getStartKey());
+ Text endKey = new Text(this.regionInfo.getEndKey());
+ if (startKey.equals(midKey)) {
+ LOG.debug("Startkey and midkey are same, not splitting");
+ return null;
+ }
+ if (midKey.equals(endKey)) {
+ LOG.debug("Endkey and midkey are same, not splitting");
+ return null;
+ }
HRegionInfo regionAInfo = new HRegionInfo(this.regionInfo.getTableDesc(),
- this.regionInfo.getStartKey(), midKey);
+ startKey, midKey);
Path dirA = new Path(splits, regionAInfo.getEncodedName());
if(fs.exists(dirA)) {
throw new IOException("Cannot split; target file collision at " + dirA);
}
HRegionInfo regionBInfo = new HRegionInfo(this.regionInfo.getTableDesc(),
- midKey, this.regionInfo.getEndKey());
+ midKey, endKey);
Path dirB = new Path(splits, regionBInfo.getEncodedName());
if(this.fs.exists(dirB)) {
throw new IOException("Cannot split; target file collision at " + dirB);
Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegionServer.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegionServer.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegionServer.java Tue Feb 19 11:03:57 2008
@@ -343,9 +343,9 @@
reportSplit(oldRegionInfo, newRegions[0].getRegionInfo(),
newRegions[1].getRegionInfo());
LOG.info("region split, META updated, and report to master all" +
- " successful. Old region=" + oldRegionInfo.getRegionName() +
- ", new regions: " + newRegions[0].getRegionName() + ", " +
- newRegions[1].getRegionName() + ". Split took " +
+ " successful. Old region=" + oldRegionInfo.toString() +
+ ", new regions: " + newRegions[0].toString() + ", " +
+ newRegions[1].toString() + ". Split took " +
StringUtils.formatTimeDiff(System.currentTimeMillis(), startTime));
// Do not serve the new regions. Let the Master assign them.
Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java Tue Feb 19 11:03:57 2008
@@ -243,12 +243,11 @@
if ( (key_memcache != null && key_memcache.equals(row))
|| (key_snapshot != null && key_snapshot.equals(row)) ) {
return row;
- } else {
- // no precise matches, so return the one that is closer to the search
- // key (greatest)
- return key_memcache.compareTo(key_snapshot) > 0 ?
- key_memcache : key_snapshot;
- }
+ }
+ // no precise matches, so return the one that is closer to the search
+ // key (greatest)
+ return key_memcache.compareTo(key_snapshot) > 0 ?
+ key_memcache : key_snapshot;
}
} finally {
this.lock.readLock().unlock();
@@ -293,13 +292,7 @@
// the tail didn't contain the key we're searching for, so we should
// use the last key in the headmap as the closest before
SortedMap<HStoreKey, byte []> headMap = map.headMap(search_key);
- if (headMap.isEmpty()) {
-/* LOG.debug("Went searching for " + key + ", found nothing!");*/
- return null;
- } else {
-/* LOG.debug("Went searching for " + key + ", found " + headMap.lastKey().getRow());*/
- return headMap.lastKey().getRow();
- }
+ return headMap.isEmpty()? null: headMap.lastKey().getRow();
}
/**
@@ -1823,9 +1816,7 @@
MapFile.Reader[] maparray = getReaders();
Text bestSoFar = null;
-
- HStoreKey rowKey = new HStoreKey(row, timestamp);
-
+
// process each store file
for(int i = maparray.length - 1; i >= 0; i--) {
Text row_from_mapfile =
@@ -2012,7 +2003,7 @@
try {
Long mapIndex = Long.valueOf(0L);
// Iterate through all the MapFiles
- for(Map.Entry<Long, HStoreFile> e: storefiles.entrySet()) {
+ for (Map.Entry<Long, HStoreFile> e: storefiles.entrySet()) {
HStoreFile curHSF = e.getValue();
long size = curHSF.length();
aggregateSize += size;
@@ -2025,29 +2016,28 @@
splitable = !curHSF.isReference();
}
}
- MapFile.Reader r = this.readers.get(mapIndex);
-
- // seek back to the beginning of mapfile
- r.reset();
-
- // get the first and last keys
- HStoreKey firstKey = new HStoreKey();
- HStoreKey lastKey = new HStoreKey();
- Writable value = new ImmutableBytesWritable();
- r.next((WritableComparable)firstKey, value);
- r.finalKey((WritableComparable)lastKey);
-
- // get the midkey
- HStoreKey midkey = (HStoreKey)r.midKey();
-
- if (midkey != null) {
- midKey.set(((HStoreKey)midkey).getRow());
- // if the midkey is the same as the first and last keys, then we cannot
- // (ever) split this region.
- if (midkey.getRow().equals(firstKey.getRow()) &&
- midkey.getRow().equals(lastKey.getRow())) {
- return new HStoreSize(aggregateSize, maxSize, false);
- }
+ if (splitable) {
+ MapFile.Reader r = this.readers.get(mapIndex);
+ // seek back to the beginning of mapfile
+ r.reset();
+ // get the first and last keys
+ HStoreKey firstKey = new HStoreKey();
+ HStoreKey lastKey = new HStoreKey();
+ Writable value = new ImmutableBytesWritable();
+ r.next(firstKey, value);
+ r.finalKey(lastKey);
+ // get the midkey
+ HStoreKey mk = (HStoreKey)r.midKey();
+ if (mk != null) {
+ // if the midkey is the same as the first and last keys, then we cannot
+ // (ever) split this region.
+ if (mk.getRow().equals(firstKey.getRow()) &&
+ mk.getRow().equals(lastKey.getRow())) {
+ return new HStoreSize(aggregateSize, maxSize, false);
+ }
+ // Otherwise, set midKey
+ midKey.set(mk.getRow());
+ }
}
} catch(IOException e) {
LOG.warn("Failed getting store size for " + this.storeName, e);
Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java Tue Feb 19 11:03:57 2008
@@ -776,12 +776,10 @@
public synchronized void finalKey(WritableComparable key)
throws IOException {
if (top) {
- checkKey(key);
super.finalKey(key);
} else {
reset();
Writable value = new ImmutableBytesWritable();
-
key = super.getClosest(midkey, value, true);
}
}
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue Feb 19 11:03:57 2008
@@ -22,6 +22,9 @@
HBASE-446 Fully qualified hbase.rootdir doesn't work
HBASE-438 XMLOutputter state should be initialized. (Edward Yoon via Stack)
HBASE-8 Delete table does not remove the table directory in the FS
+ HBASE-428 Under continuous upload of rows, WrongRegionExceptions are thrown
+ that reach the client even after retries
+
IMPROVEMENTS
HBASE-415 Rewrite leases to use DelayedBlockingQueue instead of polling
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegion.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegion.java Tue Feb 19 11:03:57 2008
@@ -536,7 +536,7 @@
HStore.HStoreSize largestHStore(Text midkey) {
HStore.HStoreSize biggest = null;
boolean splitable = true;
- for(HStore h: stores.values()) {
+ for (HStore h: stores.values()) {
HStore.HStoreSize size = h.size(midkey);
// If we came across a reference down in the store, then propagate
// fact that region is not splitable.
@@ -577,14 +577,25 @@
if(!this.fs.exists(splits)) {
this.fs.mkdirs(splits);
}
+ // Make copies just in case and add start/end key checking: hbase-428.
+ Text startKey = new Text(this.regionInfo.getStartKey());
+ Text endKey = new Text(this.regionInfo.getEndKey());
+ if (startKey.equals(midKey)) {
+ LOG.debug("Startkey and midkey are same, not splitting");
+ return null;
+ }
+ if (midKey.equals(endKey)) {
+ LOG.debug("Endkey and midkey are same, not splitting");
+ return null;
+ }
HRegionInfo regionAInfo = new HRegionInfo(this.regionInfo.getTableDesc(),
- this.regionInfo.getStartKey(), midKey);
+ startKey, midKey);
Path dirA = new Path(splits, regionAInfo.getEncodedName());
if(fs.exists(dirA)) {
throw new IOException("Cannot split; target file collision at " + dirA);
}
HRegionInfo regionBInfo = new HRegionInfo(this.regionInfo.getTableDesc(),
- midKey, this.regionInfo.getEndKey());
+ midKey, endKey);
Path dirB = new Path(splits, regionBInfo.getEncodedName());
if(this.fs.exists(dirB)) {
throw new IOException("Cannot split; target file collision at " + dirB);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionServer.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HRegionServer.java Tue Feb 19 11:03:57 2008
@@ -334,11 +334,6 @@
update.put(COL_REGIONINFO, Writables.getBytes(
newRegions[i].getRegionInfo()));
t.commit(update);
-
-/* long lockid = t.startUpdate(newRegions[i].getRegionName());
- t.put(lockid, COL_REGIONINFO, Writables.getBytes(
- newRegions[i].getRegionInfo()));
- t.commit(lockid);*/
}
// Now tell the master about the new regions
@@ -348,9 +343,9 @@
reportSplit(oldRegionInfo, newRegions[0].getRegionInfo(),
newRegions[1].getRegionInfo());
LOG.info("region split, META updated, and report to master all" +
- " successful. Old region=" + oldRegionInfo.getRegionName() +
- ", new regions: " + newRegions[0].getRegionName() + ", " +
- newRegions[1].getRegionName() + ". Split took " +
+ " successful. Old region=" + oldRegionInfo.toString() +
+ ", new regions: " + newRegions[0].toString() + ", " +
+ newRegions[1].toString() + ". Split took " +
StringUtils.formatTimeDiff(System.currentTimeMillis(), startTime));
// Do not serve the new regions. Let the Master assign them.
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStore.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStore.java Tue Feb 19 11:03:57 2008
@@ -243,12 +243,11 @@
if ( (key_memcache != null && key_memcache.equals(row))
|| (key_snapshot != null && key_snapshot.equals(row)) ) {
return row;
- } else {
- // no precise matches, so return the one that is closer to the search
- // key (greatest)
- return key_memcache.compareTo(key_snapshot) > 0 ?
- key_memcache : key_snapshot;
- }
+ }
+ // no precise matches, so return the one that is closer to the search
+ // key (greatest)
+ return key_memcache.compareTo(key_snapshot) > 0 ?
+ key_memcache : key_snapshot;
}
} finally {
this.lock.readLock().unlock();
@@ -293,13 +292,7 @@
// the tail didn't contain the key we're searching for, so we should
// use the last key in the headmap as the closest before
SortedMap<HStoreKey, byte []> headMap = map.headMap(search_key);
- if (headMap.isEmpty()) {
-/* LOG.debug("Went searching for " + key + ", found nothing!");*/
- return null;
- } else {
-/* LOG.debug("Went searching for " + key + ", found " + headMap.lastKey().getRow());*/
- return headMap.lastKey().getRow();
- }
+ return headMap.isEmpty()? null: headMap.lastKey().getRow();
}
/**
@@ -1836,9 +1829,7 @@
MapFile.Reader[] maparray = getReaders();
Text bestSoFar = null;
-
- HStoreKey rowKey = new HStoreKey(row, timestamp);
-
+
// process each store file
for(int i = maparray.length - 1; i >= 0; i--) {
Text row_from_mapfile =
@@ -2025,7 +2016,7 @@
try {
Long mapIndex = Long.valueOf(0L);
// Iterate through all the MapFiles
- for(Map.Entry<Long, HStoreFile> e: storefiles.entrySet()) {
+ for (Map.Entry<Long, HStoreFile> e: storefiles.entrySet()) {
HStoreFile curHSF = e.getValue();
long size = curHSF.length();
aggregateSize += size;
@@ -2038,29 +2029,28 @@
splitable = !curHSF.isReference();
}
}
- MapFile.Reader r = this.readers.get(mapIndex);
-
- // seek back to the beginning of mapfile
- r.reset();
-
- // get the first and last keys
- HStoreKey firstKey = new HStoreKey();
- HStoreKey lastKey = new HStoreKey();
- Writable value = new ImmutableBytesWritable();
- r.next((WritableComparable)firstKey, value);
- r.finalKey((WritableComparable)lastKey);
-
- // get the midkey
- HStoreKey midkey = (HStoreKey)r.midKey();
-
- if (midkey != null) {
- midKey.set(((HStoreKey)midkey).getRow());
- // if the midkey is the same as the first and last keys, then we cannot
- // (ever) split this region.
- if (midkey.getRow().equals(firstKey.getRow()) &&
- midkey.getRow().equals(lastKey.getRow())) {
- return new HStoreSize(aggregateSize, maxSize, false);
- }
+ if (splitable) {
+ MapFile.Reader r = this.readers.get(mapIndex);
+ // seek back to the beginning of mapfile
+ r.reset();
+ // get the first and last keys
+ HStoreKey firstKey = new HStoreKey();
+ HStoreKey lastKey = new HStoreKey();
+ Writable value = new ImmutableBytesWritable();
+ r.next(firstKey, value);
+ r.finalKey(lastKey);
+ // get the midkey
+ HStoreKey mk = (HStoreKey)r.midKey();
+ if (mk != null) {
+ // if the midkey is the same as the first and last keys, then we cannot
+ // (ever) split this region.
+ if (mk.getRow().equals(firstKey.getRow()) &&
+ mk.getRow().equals(lastKey.getRow())) {
+ return new HStoreSize(aggregateSize, maxSize, false);
+ }
+ // Otherwise, set midKey
+ midKey.set(mk.getRow());
+ }
}
} catch(IOException e) {
LOG.warn("Failed getting store size for " + this.storeName, e);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreFile.java?rev=629192&r1=629191&r2=629192&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreFile.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HStoreFile.java Tue Feb 19 11:03:57 2008
@@ -861,12 +861,10 @@
public synchronized void finalKey(WritableComparable key)
throws IOException {
if (top) {
- checkKey(key);
super.finalKey(key);
} else {
reset();
Writable value = new ImmutableBytesWritable();
-
key = super.getClosest(midkey, value, true);
}
}