You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2008/01/24 06:13:27 UTC
svn commit: r614796 - in /hadoop/core/trunk/src/contrib/hbase: CHANGES.txt
src/java/org/apache/hadoop/hbase/HMaster.java
src/java/org/apache/hadoop/hbase/HStoreFile.java
Author: stack
Date: Wed Jan 23 21:13:22 2008
New Revision: 614796
URL: http://svn.apache.org/viewvc?rev=614796&view=rev
Log:
HADOOP-2688 IllegalArgumentException processing a shutdown stops server
going down and results in millions of lines of output
Modified:
hadoop/core/trunk/src/contrib/hbase/CHANGES.txt
hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java
Modified: hadoop/core/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hbase/CHANGES.txt?rev=614796&r1=614795&r2=614796&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ hadoop/core/trunk/src/contrib/hbase/CHANGES.txt Wed Jan 23 21:13:22 2008
@@ -146,6 +146,8 @@
HADOOP-2668 Documentation and improved logging so fact that hbase now
requires migration comes as less of a surprise
HADOOP-2686 Removed tables stick around in .META.
+ HADOOP-2688 IllegalArgumentException processing a shutdown stops
+ server going down and results in millions of lines of output
IMPROVEMENTS
HADOOP-2401 Add convenience put method that takes writable
Modified: hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java?rev=614796&r1=614795&r2=614796&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (original)
+++ hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java Wed Jan 23 21:13:22 2008
@@ -223,24 +223,19 @@
int numberOfRegionsFound = 0;
while (true) {
- SortedMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
HbaseMapWritable values = regionServer.next(scannerId);
if (values == null || values.size() == 0) {
break;
}
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey key = (HStoreKey) e.getKey();
- results.put(key.getColumn(),
- ((ImmutableBytesWritable) e.getValue()).get());
- }
- byte [] bytes = results.get(COL_REGIONINFO);
- if (bytes == null) {
- LOG.warn(COL_REGIONINFO.toString() + " is empty; has keys: " +
- values.keySet().toString());
+
+ // TODO: Why does this have to be a sorted map?
+ SortedMap<Text, byte[]> results = toRowMap(values).getMap();
+
+ HRegionInfo info = getHRegionInfo(results);
+ if (info == null) {
continue;
}
- HRegionInfo info = (HRegionInfo) Writables.getWritable(bytes,
- new HRegionInfo());
+
String serverName = Writables.bytesToString(results.get(COL_SERVER));
long startCode = Writables.bytesToLong(results.get(COL_STARTCODE));
if (LOG.isDebugEnabled()) {
@@ -2007,28 +2002,13 @@
RemoteExceptionHandler.checkIOException(e));
break;
}
-
if (values == null || values.size() == 0) {
break;
}
-
- SortedMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
- Text row = null;
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey key = (HStoreKey) e.getKey();
- Text thisRow = key.getRow();
- if (row == null) {
- row = thisRow;
- } else {
- if (!row.equals(thisRow)) {
- LOG.error("Multiple rows in same scanner result set. firstRow="
- + row + ", currentRow=" + thisRow);
- }
- }
- results.put(key.getColumn(),
- ((ImmutableBytesWritable) e.getValue()).get());
- }
-
+ // TODO: Why does this have to be a sorted map?
+ RowMap rm = toRowMap(values);
+ Text row = rm.getRow();
+ SortedMap<Text, byte[]> map = rm.getMap();
if (LOG.isDebugEnabled() && row != null) {
LOG.debug("shutdown scanner looking at " + row.toString());
}
@@ -2038,8 +2018,8 @@
// missed edits in hlog because hdfs does not do write-append).
String serverName;
try {
- serverName = Writables.bytesToString(results.get(COL_SERVER));
- } catch(UnsupportedEncodingException e) {
+ serverName = Writables.bytesToString(map.get(COL_SERVER));
+ } catch (UnsupportedEncodingException e) {
LOG.error("Server name", e);
break;
}
@@ -2054,13 +2034,9 @@
}
// Bingo! Found it.
- HRegionInfo info = null;
- try {
- info = (HRegionInfo) Writables.getWritable(
- results.get(COL_REGIONINFO), new HRegionInfo());
- } catch (IOException e) {
- LOG.error("Read fields", e);
- break;
+ HRegionInfo info = getHRegionInfo(map);
+ if (info == null) {
+ continue;
}
LOG.info(info.getRegionName() + " was on shutdown server <" +
serverName + "> (or server is null). Marking unassigned in " +
@@ -2720,52 +2696,21 @@
try {
while (true) {
- HRegionInfo info = new HRegionInfo();
- String serverName = null;
- long startCode = -1L;
-
HbaseMapWritable values = server.next(scannerId);
if(values == null || values.size() == 0) {
break;
}
- boolean haveRegionInfo = false;
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
-
- byte[] value = ((ImmutableBytesWritable) e.getValue()).get();
- if (value == null || value.length == 0) {
- break;
- }
- HStoreKey key = (HStoreKey) e.getKey();
- Text column = key.getColumn();
- if (column.equals(COL_REGIONINFO)) {
- haveRegionInfo = true;
- info = (HRegionInfo) Writables.getWritable(value, info);
-
- } else if (column.equals(COL_SERVER)) {
- try {
- serverName =
- Writables.bytesToString(value);
-
- } catch (UnsupportedEncodingException ex) {
- assert(false);
- }
-
- } else if (column.equals(COL_STARTCODE)) {
- try {
- startCode = Writables.bytesToLong(value);
-
- } catch (UnsupportedEncodingException ex) {
- assert(false);
- }
- }
- }
-
- if (!haveRegionInfo) {
- throw new IOException(COL_REGIONINFO + " not found");
+ RowMap rm = toRowMap(values);
+ SortedMap<Text, byte[]> map = rm.getMap();
+ HRegionInfo info = getHRegionInfo(map);
+ if (info == null) {
+ throw new IOException(COL_REGIONINFO + " not found on " +
+ rm.getRow());
}
-
+ String serverName = Writables.bytesToString(map.get(COL_SERVER));
+ long startCode = Writables.bytesToLong(map.get(COL_STARTCODE));
if (info.getTableDesc().getName().compareTo(tableName) > 0) {
- break; // Beyond any more entries for this table
+ break; // Beyond any more entries for this table
}
tableExists = true;
@@ -2773,9 +2718,7 @@
unservedRegions.add(info);
}
processScanItem(serverName, startCode, info);
-
} // while(true)
-
} finally {
if (scannerId != -1L) {
try {
@@ -2994,7 +2937,7 @@
protected void updateRegionInfo(BatchUpdate b,
@SuppressWarnings("unused") HRegionInfo info) {
for (int i = 0; i < ALL_META_COLUMNS.length; i++) {
- // Be sure to clean all columns
+ // Be sure to clean all cells
b.delete(lockid, ALL_META_COLUMNS[i]);
}
}
@@ -3169,6 +3112,75 @@
*/
public HBaseConfiguration getConfiguration() {
return this.conf;
+ }
+
+ /*
+ * Data structure used to return results out of the toRowMap method.
+ */
+ private class RowMap {
+ final Text row;
+ final SortedMap<Text, byte[]> map;
+
+ private RowMap(final Text r, final SortedMap<Text, byte[]> m) {
+ this.row = r;
+ this.map = m;
+ }
+
+ private Text getRow() {
+ return this.row;
+ }
+
+ private SortedMap<Text, byte[]> getMap() {
+ return this.map;
+ }
+ }
+
+ /*
+ * Convert an HbaseMapWritable to a Map keyed by column.
+ * Utility method used scanning meta regions
+ * @param mw The MapWritable to convert. Cannot be null.
+ * @return Returns a SortedMap currently. TODO: This looks like it could
+ * be a plain Map.
+ */
+ protected RowMap toRowMap(final HbaseMapWritable mw) {
+ if (mw == null) {
+ throw new IllegalArgumentException("Passed MapWritable cannot be null");
+ }
+ SortedMap<Text, byte[]> m = new TreeMap<Text, byte[]>();
+ Text row = null;
+ for (Map.Entry<Writable, Writable> e: mw.entrySet()) {
+ HStoreKey key = (HStoreKey) e.getKey();
+ Text thisRow = key.getRow();
+ if (row == null) {
+ row = thisRow;
+ } else {
+ if (!row.equals(thisRow)) {
+ LOG.error("Multiple rows in same scanner result set. firstRow=" +
+ row + ", currentRow=" + thisRow);
+ }
+ }
+ m.put(key.getColumn(), ((ImmutableBytesWritable) e.getValue()).get());
+ }
+ return new RowMap(row, m);
+ }
+
+ /*
+ * Get HRegionInfo from passed META map of row values.
+ * Returns null if none found (and logs fact that expected COL_REGIONINFO
+ * was missing). Utility method used by scanners of META tables.
+ * @param map Map to do lookup in.
+ * @return Null or found HRegionInfo.
+ * @throws IOException
+ */
+ protected HRegionInfo getHRegionInfo(final Map<Text, byte[]> map)
+ throws IOException {
+ byte [] bytes = map.get(COL_REGIONINFO);
+ if (bytes == null) {
+ LOG.warn(COL_REGIONINFO.toString() + " is empty; has keys: " +
+ map.keySet().toString());
+ return null;
+ }
+ return (HRegionInfo)Writables.getWritable(bytes, new HRegionInfo());
}
/*
Modified: hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java?rev=614796&r1=614795&r2=614796&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java (original)
+++ hadoop/core/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java Wed Jan 23 21:13:22 2008
@@ -413,15 +413,13 @@
* @return MapFile.Reader
* @throws IOException
*/
- public synchronized MapFile.Reader getReader(final FileSystem fs,
- final Filter bloomFilter) throws IOException {
-
- if (isReference()) {
- return new HStoreFile.HalfMapFileReader(fs,
- getMapFilePath(reference).toString(), conf,
- reference.getFileRegion(), reference.getMidkey(), bloomFilter);
- }
- return new BloomFilterMapFile.Reader(fs, getMapFilePath().toString(),
+ public MapFile.Reader getReader(final FileSystem fs,
+ final Filter bloomFilter)
+ throws IOException {
+ return isReference()?
+ new HStoreFile.HalfMapFileReader(fs, getMapFilePath(reference).toString(),
+ conf, reference.getFileRegion(), reference.getMidkey(), bloomFilter):
+ new BloomFilterMapFile.Reader(fs, getMapFilePath().toString(),
conf, bloomFilter);
}