You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/08/11 23:58:12 UTC
svn commit: r684952 - in /hadoop/hbase/branches/0.2: CHANGES.txt
src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
src/test/org/apache/hadoop/hbase/client/TestHTable.java
Author: stack
Date: Mon Aug 11 14:58:12 2008
New Revision: 684952
URL: http://svn.apache.org/viewvc?rev=684952&view=rev
Log:
HBASE-729 client region/metadata cache should have a public method for invalidating entries
Modified:
hadoop/hbase/branches/0.2/CHANGES.txt
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java
Modified: hadoop/hbase/branches/0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/CHANGES.txt?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.2/CHANGES.txt Mon Aug 11 14:58:12 2008
@@ -1,16 +1,19 @@
Hbase Change Log
-Release 0.3.0 - Unreleased
+Release 0.2.1 - Unreleased
INCOMPATIBLE CHANGES
BUG FIXES
HBASE-805 Remove unnecessary getRow overloads in HRS (Jonathan Gray via
Jim Kellerman) (Fix whitespace changes in HRegionServer)
HBASE-811 HTD is not fully copyable (Andrew Purtell via Jim Kellerman)
+ HBASE-729 Client region/metadata cache should have a public method for
+ invalidating entries (Andrew Purtell via Stack)
IMPROVEMENTS
HBASE-801 When a table haven't disable, shell could response in a "user
friendly" way.
+ HBASE-816 TableMap should survive USE (Andrew Purtell via Stack)
NEW FEATURES
OPTIMIZATIONS
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Mon Aug 11 14:58:12 2008
@@ -371,61 +371,42 @@
return rowsScanned > 0 && result;
}
+ private class HTableDescriptorFinder
+ implements MetaScanner.MetaScannerVisitor {
+ byte[] tableName;
+ HTableDescriptor result;
+ public HTableDescriptorFinder(byte[] tableName) {
+ this.tableName = tableName;
+ }
+ public boolean processRow(RowResult rowResult) throws IOException {
+ HRegionInfo info = Writables.getHRegionInfo(
+ rowResult.get(HConstants.COL_REGIONINFO));
+ HTableDescriptor desc = info.getTableDesc();
+ if (Bytes.compareTo(desc.getName(), tableName) == 0) {
+ result = desc;
+ return false;
+ }
+ return true;
+ }
+ HTableDescriptor getResult() {
+ return result;
+ }
+ }
+
/** {@inheritDoc} */
- public HTableDescriptor getHTableDescriptor(byte[] tableName)
+ public HTableDescriptor getHTableDescriptor(final byte[] tableName)
throws IOException {
if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
return new UnmodifyableHTableDescriptor(HTableDescriptor.ROOT_TABLEDESC);
}
- if (!tableExists(tableName)) {
- throw new TableNotFoundException(Bytes.toString(tableName));
+ if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
+ return new UnmodifyableHTableDescriptor(HTableDescriptor.META_TABLEDESC);
}
- byte[] startKey =
- HRegionInfo.createRegionName(tableName, null, HConstants.ZEROES);
-
- HTableDescriptor result = null;
- HRegionInfo currentRegion = null;
- ScannerCallable s = null;
- while (result == null) {
- if (currentRegion != null) {
- byte[] endKey = currentRegion.getEndKey();
- if (endKey == null ||
- Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)) {
- // We have reached the end of the table and we're done
- break;
- }
- }
- HRegionInfo oldRegion = currentRegion;
- if (oldRegion != null) {
- startKey = oldRegion.getEndKey();
- }
- s = new ScannerCallable(this,
- (Bytes.equals(tableName, HConstants.META_TABLE_NAME) ?
- HConstants.ROOT_TABLE_NAME : HConstants.META_TABLE_NAME),
- HConstants.COL_REGIONINFO_ARRAY, startKey,
- HConstants.LATEST_TIMESTAMP, null
- );
- // Open scanner
- getRegionServerWithRetries(s);
- currentRegion = s.getHRegionInfo();
- try {
- RowResult r = null;
- while ((r = getRegionServerWithRetries(s)) != null) {
- Cell c = r.get(HConstants.COL_REGIONINFO);
- if (c != null) {
- HRegionInfo info = Writables.getHRegionInfoOrNull(c.getValue());
- if (info != null) {
- if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
- result = new UnmodifyableHTableDescriptor(info.getTableDesc());
- break;
- }
- }
- }
- }
- } finally {
- s.setClose();
- getRegionServerWithRetries(s);
- }
+ HTableDescriptorFinder finder = new HTableDescriptorFinder(tableName);
+ MetaScanner.metaScan(conf, finder);
+ HTableDescriptor result = finder.getResult();
+ if (result == null) {
+ throw new TableNotFoundException(Bytes.toString(tableName));
}
return result;
}
Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java Mon Aug 11 14:58:12 2008
@@ -26,6 +26,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Scanner;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
@@ -41,6 +42,7 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.StringUtils;
/**
* A Base for {@link TableInputFormat}s. Receives a {@link HTable}, a
@@ -85,17 +87,18 @@
implements RecordReader<ImmutableBytesWritable, RowResult> {
private byte [] startRow;
private byte [] endRow;
+ private byte [] lastRow;
private RowFilterInterface trrRowFilter;
private Scanner scanner;
private HTable htable;
private byte [][] trrInputColumns;
/**
- * Build the scanner. Not done in constructor to allow for extension.
+ * Restart from survivable exceptions by creating a new scanner.
*
* @throws IOException
*/
- public void init() throws IOException {
+ public void restart(byte[] firstRow) throws IOException {
if ((endRow != null) && (endRow.length > 0)) {
if (trrRowFilter != null) {
final Set<RowFilterInterface> rowFiltersSet =
@@ -107,15 +110,24 @@
rowFiltersSet));
} else {
this.scanner =
- this.htable.getScanner(trrInputColumns, startRow, endRow);
+ this.htable.getScanner(trrInputColumns, firstRow, endRow);
}
} else {
this.scanner =
- this.htable.getScanner(trrInputColumns, startRow, trrRowFilter);
+ this.htable.getScanner(trrInputColumns, firstRow, trrRowFilter);
}
}
/**
+ * Build the scanner. Not done in constructor to allow for extension.
+ *
+ * @throws IOException
+ */
+ public void init() throws IOException {
+ restart(startRow);
+ }
+
+ /**
* @param htable the {@link HTable} to scan.
*/
public void setHTable(HTable htable) {
@@ -190,19 +202,25 @@
/**
* @param key HStoreKey as input key.
* @param value MapWritable as input value
- *
- * Converts Scanner.next() to Text, RowResult
- *
* @return true if there was more data
* @throws IOException
*/
@SuppressWarnings("unchecked")
public boolean next(ImmutableBytesWritable key, RowResult value)
throws IOException {
- RowResult result = this.scanner.next();
+ RowResult result;
+ try {
+ result = this.scanner.next();
+ } catch (UnknownScannerException e) {
+ LOG.debug("recovered from " + StringUtils.stringifyException(e));
+ restart(lastRow);
+ this.scanner.next(); // skip presumed already mapped row
+ result = this.scanner.next();
+ }
boolean hasMore = result != null && result.size() > 0;
if (hasMore) {
key.set(result.getRow());
+ lastRow = key.get();
Writables.copyWritable(result, value);
}
return hasMore;
Modified: hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java?rev=684952&r1=684951&r2=684952&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java (original)
+++ hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/client/TestHTable.java Mon Aug 11 14:58:12 2008
@@ -150,36 +150,25 @@
// enable the table
admin.enableTable(tableAname);
- // Use a metascanner to avoid client API caching (HConnection has a
- // metadata cache)
- MetaScanner.MetaScannerVisitor visitor =
- new MetaScanner.MetaScannerVisitor() {
- public boolean processRow(RowResult rowResult) throws IOException {
- HRegionInfo info = Writables.getHRegionInfo(
- rowResult.get(HConstants.COL_REGIONINFO));
+ // test that attribute changes were applied
+ desc = a.getTableDescriptor();
+ if (Bytes.compareTo(desc.getName(), tableAname) != 0)
+ fail("wrong table descriptor returned");
+ // check HTD attribute
+ value = desc.getValue(attrName);
+ if (value == null)
+ fail("missing HTD attribute value");
+ if (Bytes.compareTo(value, attrValue) != 0)
+ fail("HTD attribute value is incorrect");
+ // check HCD attribute
+ for (HColumnDescriptor c: desc.getFamilies()) {
+ value = c.getValue(attrName);
+ if (value == null)
+ fail("missing HCD attribute value");
+ if (Bytes.compareTo(value, attrValue) != 0)
+ fail("HCD attribute value is incorrect");
+ }
- LOG.info("visiting " + info.toString());
- HTableDescriptor desc = info.getTableDesc();
- if (Bytes.compareTo(desc.getName(), tableAname) == 0) {
- // check HTD attribute
- byte[] value = desc.getValue(attrName);
- if (value == null)
- fail("missing HTD attribute value");
- if (Bytes.compareTo(value, attrValue) != 0)
- fail("HTD attribute value is incorrect");
- // check HCD attribute
- for (HColumnDescriptor c: desc.getFamilies()) {
- value = c.getValue(attrName);
- if (value == null)
- fail("missing HCD attribute value");
- if (Bytes.compareTo(value, attrValue) != 0)
- fail("HCD attribute value is incorrect");
- }
- }
- return true;
- }
- };
- MetaScanner.metaScan(conf, visitor);
} catch (Exception e) {
e.printStackTrace();
fail();