You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/05/20 18:54:29 UTC
svn commit: r658337 - in /hadoop/hbase/trunk: CHANGES.txt
src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
src/java/org/apache/hadoop/hbase/client/HTable.java
src/java/org/apache/hadoop/hbase/client/MetaScanner.java
Author: stack
Date: Tue May 20 09:54:29 2008
New Revision: 658337
URL: http://svn.apache.org/viewvc?rev=658337&view=rev
Log:
HBASE-626 Use Visitor pattern in MetaRegion to reduce code clones in HTable and HConnectionManager
Added:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=658337&r1=658336&r2=658337&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue May 20 09:54:29 2008
@@ -36,6 +36,8 @@
HBASE-522 Where new Text(string) might be used in client side method calls,
add an overload that takes string (Done as part of HBASE-82)
HBASE-570 Remove HQL unit test (Done as part of HBASE-82 commit).
+ HBASE-626 Use Visitor pattern in MetaRegion to reduce code clones in HTable
+ and HConnectionManager (Jean-Daniel Cryans via Stack)
Release 0.1.2 - 05/13/2008
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=658337&r1=658336&r2=658337&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Tue May 20 09:54:29 2008
@@ -42,6 +42,7 @@
import org.apache.hadoop.hbase.NoServerForRegionException;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
@@ -157,16 +158,19 @@
this.masterChecked = false;
this.servers = new ConcurrentHashMap<String, HRegionInterface>();
}
-
+
/** {@inheritDoc} */
public HMasterInterface getMaster() throws MasterNotRunningException {
HServerAddress masterLocation = null;
synchronized (this.masterLock) {
- for (int tries = 0; !this.closed &&
- !this.masterChecked && this.master == null && tries < numRetries;
- tries++) {
- String m = this.conf.get(MASTER_ADDRESS, DEFAULT_MASTER_ADDRESS);
- masterLocation = new HServerAddress(m);
+ for (int tries = 0;
+ !this.closed &&
+ !this.masterChecked && this.master == null &&
+ tries < numRetries;
+ tries++) {
+
+ masterLocation = new HServerAddress(this.conf.get(MASTER_ADDRESS,
+ DEFAULT_MASTER_ADDRESS));
try {
HMasterInterface tryMaster = (HMasterInterface)HbaseRPC.getProxy(
HMasterInterface.class, HMasterInterface.versionID,
@@ -178,7 +182,7 @@
}
} catch (IOException e) {
- if (tries == numRetries - 1) {
+ if(tries == numRetries - 1) {
// This was our last chance - don't bother sleeping
break;
}
@@ -197,8 +201,11 @@
this.masterChecked = true;
}
if (this.master == null) {
- throw new MasterNotRunningException(masterLocation == null? "":
- masterLocation.toString());
+ if (masterLocation == null) {
+ throw new MasterNotRunningException();
+ } else {
+ throw new MasterNotRunningException(masterLocation.toString());
+ }
}
return this.master;
}
@@ -257,40 +264,23 @@
/** {@inheritDoc} */
public HTableDescriptor[] listTables() throws IOException {
- HashSet<HTableDescriptor> uniqueTables = new HashSet<HTableDescriptor>();
- byte [] startRow = EMPTY_START_ROW;
+ final HashSet<HTableDescriptor> uniqueTables = new HashSet<HTableDescriptor>();
- // scan over the each meta region
- do {
- ScannerCallable callable = new ScannerCallable(this, META_TABLE_NAME,
- COL_REGIONINFO_ARRAY, startRow, LATEST_TIMESTAMP, null);
- try {
- // open scanner
- getRegionServerWithRetries(callable);
- // iterate through the scanner, accumulating unique table names
- while (true) {
- RowResult values = getRegionServerWithRetries(callable);
- if (values == null || values.size() == 0) {
- break;
- }
+ MetaScannerVisitor visitor = new MetaScannerVisitor() {
- HRegionInfo info =
- Writables.getHRegionInfo(values.get(COL_REGIONINFO));
+ public boolean processRow(RowResult rowResult,
+ HRegionLocation metaLocation, HRegionInfo info) throws IOException {
- // Only examine the rows where the startKey is zero length
- if (info.getStartKey().length == 0) {
- uniqueTables.add(info.getTableDesc());
- }
+ // Only examine the rows where the startKey is zero length
+ if (info.getStartKey().length == 0) {
+ uniqueTables.add(info.getTableDesc());
}
- // advance the startRow to the end key of the current region
- startRow = callable.getHRegionInfo().getEndKey();
- } finally {
- // close scanner
- callable.setClose();
- getRegionServerWithRetries(callable);
+ return true;
}
- } while (Bytes.compareTo(startRow, LAST_ROW) != 0);
-
+
+ };
+ MetaScanner.metaScan(conf, visitor);
+
return uniqueTables.toArray(new HTableDescriptor[uniqueTables.size()]);
}
@@ -321,11 +311,12 @@
// This block guards against two threads trying to find the root
// region at the same time. One will go do the find while the
// second waits. The second thread will not do find.
+
if (!useCache || rootRegionLocation == null) {
return locateRootRegion();
}
return rootRegionLocation;
- }
+ }
} else if (Bytes.equals(tableName, META_TABLE_NAME)) {
synchronized (metaRegionLock) {
// This block guards against two threads trying to load the meta
@@ -652,29 +643,30 @@
*/
private HRegionLocation locateRootRegion()
throws IOException {
+
getMaster();
+
HServerAddress rootRegionAddress = null;
+
for (int tries = 0; tries < numRetries; tries++) {
int localTimeouts = 0;
- // Ask the master which server has the root region
+
+ // ask the master which server has the root region
while (rootRegionAddress == null && localTimeouts < numRetries) {
rootRegionAddress = master.findRootRegion();
if (rootRegionAddress == null) {
- // Increment and then only sleep if retries left.
- if (++localTimeouts < numRetries) {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sleeping " + pause + "ms. Waiting for root "
- + "region. Attempt " + tries + " of " + numRetries);
- }
- Thread.sleep(pause);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Wake. Retry finding root region.");
- }
- } catch (InterruptedException iex) {
- // continue
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sleeping. Waiting for root region.");
+ }
+ Thread.sleep(pause);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Wake. Retry finding root region.");
}
+ } catch (InterruptedException iex) {
+ // continue
}
+ localTimeouts++;
}
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=658337&r1=658336&r2=658337&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java Tue May 20 09:54:29 2008
@@ -1,5 +1,5 @@
/**
- * Copyright 2007 The Apache Software Foundation
+ * Copyright 2008 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -35,15 +35,14 @@
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.filter.StopRowFilter;
import org.apache.hadoop.hbase.filter.WhileMatchRowFilter;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.RowResult;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Text;
/**
@@ -57,6 +56,7 @@
protected final long pause;
protected final int numRetries;
protected Random rand;
+ protected HBaseConfiguration configuration;
protected volatile boolean tableDoesNotExist;
@@ -96,6 +96,7 @@
public HTable(HBaseConfiguration conf, final byte [] tableName)
throws IOException {
this.connection = HConnectionManager.getConnection(conf);
+ this.configuration = conf;
this.tableName = tableName;
this.pause = conf.getLong("hbase.client.pause", 10 * 1000);
this.numRetries = conf.getInt("hbase.client.retries.number", 5);
@@ -142,145 +143,64 @@
/**
* Gets the starting row key for every region in the currently open table
+ *
* @return Array of region starting row keys
* @throws IOException
*/
@SuppressWarnings("null")
- public byte [][] getStartKeys() throws IOException {
- List<byte []> keyList = new ArrayList<byte []>();
+ public byte[][] getStartKeys() throws IOException {
+ final List<byte[]> keyList = new ArrayList<byte[]>();
- long scannerId = -1L;
- byte [] startRow =
- HRegionInfo.createRegionName(this.tableName, null, NINES);
- HRegionLocation metaLocation = null;
- HRegionInterface server;
-
- // scan over the each meta region
- do {
- try{
- // turn the start row into a location
- metaLocation =
- connection.locateRegion(META_TABLE_NAME, startRow);
-
- // connect to the server hosting the .META. region
- server =
- connection.getHRegionConnection(metaLocation.getServerAddress());
-
- // open a scanner over the meta region
- scannerId = server.openScanner(
- metaLocation.getRegionInfo().getRegionName(),
- new byte[][]{COL_REGIONINFO}, tableName, LATEST_TIMESTAMP,
- null);
-
- // iterate through the scanner, accumulating unique region names
- while (true) {
- RowResult values = server.next(scannerId);
- if (values == null || values.size() == 0) {
- break;
- }
-
- HRegionInfo info = new HRegionInfo();
- info = (HRegionInfo) Writables.getWritable(
- values.get(COL_REGIONINFO).getValue(), info);
-
- if (!Bytes.equals(info.getTableDesc().getName(), this.tableName)) {
- break;
- }
-
- if (info.isOffline() || info.isSplit()) {
- continue;
- }
+ MetaScannerVisitor visitor = new MetaScannerVisitor() {
+ public boolean processRow(RowResult rowResult,
+ HRegionLocation metaLocation, HRegionInfo info)
+ throws IOException {
+ if (!(Bytes.equals(info.getTableDesc().getName(), tableName))) {
+ return false;
+ }
+ if (!(info.isOffline() || info.isSplit())) {
keyList.add(info.getStartKey());
}
-
- // close that remote scanner
- server.close(scannerId);
-
- // advance the startRow to the end key of the current region
- startRow = metaLocation.getRegionInfo().getEndKey();
- } catch (IOException e) {
- // need retry logic?
- throw e;
+ return true;
}
- } while (Bytes.compareTo(startRow, EMPTY_START_ROW) != 0);
-
- return keyList.toArray(new byte [keyList.size()][]);
+
+ };
+ MetaScanner.metaScan(configuration, visitor, this.tableName);
+ return keyList.toArray(new byte[keyList.size()][]);
}
-
+
/**
* Get all the regions and their address for this table
+ *
* @return A map of HRegionInfo with it's server address
* @throws IOException
*/
@SuppressWarnings("null")
public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
- // TODO This code is a near exact copy of getStartKeys. To be refactored HBASE-626
- HashMap<HRegionInfo, HServerAddress> regionMap = new HashMap<HRegionInfo, HServerAddress>();
-
- long scannerId = -1L;
- byte [] startRow =
- HRegionInfo.createRegionName(this.tableName, null, NINES);
- HRegionLocation metaLocation = null;
- HRegionInterface server;
-
- // scan over the each meta region
- do {
- try{
- // turn the start row into a location
- metaLocation =
- connection.locateRegion(META_TABLE_NAME, startRow);
-
- // connect to the server hosting the .META. region
- server =
- connection.getHRegionConnection(metaLocation.getServerAddress());
-
- // open a scanner over the meta region
- scannerId = server.openScanner(
- metaLocation.getRegionInfo().getRegionName(),
- new byte [][]{COL_REGIONINFO}, tableName, LATEST_TIMESTAMP,
- null);
-
- // iterate through the scanner, accumulating regions and their regionserver
- while (true) {
- RowResult values = server.next(scannerId);
- if (values == null || values.size() == 0) {
- break;
- }
-
- HRegionInfo info = new HRegionInfo();
- info = (HRegionInfo) Writables.getWritable(
- values.get(COL_REGIONINFO).getValue(), info);
+ final HashMap<HRegionInfo, HServerAddress> regionMap =
+ new HashMap<HRegionInfo, HServerAddress>();
- if (!Bytes.equals(info.getTableDesc().getName(), this.tableName)) {
- break;
- }
-
- if (info.isOffline() || info.isSplit()) {
- continue;
- }
+ MetaScannerVisitor visitor = new MetaScannerVisitor() {
+ public boolean processRow(RowResult rowResult,
+ HRegionLocation metaLocation, HRegionInfo info)
+ throws IOException {
+ if (!(Bytes.equals(info.getTableDesc().getName(), tableName))) {
+ return false;
+ }
+ if (!(info.isOffline() || info.isSplit())) {
regionMap.put(info, metaLocation.getServerAddress());
}
-
- // close that remote scanner
- server.close(scannerId);
-
- // advance the startRow to the end key of the current region
- startRow = metaLocation.getRegionInfo().getEndKey();
-
- // turn the start row into a location
- metaLocation =
- connection.locateRegion(META_TABLE_NAME, startRow);
- } catch (IOException e) {
- // need retry logic?
- throw e;
+ return true;
}
- } while (Bytes.compareTo(startRow, EMPTY_START_ROW) != 0);
-
+
+ };
+ MetaScanner.metaScan(configuration, visitor, tableName);
return regionMap;
}
- /**
+
+ /**
* Get a single value for the specified row and column
- *
+ *
* @param row row key
* @param column column name
* @return value for specified row/column
Added: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=658337&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java (added)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/MetaScanner.java Tue May 20 09:54:29 2008
@@ -0,0 +1,98 @@
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Scanner class that contains the <code>.META.</code> table scanning logic
+ * and uses a Retryable scanner. Provided visitors will be called
+ * for each row.
+ */
+class MetaScanner implements HConstants {
+
+ /**
+ * Scans the meta table and calls a visitor on each RowResult and uses a empty
+ * start row value as table name.
+ *
+ * @param configuration
+ * @param visitor A custom visitor
+ * @throws IOException
+ */
+ public static void metaScan(HBaseConfiguration configuration,
+ MetaScannerVisitor visitor)
+ throws IOException {
+ metaScan(configuration, visitor, EMPTY_START_ROW);
+ }
+
+ /**
+ * Scans the meta table and calls a visitor on each RowResult. Uses a table
+ * name to locate meta regions.
+ *
+ * @param configuration
+ * @param visitor
+ * @param tableName
+ * @throws IOException
+ */
+ public static void metaScan(HBaseConfiguration configuration,
+ MetaScannerVisitor visitor, byte[] tableName)
+ throws IOException {
+ HConnection connection = HConnectionManager.getConnection(configuration);
+ HRegionLocation metaLocation = null;
+ boolean toContinue = true;
+ byte [] startRow = Bytes.equals(tableName, EMPTY_START_ROW)? tableName:
+ HRegionInfo.createRegionName(tableName, null, NINES);
+
+ // Scan over the each meta region
+ do {
+ ScannerCallable callable = new ScannerCallable(connection,
+ META_TABLE_NAME, COL_REGIONINFO_ARRAY, tableName, LATEST_TIMESTAMP,
+ null);
+ try {
+ // Open scanner
+ connection.getRegionServerWithRetries(callable);
+ metaLocation = connection.locateRegion(META_TABLE_NAME, startRow);
+ while (toContinue) {
+ RowResult rowResult = connection.getRegionServerWithRetries(callable);
+ if (rowResult == null || rowResult.size() == 0) {
+ break;
+ }
+ HRegionInfo info = Writables.
+ getHRegionInfo(rowResult.get(COL_REGIONINFO));
+ toContinue = visitor.processRow(rowResult, metaLocation, info);
+ }
+ // Advance the startRow to the end key of the current region
+ startRow = callable.getHRegionInfo().getEndKey();
+ } finally {
+ // Close scanner
+ callable.setClose();
+ connection.getRegionServerWithRetries(callable);
+ }
+ } while (Bytes.compareTo(startRow, LAST_ROW) != 0);
+ }
+
+ /**
+ * Visitor class called to process each row of the .META. table
+ */
+ protected interface MetaScannerVisitor {
+
+ /**
+ * Visitor method that accepts a RowResult and the meta region location.
+ * Implementations can return false to stop the region's loop if it becomes
+ * unnecessary for some reason.
+ *
+ * @param rowResult
+ * @param metaLocation
+ * @return A boolean to know if it should continue to loop in the region
+ * @throws IOException
+ */
+ public boolean processRow(RowResult rowResult,
+ HRegionLocation metaLocation, HRegionInfo info) throws IOException;
+ }
+}