You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jg...@apache.org on 2010/08/05 09:35:02 UTC
svn commit: r982489 [1/7] - in /hbase/branches/0.90_master_rewrite: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/catalog/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/executor/ s...
Author: jgray
Date: Thu Aug 5 07:35:00 2010
New Revision: 982489
URL: http://svn.apache.org/viewvc?rev=982489&view=rev
Log:
HBASE-2697 [part-v10] The big one
Added:
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterController.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerController.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/handler/
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRootHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenMetaHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRootHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableDisable.java
Removed:
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseEventHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AddColumn.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ChangeTableState.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ColumnOperation.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/DeleteColumn.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MetaRegion.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MetaScanner.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ModifyColumn.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ModifyTableMeta.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionServerMonitor.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationListener.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/RootScanner.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/TableDelete.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterCloseRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/handler/MasterOpenRegionHandler.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
Modified:
hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt
hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/HBaseExecutorService.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/RegionTransitionData.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/FileSystemManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterStatusTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedCloseRegion.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestZKBasedReopenRegion.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
Modified: hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt (original)
+++ hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt Thu Aug 5 07:35:00 2010
@@ -19,6 +19,7 @@ Branch 0.90.0 - Master Rewrite Branch
master cleanup and refactor)
HBASE-2696 [part2-v2-FinishRS_FixClient] ZooKeeper cleanup and refactor
HBASE-2695 [ZK-Master-Final-v4] Rest of Master/ZK cleanup and refactor
+ HBASE-2697 [part-v10] The big one!
NEW FEATURES
Modified: hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt (original)
+++ hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt Thu Aug 5 07:35:00 2010
@@ -1,22 +1,154 @@
List of things todo for branch, including comments from reviews not yet
implemented.
+harder stuff
+---
-Now:
+* make final decisions on root/meta timeouts. almost everyone is coordinating
+ access through CatalogTracker which should make it easy to standardize.
+ if there are operations that should just retry indefinitely, they need to
+ resubmit themselves to their executor service.
+
+* move splits to RS side, integrate new patch from stack on trunk
+ might need a new CREATED unassigned now, or new rpc, but get rid of sending
+ split notification on heartbeat?
+ how to handle splits concurrent with disable?
+
+* review master startup order
+ we should use cluster flag in zk to signal RS to
+ start rather than master availability and between master up and cluster up
+ the master can do bulk of it's initialization.
+
+* figure what to do with client table admin ops (flush, split, compact)
+ (direct to RS rpc calls are in place, need to update client)
+
+* on region open (and wherever split children notify master) should check if
+ if the table is disabled and should close the regions... maybe.
+
+* in RootEditor there is a race condition between delete and watch?
+
+* review FileSystemManager calls
+
+* there are some races with master wanting to connect for rpc
+ to regionserver and the rs starting its rpc server, need to address
+
+* figure how to handle the very rare but possible race condition where two
+ RSs will update META and the later one can squash the valid one if there was
+ a long gc pause
+
+* review synchronization in AssignmentManager
+
+* migrate TestMasterTransitions or make new?
+
+* fix or remove last couple master tests that used RSOQ
+
+* write new tests!!!
+
+
+somewhat easier stuff
+---
+
+* regionserver exit and expiration need to be finished in ServerManager
+
+* jsp pages borked
+
+* make sync calls for enable/disable (check and verify methods?)
+ this still needs some love and testing but should be much easier to control now
+
+* integrate load balancing
+ implemented but need to start a thread or chore, each time, wait for no
+ regions in transition, generate and iterate the plan, putting it in-memory
+ and then triggering the assignment. if the master crashes mid-balance,
+ it should finish up to the point of the last CLOSE RPC it sent out to an RS.
+ Regions will be sitting in CLOSED in ZK, failover master will pick it up,
+ re-executes the ClosedRegionHandler() on it
* synchronize all access to the boolean in ActiveMasterManager
+ (now this is probably just move it to extend ZKNodeTracker)
-* Remove TODO in HCM about Abortable interface
+* update client to use new admin functions straight to rs
+ possibly migrate client to use CatalogTracker?
-Think about:
-* renaming master file manager? MasterFS/MasterFileSystem
+notes from 8/4 (with what i did tonight for them, which is most of what is different in this diff)
+---
+
+* in CatalogTracker need to stabilize on one getRoot and one getMeta method to
+ use that waits and uses the default wait-for-catalogs timeout.
+
+ We should get rid of the 'refresh' boolean that I have in there and
+ should always ping the server to ensure it is serving the region before we
+ return it. If we do eventually drop root and put the meta locations into zk
+ we would no longer need this, so will not always have to pay this tax.
+
+ >> This is done. You pass default timeout in constructor. Two methods now are:
+
+ waitForRootServerConnectionDefault()
+ waitForMetaServerConnectionDefault()
+
+
+* ROOT changes
+
+ RootEditor -> RootLocationEditor, delete -> unset
+
+ Change the way we unset the root location. Set the data to null rather than
+ deleting the node. Requires changes to RootLocationEditor and RootRegionTracker.
+
+ >> Thought there was a race condition here, but there is not. In fact, we do
+ not even need to set the watch in the delete method. It is already properly
+ being handled by RootRegionTracker.
+
+
+* In AssignmentManager.processFailure() need to insert RegionState into RIT map
+
+ >> This is done. This needs tests but I think failover is all in place now.
+
+
+* On RS-side, make separate OpenRootHandler and OpenMetaHandler
+
+ >> Added four new handlers for open/close of root/meta and associated
+ executors
+
+
+* Add priorities to Opened/Closed handlers on Master
+
+ >> Added ROOT, META, USER priorities
+
+
+* In RegionTransitionData, store the actual byte[] regionName rather than
+ the encoded name
+
+ >> Done. We should also get in practice of naming variables encodedName if
+ it is that.
+
+
+* Executor services need to be using a priority queue
+
+ >> Done. I think all stuff to set pool size and add priorities is in.
+
+
+* In EventType, completely remove differentiating between Master and RS.
+ This means for a given EventType, it will map to the same handler whether it
+ is on RS or Master.
+
+ >> Done.
+
+ Also in EventType, remove fromByte() and use an ordinal() method
+
+ >> Done. Can we remove even having the (int) values for the enums now?
Later:
+* renaming master file manager? MasterFS/MasterFileSystem
+
* ServerStatus/MasterStatus
+ + We now have: Abortable as the base class (M, RS, and Client implement abort())
+ + ServerController (M and RS implement getZK/ServerName/Info/etc)
+ + RegionServerController (RS, definitely the hacky one)
+ + MasterController (get/set of shutdown, close, etc)
+
- These need new names to be more descriptive (ServerControl?)
- They should have a very clear purpose that adds value beyond passing
HMaster directly
@@ -26,7 +158,7 @@ Later:
* HBaseEventHandler/HBaseEventType/HBaseExecutorService
- - After ZK changes, renamed to EventHandler/EventType
+ X (done) After ZK changes, renamed to EventHandler/EventType
- Currently multiple types map to a single handler, we may want 1-to-1
- Need to do a full review of the semantics of these once bulk of
master rewrite is done
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/Abortable.java Thu Aug 5 07:35:00 2010
@@ -21,11 +21,11 @@ package org.apache.hadoop.hbase;
/**
* Interface to support the aborting of a given server or client.
- *
- * <p>This is used primarily for ZooKeeper usage when we could get an unexpected
+ * <p>
+ * This is used primarily for ZooKeeper usage when we could get an unexpected
* and fatal exception, requiring an abort.
- *
- * <p>Implemented by the Master, RegionServer, and TableServers (client).
+ * <p>
+ * Implemented by the Master, RegionServer, and TableServers (client).
*/
public interface Abortable {
/**
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java Thu Aug 5 07:35:00 2010
@@ -27,9 +27,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
-import java.util.NavigableMap;
import java.util.TreeMap;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
import org.apache.hadoop.io.VersionedWritable;
/**
@@ -53,7 +53,7 @@ public class ClusterStatus extends Versi
private String hbaseVersion;
private Collection<HServerInfo> liveServerInfo;
private Collection<String> deadServers;
- private NavigableMap<String, String> intransition;
+ private Map<String, RegionState> intransition;
/**
* Constructor, for Writable
@@ -186,11 +186,11 @@ public class ClusterStatus extends Versi
this.deadServers = deadServers;
}
- public Map<String, String> getRegionsInTransition() {
+ public Map<String, RegionState> getRegionsInTransition() {
return this.intransition;
}
- public void setRegionsInTransition(final NavigableMap<String, String> m) {
+ public void setRegionsInTransition(final Map<String, RegionState> m) {
this.intransition = m;
}
@@ -210,9 +210,9 @@ public class ClusterStatus extends Versi
out.writeUTF(server);
}
out.writeInt(this.intransition.size());
- for (Map.Entry<String, String> e: this.intransition.entrySet()) {
+ for (Map.Entry<String, RegionState> e: this.intransition.entrySet()) {
out.writeUTF(e.getKey());
- out.writeUTF(e.getValue());
+ e.getValue().write(out);
}
}
@@ -232,11 +232,12 @@ public class ClusterStatus extends Versi
deadServers.add(in.readUTF());
}
count = in.readInt();
- this.intransition = new TreeMap<String, String>();
+ this.intransition = new TreeMap<String, RegionState>();
for (int i = 0; i < count; i++) {
String key = in.readUTF();
- String value = in.readUTF();
- this.intransition.put(key, value);
+ RegionState regionState = new RegionState();
+ regionState.readFields(in);
+ this.intransition.put(key, regionState);
}
}
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Thu Aug 5 07:35:00 2010
@@ -316,6 +316,24 @@ public class HRegionInfo extends Version
}
/**
+ * Gets the table name from the specified region name.
+ * @param regionName
+ * @return
+ */
+ public static byte [] getTableName(byte [] regionName) {
+ int offset = -1;
+ for (int i = 0; i < regionName.length; i++) {
+ if (regionName[i] == DELIMITER) {
+ offset = i;
+ break;
+ }
+ }
+ byte [] tableName = new byte[offset];
+ System.arraycopy(regionName, 0, tableName, 0, offset);
+ return tableName;
+ }
+
+ /**
* Separate elements of a regionName.
* @param regionName
* @return Array of byte[] containing tableName, startKey and id
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HServerInfo.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/HServerInfo.java Thu Aug 5 07:35:00 2010
@@ -148,6 +148,8 @@ public class HServerInfo implements Writ
}
/**
+ * Gets the unique server instance name. Includes the hostname, port, and
+ * start code.
* @return Server name made of the concatenation of hostname, port and
* startcode formatted as <code><hostname> ',' <port> ',' <startcode></code>
*/
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java?rev=982489&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java Thu Aug 5 07:35:00 2010
@@ -0,0 +1,50 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+/**
+ * Thrown if a request is table schema modification is requested but
+ * made for an invalid family name.
+ */
+public class InvalidFamilyOperationException extends IOException {
+ private static final long serialVersionUID = 1L << 22 - 1L;
+ /** default constructor */
+ public InvalidFamilyOperationException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public InvalidFamilyOperationException(String s) {
+ super(s);
+ }
+
+ /**
+ * Constructor taking another exception.
+ * @param e Exception to grab data from.
+ */
+ public InvalidFamilyOperationException(Exception e) {
+ super(e);
+ }
+}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java Thu Aug 5 07:35:00 2010
@@ -20,25 +20,44 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.ServerConnection;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
/**
- * Set of functions that are exposed by any HBase process (implemented by the
- * master, region server, and client).
+ * Defines the set of shared functions implemented by all HBase servers (Masters
+ * and RegionServers).
*/
public interface ServerController extends Abortable {
/**
- * Return the address of the current server.
+ * Returns the address of the current server.
*/
public HServerAddress getHServerAddress();
/**
- * Get the configuration object for this server.
+ * Gets the configuration object for this server.
*/
public Configuration getConfiguration();
/**
- * Get the ZooKeeper instance for this server.
+ * Gets the ZooKeeper instance for this server.
*/
public ZooKeeperWatcher getZooKeeper();
+
+ /**
+ * Gets the unique server name for this server.
+ * @return unique server name
+ */
+ public String getServerName();
+
+ /**
+ * Return the server RPC connection object
+ */
+ public ServerConnection getServerConnection();
+
+ /**
+ * Returns the timeout to use for operations such as waiting on root and meta
+ * availability.
+ * @return default timeout to wait for root and meta
+ */
+ public long getTimeout();
}
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java?rev=982489&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java Thu Aug 5 07:35:00 2010
@@ -0,0 +1,333 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.ServerConnection;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.master.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.zookeeper.MetaNodeTracker;
+import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * Tracks the availability of the catalog tables <code>-ROOT-</code> and
+ * <code>.META.</code>.
+ * <p>
+ * This class is "read-only" in that the locations of the catalog tables cannot
+ * be explicitly set. Instead, ZooKeeper is used to learn of the availability
+ * and location of ROOT. ROOT is used to learn of the location of META. If not
+ * available in ROOT, ZooKeeper is used to monitor for a new location of META.
+ */
+public class CatalogTracker {
+ private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
+
+ private final ServerConnection connection;
+
+ private final ZooKeeperWatcher zookeeper;
+
+ private final RootRegionTracker rootRegionTracker;
+
+ private final MetaNodeTracker metaNodeTracker;
+
+ private final AtomicBoolean metaAvailable = new AtomicBoolean(false);
+ private HServerAddress metaLocation;
+
+ private final int defaultTimeout;
+
+ public static final byte [] ROOT_REGION =
+ HRegionInfo.ROOT_REGIONINFO.getRegionName();
+
+ public static final byte [] META_REGION =
+ HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
+
+ /**
+ * Constructs the catalog tracker. Find current state of catalog tables and
+ * begin active tracking by executing {@link #start()}.
+ * @param zookeeper zk reference
+ * @param connection server connection
+ * @param abortable if fatal exception
+ */
+ public CatalogTracker(ZooKeeperWatcher zookeeper, ServerConnection connection,
+ Abortable abortable, int defaultTimeout) {
+ this.zookeeper = zookeeper;
+ this.connection = connection;
+ this.rootRegionTracker = new RootRegionTracker(zookeeper, abortable);
+ this.metaNodeTracker = new MetaNodeTracker(zookeeper, this);
+ this.defaultTimeout = defaultTimeout;
+ }
+
+ /**
+ * Starts the catalog tracker.
+ * <p>
+ * Determines current availability of catalog tables and ensures all further
+ * transitions of either region is tracked.
+ * @throws IOException
+ */
+ public void start() throws IOException {
+ // Register listeners with zk
+ zookeeper.registerListener(rootRegionTracker);
+ zookeeper.registerListener(metaNodeTracker);
+ // Start root tracking
+ rootRegionTracker.start();
+ // Determine meta assignment
+ getMetaServerConnection(true);
+ }
+
+ /**
+ * Gets the current location for <code>-ROOT-</code> or null if location is
+ * not currently available.
+ * @return location of root, null if not available
+ */
+ public HServerAddress getRootLocation() {
+ return rootRegionTracker.getRootRegionLocation();
+ }
+
+ /**
+ * Waits indefinitely for availability of <code>-ROOT-</code>. Used during
+ * cluster startup.
+ * @throws InterruptedException if interrupted while waiting
+ */
+ public void waitForRoot()
+ throws InterruptedException {
+ rootRegionTracker.waitRootRegionLocation(0);
+ }
+
+ /**
+ * Gets the current location for <code>-ROOT-</code> if available and waits
+ * for up to the specified timeout if not immediately available. Returns null
+ * if the timeout elapses before root is available.
+ * @param timeout maximum time to wait for root availability, in milliseconds
+ * @return location of root
+ * @throws InterruptedException if interrupted while waiting
+ * @throws NotAllMetaRegionsOnlineException if root not available before
+ * timeout
+ */
+ public HServerAddress waitForRoot(long timeout)
+ throws InterruptedException, NotAllMetaRegionsOnlineException {
+ HServerAddress address = rootRegionTracker.waitRootRegionLocation(timeout);
+ if (address == null) {
+ throw new NotAllMetaRegionsOnlineException(
+ "Timed out (" + timeout + "ms)");
+ }
+ return address;
+ }
+
+ /**
+ * Gets a connection to the server hosting root, as reported by ZooKeeper,
+ * waiting up to the specified timeout for availability.
+ * @see #waitForRoot(long) for additional information
+ * @return connection to server hosting root
+ * @throws InterruptedException
+ * @throws NotAllMetaRegionsOnlineException if timed out waiting
+ * @throws IOException
+ */
+ public HRegionInterface waitForRootServerConnection(long timeout)
+ throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
+ return getCachedConnection(waitForRoot(timeout));
+ }
+
+ /**
+ * Gets a connection to the server hosting root, as reported by ZooKeeper,
+ * waiting for the default timeout specified on instantiation.
+ * @see #waitForRoot(long) for additional information
+ * @return connection to server hosting root
+ * @throws NotAllMetaRegionsOnlineException if timed out waiting
+ * @throws IOException
+ */
+ public HRegionInterface waitForRootServerConnectionDefault()
+ throws NotAllMetaRegionsOnlineException, IOException {
+ try {
+ return getCachedConnection(waitForRoot(defaultTimeout));
+ } catch (InterruptedException e) {
+ throw new NotAllMetaRegionsOnlineException("Interrupted");
+ }
+ }
+
+ /**
+ * Gets a connection to the server hosting root, as reported by ZooKeeper,
+ * if available. Returns null if no location is immediately available.
+ * @return connection to server hosting root, null if not available
+ * @throws IOException
+ */
+ private HRegionInterface getRootServerConnection()
+ throws IOException {
+ HServerAddress address = rootRegionTracker.getRootRegionLocation();
+ if (address == null) {
+ return null;
+ }
+ return getCachedConnection(address);
+ }
+
+ /**
+ * Gets a connection to the server currently hosting <code>.META.</code> or
+ * null if location is not currently available.
+ * <p>
+ * If a location is known, a connection to the cached location is returned.
+ * If refresh is true, the cached connection is verified first before
+ * returning. If the connection is not valid, it is reset and rechecked.
+ * <p>
+ * If no location for meta is currently known, method checks ROOT for a new
+ * location, verifies META is currently there, and returns a cached connection
+ * to the server hosting META.
+ *
+ * @return connection to server hosting meta, null if location not available
+ * @throws IOException
+ */
+ private HRegionInterface getMetaServerConnection(boolean refresh)
+ throws IOException {
+ synchronized(metaAvailable) {
+ if(metaAvailable.get()) {
+ HRegionInterface current = getCachedConnection(metaLocation);
+ if(!refresh) {
+ return current;
+ }
+ if(verifyRegionLocation(current, META_REGION)) {
+ return current;
+ }
+ resetMetaLocation();
+ }
+ HRegionInterface rootConnection = getRootServerConnection();
+ if(rootConnection == null) {
+ return null;
+ }
+ HServerAddress newLocation = MetaReader.readMetaLocation(rootConnection);
+ if(newLocation == null) {
+ return null;
+ }
+ HRegionInterface newConnection = getCachedConnection(newLocation);
+ if(verifyRegionLocation(newConnection, META_REGION)) {
+ setMetaLocation(newLocation);
+ return newConnection;
+ }
+ return null;
+ }
+ }
+
+ /**
+ * Waits indefinitely for availability of <code>.META.</code>. Used during
+ * cluster startup.
+ * @throws InterruptedException if interrupted while waiting
+ */
+ public void waitForMeta() throws InterruptedException {
+ synchronized(metaAvailable) {
+ while(!metaAvailable.get()) {
+ metaAvailable.wait();
+ }
+ }
+ }
+
+ /**
+ * Gets the current location for <code>.META.</code> if available and waits
+ * for up to the specified timeout if not immediately available. Throws an
+ * exception if timed out waiting.
+ * @param timeout maximum time to wait for meta availability, in milliseconds
+ * @return location of meta
+ * @throws InterruptedException if interrupted while waiting
+ * @throws IOException unexpected exception connecting to meta server
+ * @throws NotAllMetaRegionsOnlineException if meta not available before
+ * timeout
+ */
+ public HServerAddress waitForMeta(long timeout)
+ throws InterruptedException, IOException, NotAllMetaRegionsOnlineException {
+ long stop = System.currentTimeMillis() + timeout;
+ synchronized(metaAvailable) {
+ if(getMetaServerConnection(true) != null) {
+ return metaLocation;
+ }
+ while(!metaAvailable.get() &&
+ (timeout == 0 || System.currentTimeMillis() < stop)) {
+ metaAvailable.wait(timeout);
+ }
+ if(getMetaServerConnection(true) == null) {
+ throw new NotAllMetaRegionsOnlineException(
+ "Timed out (" + timeout + "ms");
+ }
+ return metaLocation;
+ }
+ }
+
+ /**
+ * Gets a connection to the server hosting meta, as reported by ZooKeeper,
+ * waiting up to the specified timeout for availability.
+ * @see #waitForMeta(long) for additional information
+ * @return connection to server hosting meta
+ * @throws InterruptedException
+ * @throws NotAllMetaRegionsOnlineException if timed out waiting
+ * @throws IOException
+ */
+ public HRegionInterface waitForMetaServerConnection(long timeout)
+ throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
+ return getCachedConnection(waitForMeta(timeout));
+ }
+
+ /**
+ * Gets a connection to the server hosting meta, as reported by ZooKeeper,
+ * waiting up to the specified timeout for availability.
+ * @see #waitForMeta(long) for additional information
+ * @return connection to server hosting meta
+ * @throws NotAllMetaRegionsOnlineException if timed out or interrupted
+ * @throws IOException
+ */
+ public HRegionInterface waitForMetaServerConnectionDefault()
+ throws NotAllMetaRegionsOnlineException, IOException {
+ try {
+ return getCachedConnection(waitForMeta(defaultTimeout));
+ } catch (InterruptedException e) {
+ throw new NotAllMetaRegionsOnlineException("Interrupted");
+ }
+ }
+
+ private void resetMetaLocation() {
+ LOG.info("Current cached META location is not valid, resetting");
+ metaAvailable.set(false);
+ metaLocation = null;
+ }
+
+ private void setMetaLocation(HServerAddress metaLocation) {
+ LOG.info("Found new META location, " + metaLocation);
+ metaAvailable.set(true);
+ this.metaLocation = metaLocation;
+ // no synchronization because these are private and already under lock
+ metaAvailable.notifyAll();
+ }
+
+ private HRegionInterface getCachedConnection(HServerAddress address)
+ throws IOException {
+ return connection.getHRegionConnection(address, false);
+ }
+
+ private boolean verifyRegionLocation(HRegionInterface metaServer,
+ byte [] regionName) {
+ try {
+ return metaServer.getRegionInfo(regionName) != null;
+ } catch (NotServingRegionException e) {
+ return false;
+ }
+ }
+}
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=982489&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java Thu Aug 5 07:35:00 2010
@@ -0,0 +1,158 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerInfo;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Writes region and assignment information to <code>.META.</code>.
+ * <p>
+ * Uses the {@link CatalogTracker} to obtain locations and connections to
+ * catalogs.
+ */
+public class MetaEditor {
+ private static final Log LOG = LogFactory.getLog(MetaEditor.class);
+
+ /**
+ * Adds a META row for the specified new region.
+ * @param info region information
+ * @throws IOException if problem connecting or updating meta
+ */
+ public static void addRegionToMeta(CatalogTracker catalogTracker,
+ HRegionInfo regionInfo)
+ throws IOException {
+ Put put = new Put(regionInfo.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(regionInfo));
+ catalogTracker.waitForMetaServerConnectionDefault().put(
+ CatalogTracker.META_REGION, put);
+ LOG.info("Added region " + regionInfo + " to META");
+ }
+
+ /**
+ * Updates the location of the specified META region in ROOT to be the
+ * specified server hostname and startcode.
+ * <p>
+ * Uses passed catalog tracker to get a connection to the server hosting
+ * ROOT and makes edits to that region.
+ *
+ * @param catalogTracker catalog tracker
+ * @param regionInfo region to update location of
+ * @param serverInfo server the region is located on
+ * @throws IOException
+ */
+ public static void updateMetaLocation(CatalogTracker catalogTracker,
+ HRegionInfo regionInfo, HServerInfo serverInfo)
+ throws IOException {
+ updateLocation(catalogTracker.waitForRootServerConnectionDefault(),
+ CatalogTracker.ROOT_REGION, regionInfo, serverInfo);
+ }
+
+ /**
+ * Updates the location of the specified region in META to be the specified
+ * server hostname and startcode.
+ * <p>
+ * Uses passed catalog tracker to get a connection to the server hosting
+ * META and makes edits to that region.
+ *
+ * @param catalogTracker catalog tracker
+ * @param regionInfo region to update location of
+ * @param serverInfo server the region is located on
+ * @throws IOException
+ */
+ public static void updateRegionLocation(CatalogTracker catalogTracker,
+ HRegionInfo regionInfo, HServerInfo serverInfo)
+ throws IOException {
+ updateLocation(catalogTracker.waitForMetaServerConnectionDefault(),
+ CatalogTracker.META_REGION, regionInfo, serverInfo);
+ }
+
+ /**
+ * Updates the location of the specified region to be the specified server.
+ * <p>
+ * Connects to the specified server which should be hosting the specified
+ * catalog region name to perform the edit.
+ *
+ * @param server connection to server hosting catalog region
+ * @param catalogRegionName name of catalog region being updated
+ * @param regionInfo region to update location of
+ * @param serverInfo server the region is located on
+ * @throws IOException
+ */
+ public static void updateLocation(HRegionInterface server,
+ byte [] catalogRegionName, HRegionInfo regionInfo, HServerInfo serverInfo)
+ throws IOException {
+ Put put = new Put(regionInfo.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+ Bytes.toBytes(serverInfo.getHostnamePort()));
+ put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
+ Bytes.toBytes(serverInfo.getStartCode()));
+ server.put(catalogRegionName, put);
+
+ LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
+ " in region " + Bytes.toString(catalogRegionName) + " with " +
+ "server=" + serverInfo.getHostnamePort() + ", " +
+ "startcode=" + serverInfo.getStartCode());
+ }
+
+ /**
+ * Deletes the specified region from META.
+ * @param catalogTracker
+ * @param regionInfo region to be deleted from META
+ * @throws IOException
+ */
+ public static void deleteRegion(CatalogTracker catalogTracker,
+ HRegionInfo regionInfo)
+ throws IOException {
+ Delete delete = new Delete(regionInfo.getRegionName());
+ catalogTracker.waitForMetaServerConnectionDefault().delete(
+ CatalogTracker.META_REGION, delete);
+
+ LOG.info("Deleted region " + regionInfo + " from META");
+ }
+
+ /**
+ * Updates the region information for the specified region in META.
+ * @param catalogTracker
+ * @param regionInfo region to be updated in META
+ * @throws IOException
+ */
+ public static void updateRegionInfo(CatalogTracker catalogTracker,
+ HRegionInfo regionInfo)
+ throws IOException {
+ Put put = new Put(regionInfo.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(regionInfo));
+ catalogTracker.waitForMetaServerConnectionDefault().put(
+ CatalogTracker.META_REGION, put);
+ LOG.info("Updated region " + regionInfo + " in META");
+ }
+}
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=982489&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Thu Aug 5 07:35:00 2010
@@ -0,0 +1,261 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Reads region and assignment information from <code>.META.</code>.
+ * <p>
+ * Uses the {@link CatalogTracker} to obtain locations and connections to
+ * catalogs.
+ */
+public class MetaReader {
+
+ /**
+ * Performs a full scan of <code>.META.</code>.
+ * <p>
+ * Returns a map of every region to it's currently assigned server, according
+ * to META. If the region does not have an assignment it will have a null
+ * value in the map.
+ *
+ * @return map of regions to their currently assigned server
+ * @throws IOException
+ */
+ public static Map<HRegionInfo,HServerAddress> fullScan(
+ CatalogTracker catalogTracker)
+ throws IOException {
+ HRegionInterface metaServer =
+ catalogTracker.waitForMetaServerConnectionDefault();
+ Map<HRegionInfo,HServerAddress> allRegions =
+ new TreeMap<HRegionInfo,HServerAddress>();
+ Scan scan = new Scan();
+ scan.addFamily(HConstants.CATALOG_FAMILY);
+ long scannerid = metaServer.openScanner(
+ HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+ try {
+ Result data;
+ while((data = metaServer.next(scannerid)) != null) {
+ if (!data.isEmpty()) {
+ Pair<HRegionInfo,HServerAddress> region =
+ metaRowToRegionPair(data);
+ allRegions.put(region.getFirst(), region.getSecond());
+ }
+ }
+ } finally {
+ metaServer.close(scannerid);
+ }
+ return allRegions;
+ }
+
+ /**
+ * Reads the location of META from ROOT.
+ * @param metaServer connection to server hosting ROOT
+ * @return location of META in ROOT, null if not available
+ * @throws IOException
+ */
+ public static HServerAddress readMetaLocation(HRegionInterface metaServer)
+ throws IOException {
+ return readLocation(metaServer, CatalogTracker.ROOT_REGION,
+ CatalogTracker.META_REGION);
+ }
+
+ /**
+ * Reads the location of the specified region from META.
+ * @param catalogTracker
+ * @param regionName region to read location of
+ * @return location of region in META, null if not available
+ * @throws IOException
+ */
+ public static HServerAddress readRegionLocation(CatalogTracker catalogTracker,
+ byte [] regionName)
+ throws IOException {
+ return readLocation(catalogTracker.waitForMetaServerConnectionDefault(),
+ CatalogTracker.META_REGION, regionName);
+ }
+
+ private static HServerAddress readLocation(HRegionInterface metaServer,
+ byte [] catalogRegionName, byte [] regionName)
+ throws IOException {
+ Result r = metaServer.get(catalogRegionName, new Get(regionName).addColumn(
+ HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER));
+ if(r == null || r.isEmpty()) {
+ return null;
+ }
+ byte [] value =
+ r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+ return new HServerAddress(Bytes.toString(value));
+ }
+
+ /**
+ * Gets the region info and assignment for the specified region from META.
+ * @param catalogTracker
+ * @param regionName
+ * @return region info and assignment from META, null if not available
+ * @throws IOException
+ */
+ public static Pair<HRegionInfo, HServerAddress> getRegion(
+ CatalogTracker catalogTracker, byte [] regionName)
+ throws IOException {
+ Get get = new Get(regionName);
+ get.addFamily(HConstants.CATALOG_FAMILY);
+ Result r = catalogTracker.waitForMetaServerConnectionDefault().get(
+ CatalogTracker.META_REGION, get);
+ if(r == null || r.isEmpty()) {
+ return null;
+ }
+ return metaRowToRegionPair(r);
+ }
+
+ public static Pair<HRegionInfo, HServerAddress> metaRowToRegionPair(
+ Result data) throws IOException {
+ HRegionInfo info = Writables.getHRegionInfo(
+ data.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
+ final byte[] value = data.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.SERVER_QUALIFIER);
+ if (value != null && value.length > 0) {
+ HServerAddress server = new HServerAddress(Bytes.toString(value));
+ return new Pair<HRegionInfo,HServerAddress>(info, server);
+ } else {
+ return new Pair<HRegionInfo, HServerAddress>(info, null);
+ }
+ }
+
+ /**
+ * Checks if the specified table exists. Looks at the META table hosted on
+ * the specified server.
+ * @param metaServer server hosting meta
+ * @param tableName table to check
+ * @return true if the table exists in meta, false if not
+ * @throws IOException
+ */
+ public static boolean tableExists(CatalogTracker catalogTracker,
+ String tableName)
+ throws IOException {
+ HRegionInterface metaServer =
+ catalogTracker.waitForMetaServerConnectionDefault();
+ byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
+ Scan scan = new Scan(firstRowInTable);
+ scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ long scannerid = metaServer.openScanner(
+ HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+ try {
+ Result data = metaServer.next(scannerid);
+ if (data != null && data.size() > 0) {
+ HRegionInfo info = Writables.getHRegionInfo(
+ data.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
+ if (info.getTableDesc().getNameAsString().equals(tableName)) {
+ // A region for this table already exists. Ergo table exists.
+ return true;
+ }
+ }
+ return false;
+ } finally {
+ metaServer.close(scannerid);
+ }
+ }
+
+ /**
+ * Gets all of the regions of the specified table from META.
+ * @param catalogTracker
+ * @param tableName
+ * @return
+ * @throws IOException
+ */
+ public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
+ byte [] tableName)
+ throws IOException {
+ HRegionInterface metaServer =
+ catalogTracker.waitForMetaServerConnectionDefault();
+ List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+ String tableString = Bytes.toString(tableName);
+ byte[] firstRowInTable = Bytes.toBytes(tableString + ",,");
+ Scan scan = new Scan(firstRowInTable);
+ scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ long scannerid = metaServer.openScanner(
+ HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+ try {
+ Result data;
+ while((data = metaServer.next(scannerid)) != null) {
+ if (data != null && data.size() > 0) {
+ HRegionInfo info = Writables.getHRegionInfo(
+ data.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
+ if (info.getTableDesc().getNameAsString().equals(tableString)) {
+ regions.add(info);
+ } else {
+ break;
+ }
+ }
+ }
+ return regions;
+ } finally {
+ metaServer.close(scannerid);
+ }
+ }
+
+ public static List<Pair<HRegionInfo, HServerAddress>>
+ getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
+ throws IOException {
+ HRegionInterface metaServer =
+ catalogTracker.waitForMetaServerConnectionDefault();
+ List<Pair<HRegionInfo, HServerAddress>> regions =
+ new ArrayList<Pair<HRegionInfo, HServerAddress>>();
+ byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
+ Scan scan = new Scan(firstRowInTable);
+ scan.addFamily(HConstants.CATALOG_FAMILY);
+ long scannerid = metaServer.openScanner(
+ HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
+ try {
+ Result data;
+ while((data = metaServer.next(scannerid)) != null) {
+ if (data != null && data.size() > 0) {
+ Pair<HRegionInfo, HServerAddress> region = metaRowToRegionPair(data);
+ if (region.getFirst().getTableDesc().getNameAsString().equals(
+ tableName)) {
+ regions.add(region);
+ } else {
+ break;
+ }
+ }
+ }
+ return regions;
+ } finally {
+ metaServer.close(scannerid);
+ }
+ }
+}
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java?rev=982489&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java Thu Aug 5 07:35:00 2010
@@ -0,0 +1,72 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Makes changes to the location of <code>-ROOT-</code> in ZooKeeper.
+ */
+public class RootLocationEditor {
+ private static final Log LOG = LogFactory.getLog(RootLocationEditor.class);
+
+ /**
+ * Deletes the location of <code>-ROOT-</code> in ZooKeeper.
+ * @param zookeeper zookeeper reference
+ * @throws KeeperException unexpected zookeeper exception
+ */
+ public static void deleteRootLocation(ZooKeeperWatcher zookeeper)
+ throws KeeperException {
+ LOG.info("Unsetting ROOT region location in ZooKeeper");
+ try {
+ // Just delete the node. Don't need any watches, only we will create it.
+ ZKUtil.deleteNode(zookeeper, zookeeper.rootServerZNode);
+ } catch(KeeperException.NoNodeException nne) {
+ // Has already been deleted
+ }
+ }
+
+ /**
+ * Sets the location of <code>-ROOT-</code> in ZooKeeper to the
+ * specified server address.
+ * @param zookeeper zookeeper reference
+ * @param location server address hosting root
+ * @throws KeeperException unexpected zookeeper exception
+ */
+ public static void setRootLocation(ZooKeeperWatcher zookeeper,
+ HServerAddress location)
+ throws KeeperException {
+ LOG.info("Setting ROOT region location in ZooKeeper as " + location);
+ try {
+ ZKUtil.createAndWatch(zookeeper, zookeeper.rootServerZNode,
+ Bytes.toBytes(location.toString()));
+ } catch(KeeperException.NodeExistsException nee) {
+ LOG.debug("ROOT region location already existed, updated location");
+ ZKUtil.setData(zookeeper, zookeeper.rootServerZNode,
+ Bytes.toBytes(location.toString()));
+ }
+ }
+}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Thu Aug 5 07:35:00 2010
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@@ -642,12 +643,15 @@ public class HBaseAdmin {
* Asynchronous operation.
*
* @param regionname region name to close
- * @param args Optional server name. Otherwise, we'll send close to the
+ * @param Optional server name. Otherwise, we'll send close to the
* server registered in .META.
* @throws IOException if a remote or network exception occurs
*/
public void closeRegion(final String regionname, final Object... args)
throws IOException {
+ // TODO: reimplement this. i don't think we will be able to send a
+ // server name anymore as client does not have this, would have to
+ // use META or be passed an HServerAddress
closeRegion(Bytes.toBytes(regionname), args);
}
@@ -693,7 +697,40 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void flush(final byte [] tableNameOrRegionName) throws IOException {
- modifyTable(tableNameOrRegionName, HConstants.Modify.TABLE_FLUSH);
+ // TODO: implement two new connection methods and change split/compact
+ boolean isTable = checkTableOrRegion(tableNameOrRegionName);
+ if(!isTable) {
+ HRegionLocation hrl = connection.locateRegion(tableNameOrRegionName);
+ HRegionInterface hri =
+ connection.getHRegionConnection(hrl.getServerAddress());
+ hri.flushRegion(hrl.getRegionInfo());
+ return;
+ }
+ List<HRegionLocation> regionLocations =
+ connection.locateRegions(tableNameOrRegionName);
+ for(HRegionLocation hrl : regionLocations) {
+ HRegionInterface hri =
+ connection.getHRegionConnection(hrl.getServerAddress());
+ hri.flushRegion(hrl.getRegionInfo());
+ }
+ }
+
+ /**
+ * Checks if the specified table or region name is a table or region. Returns
+ * true if it is a table name and false if not (so likely a region name).
+ * @param tableNameOrRegionName
+ * @return true if table, false if not
+ * @throws IllegalArgumentException
+ * @throws ZooKeeperConnectionException
+ * @throws MasterNotRunningException
+ */
+ private boolean checkTableOrRegion(final byte [] tableNameOrRegionName)
+ throws IllegalArgumentException, MasterNotRunningException,
+ ZooKeeperConnectionException {
+ if (tableNameOrRegionName == null) {
+ throw new IllegalArgumentException("Pass a table name or region name");
+ }
+ return tableExists(tableNameOrRegionName);
}
/**
@@ -825,7 +862,7 @@ public class HBaseAdmin {
}
arr = new Writable[1];
arr[0] = (HTableDescriptor)args[0];
- getMaster().modifyTable(tableName, op, arr);
+// getMaster().modifyTable(tableName, op, arr);
break;
case TABLE_COMPACT:
@@ -845,7 +882,7 @@ public class HBaseAdmin {
"ImmutableBytesWritable");
}
}
- getMaster().modifyTable(tableName, op, arr);
+// getMaster().modifyTable(tableName, op, arr);
break;
case CLOSE_REGION:
@@ -867,7 +904,7 @@ public class HBaseAdmin {
"ImmutableBytesWritable, not " + args[i]);
}
}
- getMaster().modifyTable(tableName, op, arr);
+// getMaster().modifyTable(tableName, op, arr);
break;
default:
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Thu Aug 5 07:35:00 2010
@@ -136,7 +136,7 @@ public interface HConnection {
* lives in, ignoring any value that might be in the cache.
* @param tableName name of the table <i>row</i> is in
* @param row row key you're trying to find the region of
- * @return HRegionLocation that describes where to find the reigon in
+ * @return HRegionLocation that describes where to find the region in
* question
* @throws IOException if a remote or network exception occurs
*/
@@ -145,6 +145,25 @@ public interface HConnection {
throws IOException;
/**
+ * Gets the location of the region of <i>regionName</i>.
+ * @param regionName name of the region to locate
+ * @return HRegionLocation that describes where to find the region in
+ * question
+ * @throws IOException if a remote or network exception occurs
+ */
+ public HRegionLocation locateRegion(final byte [] regionName)
+ throws IOException;
+
+ /**
+ * Gets the locations of all regions in the specified table, <i>tableName</i>.
+ * @param tableName table to get regions of
+ * @return list of region locations for all regions of table
+ * @throws IOException
+ */
+ public List<HRegionLocation> locateRegions(byte[] tableName)
+ throws IOException;
+
+ /**
* Establishes a connection to the region server at the specified address.
* @param regionServer - the server to connect to
* @return proxy for HRegionServer
@@ -199,7 +218,7 @@ public interface HConnection {
* @throws IOException if a remote or network exception occurs
* @throws RuntimeException other unspecified error
*/
- public <T> T getRegionServerWithoutRetries(ServerCallable<T> callable)
+ public <T> T getRegionServerWithoutRetries(ServerCallable<T> callable)
throws IOException, RuntimeException;
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Aug 5 07:35:00 2010
@@ -40,6 +40,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
@@ -50,7 +51,6 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.ServerController;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
@@ -185,7 +185,7 @@ public class HConnectionManager {
}
/* Encapsulates finding the servers for an HBase instance */
- static class TableServers implements ServerConnection, ServerController {
+ static class TableServers implements ServerConnection, Abortable {
static final Log LOG = LogFactory.getLog(TableServers.class);
private final Class<? extends HRegionInterface> serverInterfaceClass;
private final long pause;
@@ -262,7 +262,8 @@ public class HConnectionManager {
// initialize zookeeper and master address manager
getZooKeeperWatcher();
masterAddressManager = new MasterAddressManager(zooKeeper, this);
- masterAddressManager.monitorMaster();
+ zooKeeper.registerListener(masterAddressManager);
+ masterAddressManager.start();
this.master = null;
this.masterChecked = false;
@@ -581,6 +582,20 @@ public class HConnectionManager {
return result;
}
+ @Override
+ public HRegionLocation locateRegion(final byte [] regionName)
+ throws IOException {
+ // TODO implement. use old stuff or new stuff?
+ return null;
+ }
+
+ @Override
+ public List<HRegionLocation> locateRegions(final byte [] tableName)
+ throws IOException {
+ // TODO implement. use old stuff or new stuff?
+ return null;
+ }
+
public HRegionLocation locateRegion(final byte [] tableName,
final byte [] row)
throws IOException{
@@ -972,6 +987,7 @@ public class HConnectionManager {
regionServer.getInetSocketAddress(), this.conf,
this.maxRPCAttempts, this.rpcTimeout);
} catch (RemoteException e) {
+ LOG.warn("Remove exception connecting to RS", e);
throw RemoteExceptionHandler.decodeRemoteException(e);
}
this.servers.put(regionServer.toString(), server);
@@ -1006,7 +1022,7 @@ public class HConnectionManager {
return zooKeeper;
}
- /*
+ /**
* Repeatedly try to find the root region in ZK
* @return HRegionLocation for root region if found
* @throws NoServerForRegionException - if the root region can not be
@@ -1579,13 +1595,6 @@ public class HConnectionManager {
}
}
- // ServerController implementation so that we can use ZooKeeperWatcher
- // Our abort() call does the ZK reset() as was previously done when
- // getting ZK expiration
- // TODO: Maybe this is not right. Should there be a super-interface to
- // ServerStatus/Controller that _just_ has the abort method?
- // The only method that really makes no sense here is get address
-
@Override
public void abort() {
if(zooKeeper != null) {
@@ -1593,25 +1602,5 @@ public class HConnectionManager {
zooKeeper = null;
}
}
-
- @Override
- public Configuration getConfiguration() {
- return conf;
- }
-
- @Override
- public HServerAddress getHServerAddress() {
- return null;
- }
-
- @Override
- public ZooKeeperWatcher getZooKeeper() {
- try {
- return getZooKeeperWatcher();
- } catch (IOException e) {
- LOG.error("Problem getting zk watcher", e);
- return null;
- }
- }
}
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=982489&r1=982488&r2=982489&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java Thu Aug 5 07:35:00 2010
@@ -20,6 +20,10 @@
package org.apache.hadoop.hbase.client;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -27,8 +31,6 @@ import org.apache.hadoop.hbase.TableNotF
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
-import java.io.IOException;
-
/**
* Scanner class that contains the <code>.META.</code> table scanning logic
* and uses a Retryable scanner. Provided visitors will be called
@@ -170,6 +172,32 @@ public class MetaScanner {
}
/**
+ * Lists all of the regions currently in META.
+ * @return
+ * @throws IOException
+ */
+ public static List<HRegionInfo> listAllRegions(Configuration conf)
+ throws IOException {
+ final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+ MetaScannerVisitor visitor =
+ new MetaScannerVisitor() {
+ @Override
+ public boolean processRow(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return true;
+ }
+ HRegionInfo regionInfo = Writables.getHRegionInfo(
+ result.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
+ regions.add(regionInfo);
+ return true;
+ }
+ };
+ metaScan(conf, visitor);
+ return regions;
+ }
+
+ /**
* Visitor class called to process each row of the .META. table
*/
public interface MetaScannerVisitor {
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java?rev=982489&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java Thu Aug 5 07:35:00 2010
@@ -0,0 +1,312 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.executor;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.ServerController;
+import org.apache.hadoop.hbase.executor.HBaseExecutorService.HBaseExecutorServiceType;
+
+
+/**
+ * Abstract base class for all HBase event handlers. Subclasses should
+ * implement the process() method where the actual handling of the event
+ * happens.
+ * <p>
+ * HBaseEventType is a list of ALL events (which also corresponds to messages -
+ * either internal to one component or between components). The event type
+ * names specify the component from which the event originated, and the
+ * component which is supposed to handle it.
+ * <p>
+ * Listeners can listen to all the events by implementing the interface
+ * HBaseEventHandlerListener, and by registering themselves as a listener. They
+ * will be called back before and after the process of every event.
+ */
+public abstract class EventHandler implements Runnable, Comparable<Runnable> {
+ private static final Log LOG = LogFactory.getLog(EventHandler.class);
+ // type of event this object represents
+ protected EventType eventType;
+ // server controller
+ protected ServerController server;
+ // listeners that are called before and after an event is processed
+ protected static List<EventHandlerListener> eventHandlerListeners =
+ Collections.synchronizedList(new ArrayList<EventHandlerListener>());
+ // sequence id generator for default FIFO ordering of events
+ protected static AtomicLong seqids = new AtomicLong(0);
+ // sequence id for this event
+ protected long seqid;
+
+ /**
+ * This interface provides hooks to listen to various events received by the
+ * queue. A class implementing this can listen to the updates by calling
+ * registerListener and stop receiving updates by calling unregisterListener
+ */
+ public interface EventHandlerListener {
+ /**
+ * Called before any event is processed
+ */
+ public void beforeProcess(EventHandler event);
+ /**
+ * Called after any event is processed
+ */
+ public void afterProcess(EventHandler event);
+ }
+
+ /**
+ * These are a list of HBase events that can be handled by the various
+ * HBaseExecutorService's. All the events are serialized as byte values.
+ */
+ public enum EventType {
+ // Messages originating from RS (NOTE: there is NO direct communication from
+ // RS to Master). These are a result of RS updates into ZK.
+ RS2ZK_REGION_CLOSING (1), // RS is in process of closing a region
+ RS2ZK_REGION_CLOSED (2), // RS has finished closing a region
+ RS2ZK_REGION_OPENING (3), // RS is in process of opening a region
+ RS2ZK_REGION_OPENED (4), // RS has finished opening a region
+
+ // Messages originating from Master to RS
+ M2RS_OPEN_REGION (20), // Master asking RS to open a region
+ M2RS_OPEN_ROOT (21), // Master asking RS to open root
+ M2RS_OPEN_META (22), // Master asking RS to open meta
+ M2RS_CLOSE_REGION (23), // Master asking RS to close a region
+ M2RS_CLOSE_ROOT (24), // Master asking RS to close root
+ M2RS_CLOSE_META (25), // Master asking RS to close meta
+
+ // Messages originating from Client to Master
+ C2M_DELETE_TABLE (40), // Client asking Master to delete a table
+ C2M_DISABLE_TABLE (41), // Client asking Master to disable a table
+ C2M_ENABLE_TABLE (42), // Client asking Master to enable a table
+ C2M_MODIFY_TABLE (43), // Client asking Master to modify a table
+ C2M_ADD_FAMILY (44), // Client asking Master to add family to table
+ C2M_DELETE_FAMILY (45), // Client asking Master to delete family of table
+ C2M_MODIFY_FAMILY (46), // Client asking Master to modify family of table
+
+ // Updates from master to ZK. This is done by the master and there is
+ // nothing to process by either Master or RS
+ M2ZK_REGION_OFFLINE (50), // Master adds this region as offline in ZK
+
+ // Master controlled events to be executed on the master
+ M_SERVER_SHUTDOWN (70); // Master is processing shutdown of a RS
+
+ /**
+ * Returns the executor service type (the thread pool instance) for this
+ * event type. Every type must be handled here. Multiple types map to
+ * Called by the HMaster. Returns a name of the executor service given an
+ * event type. Every event type has an entry - if the event should not be
+ * handled just add the NONE executor.
+ * @return name of the executor service
+ */
+ public HBaseExecutorServiceType getExecutorServiceType() {
+ switch(this) {
+
+ // Master executor services
+
+ case RS2ZK_REGION_CLOSED:
+ return HBaseExecutorServiceType.MASTER_CLOSE_REGION;
+
+ case RS2ZK_REGION_OPENED:
+ return HBaseExecutorServiceType.MASTER_OPEN_REGION;
+
+ case M_SERVER_SHUTDOWN:
+ return HBaseExecutorServiceType.MASTER_SERVER_OPERATIONS;
+
+ case C2M_DELETE_TABLE:
+ case C2M_DISABLE_TABLE:
+ case C2M_ENABLE_TABLE:
+ case C2M_MODIFY_TABLE:
+ return HBaseExecutorServiceType.MASTER_TABLE_OPERATIONS;
+
+ // RegionServer executor services
+
+ case M2RS_OPEN_REGION:
+ return HBaseExecutorServiceType.RS_OPEN_REGION;
+
+ case M2RS_OPEN_ROOT:
+ return HBaseExecutorServiceType.RS_OPEN_ROOT;
+
+ case M2RS_OPEN_META:
+ return HBaseExecutorServiceType.RS_OPEN_META;
+
+ case M2RS_CLOSE_REGION:
+ return HBaseExecutorServiceType.RS_CLOSE_REGION;
+
+ case M2RS_CLOSE_ROOT:
+ return HBaseExecutorServiceType.RS_CLOSE_ROOT;
+
+ case M2RS_CLOSE_META:
+ return HBaseExecutorServiceType.RS_CLOSE_META;
+
+ default:
+ throw new RuntimeException("Unhandled event type " + this.name());
+ }
+ }
+
+ /**
+ * Start the executor service that handles the passed in event type. The
+ * server that starts these event executor services wants to handle these
+ * event types.
+ */
+ public void startExecutorService(String serverName, int maxThreads) {
+ getExecutorServiceType().startExecutorService(serverName, maxThreads);
+ }
+
+ EventType(int value) {}
+
+ @Override
+ public String toString() {
+ switch(this) {
+ case RS2ZK_REGION_CLOSED: return "CLOSED";
+ case RS2ZK_REGION_CLOSING: return "CLOSING";
+ case RS2ZK_REGION_OPENED: return "OPENED";
+ case RS2ZK_REGION_OPENING: return "OPENING";
+ case M2ZK_REGION_OFFLINE: return "OFFLINE";
+ default: return this.name();
+ }
+ }
+ }
+
+ /**
+ * Default base class constructor.
+ */
+ public EventHandler(ServerController server, EventType eventType) {
+ this.server = server;
+ this.eventType = eventType;
+ seqid = seqids.incrementAndGet();
+ }
+
+ /**
+ * This is a wrapper around process, used to update listeners before and after
+ * events are processed.
+ */
+ public void run() {
+ // fire all beforeProcess listeners
+ for(EventHandlerListener listener : eventHandlerListeners) {
+ listener.beforeProcess(this);
+ }
+
+ // call the main process function
+ try {
+ process();
+ } catch(Throwable t) {
+ LOG.error("Caught throwable while processing event " + eventType, t);
+ }
+
+ // fire all afterProcess listeners
+ for(EventHandlerListener listener : eventHandlerListeners) {
+ LOG.debug("Firing " + listener.getClass().getName() +
+ ".afterProcess event listener for event " + eventType);
+ listener.afterProcess(this);
+ }
+ }
+
+ /**
+ * This method is the main processing loop to be implemented by the various
+ * subclasses.
+ */
+ public abstract void process();
+
+ /**
+ * Subscribe to updates before and after processing events
+ */
+ public static void registerListener(EventHandlerListener listener) {
+ eventHandlerListeners.add(listener);
+ }
+
+ /**
+ * Stop receiving updates before and after processing events
+ */
+ public static void unregisterListener(EventHandlerListener listener) {
+ eventHandlerListeners.remove(listener);
+ }
+
+ /**
+ * Return the name for this event type.
+ * @return
+ */
+ public HBaseExecutorServiceType getEventHandlerName() {
+ return eventType.getExecutorServiceType();
+ }
+
+ /**
+ * Return the event type
+ * @return
+ */
+ public EventType getEventType() {
+ return eventType;
+ }
+
+ /**
+ * Submits this event object to the correct executor service. This is causes
+ * this object to get executed by the correct ExecutorService.
+ */
+ public void submit() {
+ HBaseExecutorServiceType serviceType = getEventHandlerName();
+ if(serviceType == null) {
+ throw new RuntimeException("Event " + eventType + " not handled on " +
+ "this server " + server.getServerName());
+ }
+ serviceType.getExecutor(server.getServerName()).submit(this);
+ }
+
+
+ /**
+ * Get the priority level for this handler instance. This uses natural
+ * ordering so lower numbers are higher priority.
+ * <p>
+ * Lowest priority is Integer.MAX_VALUE. Highest priority is 0.
+ * <p>
+ * Subclasses should override this method to allow prioritizing handlers.
+ * <p>
+ * Handlers with the same priority are handled in FIFO order.
+ * <p>
+ * @return Integer.MAX_VALUE by default, override to set higher priorities
+ */
+ public int getPriority() {
+ return Integer.MAX_VALUE;
+ }
+
+ /**
+ * Default prioritized runnable comparator which implements a FIFO ordering.
+ * <p>
+ * Subclasses should not override this. Instead, if they want to implement
+ * priority beyond FIFO, they should override {@link #getPriority()}.
+ */
+ @Override
+ public int compareTo(Runnable o) {
+ EventHandler eh = (EventHandler)o;
+ if(getPriority() != eh.getPriority()) {
+ return (getPriority() < eh.getPriority()) ? -1 : 1;
+ }
+ return (this.seqid < eh.seqid) ? -1 : 1;
+ }
+
+ /**
+ * Executes this event object in the caller's thread. This is a synchronous
+ * way of executing the event.
+ */
+ public void execute() {
+ this.run();
+ }
+}