You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jg...@apache.org on 2010/07/15 22:32:32 UTC
svn commit: r964570 [1/2] - in /hbase/branches/0.90_master_rewrite: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/avro/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/master/ src/ma...
Author: jgray
Date: Thu Jul 15 20:32:31 2010
New Revision: 964570
URL: http://svn.apache.org/viewvc?rev=964570&view=rev
Log:
HBASE-2696 [part2-v2-FinishRS_FixClient] ZooKeeper cleanup and refactor
Added:
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
Removed:
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerStatus.java
Modified:
hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt
hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
hbase/branches/0.90_master_rewrite/src/main/resources/hbase-webapps/master/zk.jsp
hbase/branches/0.90_master_rewrite/src/main/resources/hbase-webapps/regionserver/regionserver.jsp
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
hbase/branches/0.90_master_rewrite/src/test/java/org/apache/hadoop/hbase/regionserver/TestMasterAddressManager.java
Modified: hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt (original)
+++ hbase/branches/0.90_master_rewrite/BRANCH_CHANGES.txt Thu Jul 15 20:32:31 2010
@@ -17,6 +17,7 @@ Branch 0.90.0 - Master Rewrite Branch
HBASE-2695 [MasterStartupCleanup-v4] Cleans up the master startup, adds
new ZK tool ActiveMasterManager for master-side (part of
master cleanup and refactor)
+ HBASE-2696 [part2-v2-FinishRS_FixClient] ZooKeeper cleanup and refactor
NEW FEATURES
Modified: hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt (original)
+++ hbase/branches/0.90_master_rewrite/BRANCH_TODO.txt Thu Jul 15 20:32:31 2010
@@ -6,6 +6,8 @@ Now:
* synchronize all access to the boolean in ActiveMasterManager
+* Abortable interface as superclass to all the Status/Controller stuff
+
Think about:
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java?rev=964570&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ServerController.java Thu Jul 15 20:32:31 2010
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * Set of functions that are exposed by any HBase process (implemented by the
+ * master, region server, and client).
+ */
+public interface ServerController {
+ /**
+ * Return the address of the current server.
+ */
+ public HServerAddress getHServerAddress();
+
+ /**
+ * Get the configuration object for this server.
+ */
+ public Configuration getConfiguration();
+
+ /**
+ * Get the ZooKeeper instance for this server.
+ */
+ public ZooKeeperWatcher getZooKeeper();
+
+ /**
+ * Stub method into ServerStatus to move forward with ZK cleanup.
+ */
+ public void abortServer();
+}
Added: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java?rev=964570&view=auto
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java (added)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java Thu Jul 15 20:32:31 2010
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+/**
+ * Thrown if the client can't connect to zookeeper
+ */
+public class ZooKeeperConnectionException extends IOException {
+ private static final long serialVersionUID = 1L << 23 - 1L;
+ /** default constructor */
+ public ZooKeeperConnectionException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public ZooKeeperConnectionException(String s) {
+ super(s);
+ }
+
+ /**
+ * Constructor taking another exception.
+ * @param e Exception to grab data from.
+ */
+ public ZooKeeperConnectionException(Exception e) {
+ super(e);
+ }
+}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java Thu Jul 15 20:32:31 2010
@@ -20,10 +20,7 @@ package org.apache.hadoop.hbase.avro;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Collection;
import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericArray;
@@ -33,33 +30,18 @@ import org.apache.avro.specific.Specific
import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.HTablePool;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.avro.generated.AClusterStatus;
-import org.apache.hadoop.hbase.avro.generated.AColumnValue;
-import org.apache.hadoop.hbase.avro.generated.ACompressionAlgorithm;
import org.apache.hadoop.hbase.avro.generated.ADelete;
import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor;
import org.apache.hadoop.hbase.avro.generated.AGet;
-import org.apache.hadoop.hbase.avro.generated.AIllegalArgument;
import org.apache.hadoop.hbase.avro.generated.AIOError;
+import org.apache.hadoop.hbase.avro.generated.AIllegalArgument;
import org.apache.hadoop.hbase.avro.generated.AMasterNotRunning;
import org.apache.hadoop.hbase.avro.generated.APut;
import org.apache.hadoop.hbase.avro.generated.AResult;
@@ -67,6 +49,13 @@ import org.apache.hadoop.hbase.avro.gene
import org.apache.hadoop.hbase.avro.generated.ATableDescriptor;
import org.apache.hadoop.hbase.avro.generated.ATableExists;
import org.apache.hadoop.hbase.avro.generated.HBase;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.HTablePool;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* Start an Avro server
@@ -137,8 +126,9 @@ public class AvroServer {
* Constructs an HBaseImpl object.
*
* @throws MasterNotRunningException
+ * @throws ZooKeeperConnectionException
*/
- HBaseImpl() throws MasterNotRunningException {
+ HBaseImpl() throws MasterNotRunningException, ZooKeeperConnectionException {
conf = HBaseConfiguration.create();
admin = new HBaseAdmin(conf);
htablePool = new HTablePool(conf, 10);
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Thu Jul 15 20:32:31 2010
@@ -19,6 +19,11 @@
*/
package org.apache.hadoop.hbase.client;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.NavigableMap;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +38,7 @@ import org.apache.hadoop.hbase.MasterNot
import org.apache.hadoop.hbase.RegionException;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -43,11 +49,6 @@ import org.apache.hadoop.io.BooleanWrita
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.RemoteException;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.NavigableMap;
-
/**
* Provides administrative functions for HBase
*/
@@ -58,20 +59,22 @@ public class HBaseAdmin {
private volatile Configuration conf;
private final long pause;
private final int numRetries;
- private volatile HMasterInterface master;
/**
* Constructor
*
* @param conf Configuration object
* @throws MasterNotRunningException if the master is not running
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
- public HBaseAdmin(Configuration conf) throws MasterNotRunningException {
+ public HBaseAdmin(Configuration conf)
+ throws MasterNotRunningException, ZooKeeperConnectionException {
this.connection = HConnectionManager.getConnection(conf);
this.conf = conf;
this.pause = conf.getLong("hbase.client.pause", 30 * 1000);
this.numRetries = conf.getInt("hbase.client.retries.number", 5);
- this.master = connection.getMaster();
+ // make sure we can get to the master
+ connection.getMaster();
}
/** @return HConnection used by this object. */
@@ -80,15 +83,21 @@ public class HBaseAdmin {
}
/**
+ * Get a connection to the currently set master.
* @return proxy connection to master server for this instance
* @throws MasterNotRunningException if the master is not running
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
- public HMasterInterface getMaster() throws MasterNotRunningException{
+ public HMasterInterface getMaster()
+ throws MasterNotRunningException, ZooKeeperConnectionException {
return this.connection.getMaster();
}
- /** @return - true if the master server is running */
- public boolean isMasterRunning() {
+ /** @return - true if the master server is running
+ * @throws ZooKeeperConnectionException
+ * @throws MasterNotRunningException */
+ public boolean isMasterRunning()
+ throws MasterNotRunningException, ZooKeeperConnectionException {
return this.connection.isMasterRunning();
}
@@ -96,9 +105,10 @@ public class HBaseAdmin {
* @param tableName Table to check.
* @return True if table exists already.
* @throws MasterNotRunningException if the master is not running
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
public boolean tableExists(final String tableName)
- throws MasterNotRunningException {
+ throws MasterNotRunningException, ZooKeeperConnectionException {
return tableExists(Bytes.toBytes(tableName));
}
@@ -106,12 +116,11 @@ public class HBaseAdmin {
* @param tableName Table to check.
* @return True if table exists already.
* @throws MasterNotRunningException if the master is not running
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
public boolean tableExists(final byte [] tableName)
- throws MasterNotRunningException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
+ throws MasterNotRunningException, ZooKeeperConnectionException {
+ connection.isMasterRunning();
return connection.tableExists(tableName);
}
@@ -143,8 +152,9 @@ public class HBaseAdmin {
private long getPauseTime(int tries) {
int triesCount = tries;
- if (triesCount >= HConstants.RETRY_BACKOFF.length)
+ if (triesCount >= HConstants.RETRY_BACKOFF.length) {
triesCount = HConstants.RETRY_BACKOFF.length - 1;
+ }
return this.pause * HConstants.RETRY_BACKOFF[triesCount];
}
@@ -273,12 +283,9 @@ public class HBaseAdmin {
*/
public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys)
throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
HTableDescriptor.isLegalTableName(desc.getName());
try {
- this.master.createTable(desc, splitKeys);
+ getMaster().createTable(desc, splitKeys);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
@@ -303,13 +310,11 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void deleteTable(final byte [] tableName) throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
+ isMasterRunning();
HTableDescriptor.isLegalTableName(tableName);
HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
try {
- this.master.deleteTable(tableName);
+ getMaster().deleteTable(tableName);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
@@ -397,21 +402,21 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void enableTable(final byte [] tableName) throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
+ isMasterRunning();
// Wait until all regions are enabled
boolean enabled = false;
for (int tries = 0; tries < this.numRetries; tries++) {
try {
- this.master.enableTable(tableName);
+ getMaster().enableTable(tableName);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
enabled = isTableEnabled(tableName);
- if (enabled) break;
+ if (enabled) {
+ break;
+ }
long sleep = getPauseTime(tries);
if (LOG.isDebugEnabled()) {
LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
@@ -427,9 +432,10 @@ public class HBaseAdmin {
Bytes.toString(tableName));
}
}
- if (!enabled)
+ if (!enabled) {
throw new IOException("Unable to enable table " +
Bytes.toString(tableName));
+ }
LOG.info("Enabled table " + Bytes.toString(tableName));
}
@@ -454,20 +460,20 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void disableTable(final byte [] tableName) throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
+ isMasterRunning();
// Wait until all regions are disabled
boolean disabled = false;
for (int tries = 0; tries < this.numRetries; tries++) {
try {
- this.master.disableTable(tableName);
+ getMaster().disableTable(tableName);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
disabled = isTableDisabled(tableName);
- if (disabled) break;
+ if (disabled) {
+ break;
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("Sleep. Waiting for all regions to be disabled from " +
Bytes.toString(tableName));
@@ -556,12 +562,9 @@ public class HBaseAdmin {
*/
public void addColumn(final byte [] tableName, HColumnDescriptor column)
throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
HTableDescriptor.isLegalTableName(tableName);
try {
- this.master.addColumn(tableName, column);
+ getMaster().addColumn(tableName, column);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
@@ -590,12 +593,9 @@ public class HBaseAdmin {
*/
public void deleteColumn(final byte [] tableName, final byte [] columnName)
throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
HTableDescriptor.isLegalTableName(tableName);
try {
- this.master.deleteColumn(tableName, columnName);
+ getMaster().deleteColumn(tableName, columnName);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
@@ -629,12 +629,9 @@ public class HBaseAdmin {
public void modifyColumn(final byte [] tableName, final byte [] columnName,
HColumnDescriptor descriptor)
throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
HTableDescriptor.isLegalTableName(tableName);
try {
- this.master.modifyColumn(tableName, columnName, descriptor);
+ getMaster().modifyColumn(tableName, columnName, descriptor);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
@@ -813,9 +810,6 @@ public class HBaseAdmin {
public void modifyTable(final byte [] tableName, HConstants.Modify op,
Object... args)
throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
// Let pass if its a catalog table. Used by admins.
if (tableName != null && !MetaUtils.isMetaTableName(tableName)) {
// This will throw exception
@@ -831,7 +825,7 @@ public class HBaseAdmin {
}
arr = new Writable[1];
arr[0] = (HTableDescriptor)args[0];
- this.master.modifyTable(tableName, op, arr);
+ getMaster().modifyTable(tableName, op, arr);
break;
case TABLE_COMPACT:
@@ -851,7 +845,7 @@ public class HBaseAdmin {
"ImmutableBytesWritable");
}
}
- this.master.modifyTable(tableName, op, arr);
+ getMaster().modifyTable(tableName, op, arr);
break;
case CLOSE_REGION:
@@ -873,7 +867,7 @@ public class HBaseAdmin {
"ImmutableBytesWritable, not " + args[i]);
}
}
- this.master.modifyTable(tableName, op, arr);
+ getMaster().modifyTable(tableName, op, arr);
break;
default:
@@ -889,15 +883,11 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public synchronized void shutdown() throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
+ isMasterRunning();
try {
- this.master.shutdown();
+ getMaster().shutdown();
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
- } finally {
- this.master = null;
}
}
@@ -906,10 +896,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public ClusterStatus getClusterStatus() throws IOException {
- if (this.master == null) {
- throw new MasterNotRunningException("master has been shut down");
- }
- return this.master.getClusterStatus();
+ return getMaster().getClusterStatus();
}
private HRegionLocation getFirstMetaServerForTable(final byte [] tableName)
@@ -923,9 +910,10 @@ public class HBaseAdmin {
*
* @param conf system configuration
* @throws MasterNotRunningException if a remote or network exception occurs
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
public static void checkHBaseAvailable(Configuration conf)
- throws MasterNotRunningException {
+ throws MasterNotRunningException, ZooKeeperConnectionException {
Configuration copyOfConf = HBaseConfiguration.create(conf);
copyOfConf.setInt("hbase.client.retries.number", 1);
new HBaseAdmin(copyOfConf);
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Thu Jul 15 20:32:31 2010
@@ -19,20 +19,21 @@
*/
package org.apache.hadoop.hbase.client;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
/**
* Cluster connection.
@@ -40,29 +41,33 @@ import java.util.concurrent.ExecutorServ
*/
public interface HConnection {
/**
- * Retrieve ZooKeeperWrapper used by the connection.
- * @return ZooKeeperWrapper handle being used by the connection.
+ * Retrieve ZooKeeperWatcher used by the connection.
+ * @return ZooKeeperWatcher handle being used by the connection.
* @throws IOException if a remote or network exception occurs
*/
- public ZooKeeperWrapper getZooKeeperWrapper() throws IOException;
+ public ZooKeeperWatcher getZooKeeperWatcher() throws IOException;
/**
* @return proxy connection to master server for this instance
* @throws MasterNotRunningException if the master is not running
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
- public HMasterInterface getMaster() throws MasterNotRunningException;
+ public HMasterInterface getMaster()
+ throws MasterNotRunningException, ZooKeeperConnectionException;
/** @return - true if the master server is running */
- public boolean isMasterRunning();
+ public boolean isMasterRunning()
+ throws MasterNotRunningException, ZooKeeperConnectionException;
/**
* Checks if <code>tableName</code> exists.
* @param tableName Table to check.
* @return True if table exists already.
* @throws MasterNotRunningException if the master is not running
+ * @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
public boolean tableExists(final byte [] tableName)
- throws MasterNotRunningException;
+ throws MasterNotRunningException, ZooKeeperConnectionException;
/**
* A table that isTableEnabled == false and isTableDisabled == false
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Jul 15 20:32:31 2010
@@ -50,22 +50,23 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.ServerController;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.MasterAddressManager;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.MetaUtils;
import org.apache.hadoop.hbase.util.SoftValueSortedMap;
import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
import org.apache.hadoop.ipc.RemoteException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.apache.zookeeper.KeeperException;
/**
* A non-instantiable class that manages connections to multiple tables in
@@ -107,16 +108,15 @@ public class HConnectionManager {
}
};
- private static final Map<String, ClientZKWatcher> ZK_WRAPPERS =
- new HashMap<String, ClientZKWatcher>();
-
/**
* Get the connection object for the instance specified by the configuration
* If no current connection exists, create a new connection for that instance
* @param conf configuration
* @return HConnection object for the instance specified by the configuration
+ * @throws ZooKeeperConnectionException
*/
- public static HConnection getConnection(Configuration conf) {
+ public static HConnection getConnection(Configuration conf)
+ throws ZooKeeperConnectionException {
TableServers connection;
Integer key = HBaseConfiguration.hashCode(conf);
synchronized (HBASE_INSTANCES) {
@@ -148,7 +148,7 @@ public class HConnectionManager {
/**
* Delete information for all connections.
* @param stopProxy stop the proxy as well
- * @throws IOException
+ * @throws IOException
*/
public static void deleteAllConnections(boolean stopProxy) {
synchronized (HBASE_INSTANCES) {
@@ -158,103 +158,16 @@ public class HConnectionManager {
}
}
}
- synchronized (ZK_WRAPPERS) {
- for (ClientZKWatcher watch : ZK_WRAPPERS.values()) {
- watch.resetZooKeeper();
- }
- }
- }
-
- /**
- * Get a watcher of a zookeeper connection for a given quorum address.
- * If the connection isn't established, a new one is created.
- * This acts like a multiton.
- * @param conf configuration
- * @return ZKW watcher
- * @throws IOException if a remote or network exception occurs
- */
- public static synchronized ClientZKWatcher getClientZooKeeperWatcher(
- Configuration conf) throws IOException {
- if (!ZK_WRAPPERS.containsKey(
- ZooKeeperWrapper.getZookeeperClusterKey(conf))) {
- ZK_WRAPPERS.put(ZooKeeperWrapper.getZookeeperClusterKey(conf),
- new ClientZKWatcher(conf));
- }
- return ZK_WRAPPERS.get(ZooKeeperWrapper.getZookeeperClusterKey(conf));
- }
-
- /**
- * This class is responsible to handle connection and reconnection
- * to a zookeeper quorum.
- *
- */
- public static class ClientZKWatcher implements Watcher {
-
- static final Log LOG = LogFactory.getLog(ClientZKWatcher.class);
- private ZooKeeperWrapper zooKeeperWrapper;
- private Configuration conf;
-
- /**
- * Takes a configuration to pass it to ZKW but won't instanciate it
- * @param conf configuration
- */
- public ClientZKWatcher(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * Called by ZooKeeper when an event occurs on our connection. We use this to
- * detect our session expiring. When our session expires, we have lost our
- * connection to ZooKeeper. Our handle is dead, and we need to recreate it.
- *
- * See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
- * for more information.
- *
- * @param event WatchedEvent witnessed by ZooKeeper.
- */
- public void process(WatchedEvent event) {
- KeeperState state = event.getState();
- if(!state.equals(KeeperState.SyncConnected)) {
- LOG.debug("Got ZooKeeper event, state: " + state + ", type: "
- + event.getType() + ", path: " + event.getPath());
- }
- if (state == KeeperState.Expired) {
- resetZooKeeper();
- }
- }
-
- /**
- * Get this watcher's ZKW, instantiate it if necessary.
- * @return ZKW
- * @throws java.io.IOException if a remote or network exception occurs
- */
- public synchronized ZooKeeperWrapper getZooKeeperWrapper() throws IOException {
- if(zooKeeperWrapper == null) {
- zooKeeperWrapper = new ZooKeeperWatcher(conf,
- HConnectionManager.class.getName(), null);
- zooKeeperWrapper.registerListener(this);
- }
- return zooKeeperWrapper;
- }
-
- /**
- * Clear this connection to zookeeper.
- */
- private synchronized void resetZooKeeper() {
- if (zooKeeperWrapper != null) {
- zooKeeperWrapper.close();
- zooKeeperWrapper = null;
- }
- }
}
/**
* It is provided for unit test cases which verify the behavior of region
* location cache prefetch.
* @return Number of cached regions for the table.
+ * @throws ZooKeeperConnectionException
*/
static int getCachedRegionCount(Configuration conf,
- byte[] tableName) {
+ byte[] tableName) throws ZooKeeperConnectionException {
TableServers connection = (TableServers)getConnection(conf);
return connection.getNumberOfCachedRegionLocations(tableName);
}
@@ -263,15 +176,16 @@ public class HConnectionManager {
* It's provided for unit test cases which verify the behavior of region
* location cache prefetch.
* @return true if the region where the table and row reside is cached.
+ * @throws ZooKeeperConnectionException
*/
static boolean isRegionCached(Configuration conf,
- byte[] tableName, byte[] row) {
+ byte[] tableName, byte[] row) throws ZooKeeperConnectionException {
TableServers connection = (TableServers)getConnection(conf);
return connection.isRegionCached(tableName, row);
}
/* Encapsulates finding the servers for an HBase instance */
- static class TableServers implements ServerConnection {
+ static class TableServers implements ServerConnection, ServerController {
static final Log LOG = LogFactory.getLog(TableServers.class);
private final Class<? extends HRegionInterface> serverInterfaceClass;
private final long pause;
@@ -284,6 +198,10 @@ public class HConnectionManager {
private volatile boolean closed;
private volatile HMasterInterface master;
private volatile boolean masterChecked;
+ // ZooKeeper reference
+ private ZooKeeperWatcher zooKeeper;
+ // ZooKeeper-based master address tracker
+ private MasterAddressManager masterAddressManager;
private final Object rootRegionLock = new Object();
private final Object metaRegionLock = new Object();
@@ -312,7 +230,8 @@ public class HConnectionManager {
* @param conf Configuration object
*/
@SuppressWarnings("unchecked")
- public TableServers(Configuration conf) {
+ public TableServers(Configuration conf)
+ throws ZooKeeperConnectionException {
this.conf = conf;
String serverClassName =
@@ -340,14 +259,20 @@ public class HConnectionManager {
this.prefetchRegionLimit = conf.getInt("hbase.client.prefetch.limit",
10);
+ // initialize zookeeper and master address manager
+ getZooKeeperWatcher();
+ masterAddressManager = new MasterAddressManager(zooKeeper, this);
+ masterAddressManager.monitorMaster();
+
this.master = null;
this.masterChecked = false;
}
private long getPauseTime(int tries) {
int ntries = tries;
- if (ntries >= HConstants.RETRY_BACKOFF.length)
+ if (ntries >= HConstants.RETRY_BACKOFF.length) {
ntries = HConstants.RETRY_BACKOFF.length - 1;
+ }
return this.pause * HConstants.RETRY_BACKOFF[ntries];
}
@@ -365,12 +290,22 @@ public class HConnectionManager {
this.rootRegionLocation = rootRegion;
}
- public HMasterInterface getMaster() throws MasterNotRunningException {
- ZooKeeperWrapper zk;
+ public HMasterInterface getMaster()
+ throws MasterNotRunningException, ZooKeeperConnectionException {
+
+ // Check if we already have a good master connection
+ if (master != null) {
+ if(master.isMasterRunning()) {
+ return master;
+ }
+ }
+
+ // If not, we need to connect to ZK to get the
+ ZooKeeperWatcher zk;
try {
- zk = getZooKeeperWrapper();
+ zk = getZooKeeperWatcher();
} catch (IOException e) {
- throw new MasterNotRunningException(e);
+ throw new ZooKeeperConnectionException(e);
}
HServerAddress masterLocation = null;
@@ -382,7 +317,11 @@ public class HConnectionManager {
tries++) {
try {
- masterLocation = zk.readMasterAddressOrThrow();
+ masterLocation = masterAddressManager.getMasterAddress();
+ if(masterLocation == null) {
+ LOG.info("ZooKeeper available but no active master location found");
+ throw new MasterNotRunningException();
+ }
HMasterInterface tryMaster = (HMasterInterface)HBaseRPC.getProxy(
HMasterInterface.class, HBaseRPCProtocolVersion.versionID,
@@ -424,20 +363,20 @@ public class HConnectionManager {
return this.master;
}
- public boolean isMasterRunning() {
+ public boolean isMasterRunning()
+ throws MasterNotRunningException, ZooKeeperConnectionException {
if (this.master == null) {
- try {
- getMaster();
-
- } catch (MasterNotRunningException e) {
- return false;
- }
+ getMaster();
}
- return true;
+ boolean isRunning = master.isMasterRunning();
+ if(isRunning) {
+ return true;
+ }
+ throw new MasterNotRunningException();
}
public boolean tableExists(final byte [] tableName)
- throws MasterNotRunningException {
+ throws MasterNotRunningException, ZooKeeperConnectionException {
getMaster();
if (tableName == null) {
throw new IllegalArgumentException("Table name cannot be null");
@@ -1047,10 +986,24 @@ public class HConnectionManager {
return getHRegionConnection(regionServer, false);
}
- public synchronized ZooKeeperWrapper getZooKeeperWrapper()
- throws IOException {
- return HConnectionManager.getClientZooKeeperWatcher(conf)
- .getZooKeeperWrapper();
+ /**
+ * Get the ZooKeeper instance for this TableServers instance.
+ *
+ * If ZK has not been initialized yet, this will connect to ZK.
+ * @returns zookeeper reference
+ * @throws ZooKeeperConncetionException if there's a problem connecting to zk
+ */
+ public synchronized ZooKeeperWatcher getZooKeeperWatcher()
+ throws ZooKeeperConnectionException {
+ if(zooKeeper == null) {
+ try {
+ zooKeeper = new ZooKeeperWatcher(conf,
+ ZKUtil.getZooKeeperClusterKey(conf), this);
+ } catch (IOException e) {
+ throw new ZooKeeperConnectionException(e);
+ }
+ }
+ return zooKeeper;
}
/*
@@ -1065,7 +1018,12 @@ public class HConnectionManager {
// We lazily instantiate the ZooKeeper object because we don't want to
// make the constructor have to throw IOException or handle it itself.
- ZooKeeperWrapper zk = getZooKeeperWrapper();
+ ZooKeeperWatcher zk;
+ try {
+ zk = getZooKeeperWatcher();
+ } catch (IOException e) {
+ throw new ZooKeeperConnectionException(e);
+ }
HServerAddress rootRegionAddress = null;
for (int tries = 0; tries < numRetries; tries++) {
@@ -1074,7 +1032,13 @@ public class HConnectionManager {
while (rootRegionAddress == null && localTimeouts < numRetries) {
// Don't read root region until we're out of safe mode so we know
// that the meta regions have been assigned.
- rootRegionAddress = zk.readRootRegionLocation();
+ try {
+ rootRegionAddress = ZKUtil.getDataAsAddress(zk, zk.rootServerZNode);
+ } catch (KeeperException e) {
+ LOG.error("Unexpected ZooKeeper error attempting to read the root " +
+ "region server address");
+ throw new IOException(e);
+ }
if (rootRegionAddress == null) {
try {
if (LOG.isDebugEnabled()) {
@@ -1333,8 +1297,12 @@ public class HConnectionManager {
public int processBatchOfRows(final ArrayList<Put> list,
final byte[] tableName)
throws IOException {
- if (list.isEmpty()) return 0;
- if (list.size() > 1) Collections.sort(list);
+ if (list.isEmpty()) {
+ return 0;
+ }
+ if (list.size() > 1) {
+ Collections.sort(list);
+ }
Batch b = new Batch(this) {
@SuppressWarnings("unchecked")
@Override
@@ -1356,8 +1324,12 @@ public class HConnectionManager {
public int processBatchOfDeletes(final List<Delete> list,
final byte[] tableName)
throws IOException {
- if (list.isEmpty()) return 0;
- if (list.size() > 1) Collections.sort(list);
+ if (list.isEmpty()) {
+ return 0;
+ }
+ if (list.size() > 1) {
+ Collections.sort(list);
+ }
Batch b = new Batch(this) {
@SuppressWarnings("unchecked")
@Override
@@ -1606,5 +1578,40 @@ public class HConnectionManager {
new HRegionLocation(e.getKey(), e.getValue()));
}
}
+
+ // ServerController implementation so that we can use ZooKeeperWatcher
+ // Our abort() call does the ZK reset() as was previously done when
+ // getting ZK expiration
+ // TODO: Maybe this is not right. Should there be a super-interface to
+ // ServerStatus/Controller that _just_ has the abort method?
+ // The only method that really makes no sense here is get address
+
+ @Override
+ public void abortServer() {
+ if(zooKeeper != null) {
+ zooKeeper.close();
+ zooKeeper = null;
+ }
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ @Override
+ public HServerAddress getHServerAddress() {
+ return null;
+ }
+
+ @Override
+ public ZooKeeperWatcher getZooKeeper() {
+ try {
+ return getZooKeeperWatcher();
+ } catch (IOException e) {
+ LOG.error("Problem getting zk watcher", e);
+ return null;
+ }
+ }
}
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/HTable.java Thu Jul 15 20:32:31 2010
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -626,7 +627,7 @@ public class HTable implements HTableInt
public Boolean call() throws IOException {
return server.checkAndDelete(
location.getRegionInfo().getRegionName(),
- row, family, qualifier, value, delete)
+ row, family, qualifier, value, delete)
? Boolean.TRUE : Boolean.FALSE;
}
}
@@ -1097,10 +1098,12 @@ public class HTable implements HTableInt
Thread t = new Thread(group, r,
namePrefix + threadNumber.getAndIncrement(),
0);
- if (!t.isDaemon())
- t.setDaemon(true);
- if (t.getPriority() != Thread.NORM_PRIORITY)
- t.setPriority(Thread.NORM_PRIORITY);
+ if (!t.isDaemon()) {
+ t.setDaemon(true);
+ }
+ if (t.getPriority() != Thread.NORM_PRIORITY) {
+ t.setPriority(Thread.NORM_PRIORITY);
+ }
return t;
}
}
@@ -1112,9 +1115,10 @@ public class HTable implements HTableInt
* @param tableName name of table to configure.
* @param enable Set to true to enable region cache prefetch. Or set to
* false to disable it.
+ * @throws ZooKeeperConnectionException
*/
public static void setRegionCachePrefetch(final byte[] tableName,
- boolean enable) {
+ boolean enable) throws ZooKeeperConnectionException {
HConnectionManager.getConnection(HBaseConfiguration.create()).
setRegionCachePrefetch(tableName, enable);
}
@@ -1127,9 +1131,10 @@ public class HTable implements HTableInt
* @param tableName name of table to configure.
* @param enable Set to true to enable region cache prefetch. Or set to
* false to disable it.
+ * @throws ZooKeeperConnectionException
*/
public static void setRegionCachePrefetch(final Configuration conf,
- final byte[] tableName, boolean enable) {
+ final byte[] tableName, boolean enable) throws ZooKeeperConnectionException {
HConnectionManager.getConnection(conf).setRegionCachePrefetch(
tableName, enable);
}
@@ -1140,9 +1145,10 @@ public class HTable implements HTableInt
* @param tableName name of table to check
* @return true if table's region cache prefecth is enabled. Otherwise
* it is disabled.
+ * @throws ZooKeeperConnectionException
*/
public static boolean getRegionCachePrefetch(final Configuration conf,
- final byte[] tableName) {
+ final byte[] tableName) throws ZooKeeperConnectionException {
return HConnectionManager.getConnection(conf).getRegionCachePrefetch(
tableName);
}
@@ -1152,8 +1158,9 @@ public class HTable implements HTableInt
* @param tableName name of table to check
* @return true if table's region cache prefecth is enabled. Otherwise
* it is disabled.
+ * @throws ZooKeeperConnectionException
*/
- public static boolean getRegionCachePrefetch(final byte[] tableName) {
+ public static boolean getRegionCachePrefetch(final byte[] tableName) throws ZooKeeperConnectionException {
return HConnectionManager.getConnection(HBaseConfiguration.create()).
getRegionCachePrefetch(tableName);
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java Thu Jul 15 20:32:31 2010
@@ -21,6 +21,7 @@
package org.apache.hadoop.hbase.client;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
/**
@@ -38,8 +39,9 @@ public class ServerConnectionManager ext
* If no current connection exists, create a new connection for that instance
* @param conf configuration
* @return HConnection object for the instance specified by the configuration
+ * @throws ZooKeeperConnectionException
*/
- public static ServerConnection getConnection(Configuration conf) {
+ public static ServerConnection getConnection(Configuration conf) throws ZooKeeperConnectionException {
return (ServerConnection) HConnectionManager.getConnection(conf);
}
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu Jul 15 20:32:31 2010
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.MasterNot
import org.apache.hadoop.hbase.MiniZooKeeperCluster;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.MetaScanner;
@@ -1136,6 +1137,9 @@ public class HMaster extends Thread impl
} catch (MasterNotRunningException e) {
LOG.error("Master not running");
System.exit(0);
+ } catch (ZooKeeperConnectionException e) {
+ LOG.error("ZooKeeper not available");
+ System.exit(0);
}
try {
adm.shutdown();
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java Thu Jul 15 20:32:31 2010
@@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.master;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.hbase.ServerStatus;
+import org.apache.hadoop.hbase.ServerController;
import org.apache.hadoop.hbase.client.ServerConnection;
/**
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.client.Se
* TODO: this list has to be cleaned up, this is a re-factor only change that
* preserves the functions in the interface.
*/
-public interface MasterStatus extends ServerStatus {
+public interface MasterStatus extends ServerController {
/**
* Return the server manager for region server related info
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Jul 15 20:32:31 2010
@@ -71,7 +71,7 @@ import org.apache.hadoop.hbase.Leases;
import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.ServerStatus;
+import org.apache.hadoop.hbase.ServerController;
import org.apache.hadoop.hbase.UnknownRowLockException;
import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.YouAreDeadException;
@@ -109,13 +109,14 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
+import org.apache.zookeeper.KeeperException;
/**
* HRegionServer makes a set of HRegions available to clients. It checks in with
* the HMaster. There are many HRegionServers in a single HBase deployment.
*/
public class HRegionServer implements HRegionInterface,
- HBaseRPCErrorHandler, Runnable, Stoppable, ServerStatus {
+ HBaseRPCErrorHandler, Runnable, Stoppable, ServerController {
public static final Log LOG = LogFactory.getLog(HRegionServer.class);
private static final HMsg REPORT_EXITING = new HMsg(Type.MSG_REPORT_EXITING);
private static final HMsg REPORT_QUIESCED = new HMsg(Type.MSG_REPORT_QUIESCED);
@@ -1171,11 +1172,15 @@ public class HRegionServer implements HR
if (LOG.isDebugEnabled())
LOG.debug("sending initial server load: " + hsl);
lastMsg = System.currentTimeMillis();
- zooKeeper.writeRSLocation(this.serverInfo);
+ ZKUtil.setAddressAndWatch(zooKeeper,
+ ZKUtil.joinZNode(zooKeeper.rsZNode, ZKUtil.getNodeName(serverInfo)),
+ address);
result = this.hbaseMaster.regionServerStartup(this.serverInfo);
break;
} catch (IOException e) {
LOG.warn("error telling master we are up", e);
+ } catch (KeeperException e) {
+ LOG.warn("error putting up ephemeral node in zookeeper", e);
}
sleeper.sleep(lastMsg);
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/MasterAddressManager.java Thu Jul 15 20:32:31 2010
@@ -22,7 +22,7 @@ package org.apache.hadoop.hbase.regionse
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.ServerStatus;
+import org.apache.hadoop.hbase.ServerController;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -49,7 +49,7 @@ public class MasterAddressManager extend
private HServerAddress masterAddress;
// Status and controller for the regionserver
- private ServerStatus status;
+ private ServerController status;
/**
* Construct a master address listener with the specified zookeeper reference.
@@ -60,7 +60,7 @@ public class MasterAddressManager extend
*
* @param watcher zk reference and watcher
*/
- public MasterAddressManager(ZooKeeperWatcher watcher, ServerStatus status) {
+ public MasterAddressManager(ZooKeeperWatcher watcher, ServerController status) {
super(watcher);
this.status = status;
this.masterAddress = null;
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/regionserver/RSZookeeperUpdater.java Thu Jul 15 20:32:31 2010
@@ -8,21 +8,24 @@ import org.apache.hadoop.hbase.HMsg;
import org.apache.hadoop.hbase.executor.RegionTransitionEventData;
import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType;
import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
/**
- * This is a helper class for region servers to update various states in
- * Zookeeper. The various updates are abstracted out here.
- *
- * The "startRegionXXX" methods are to be called first, followed by the
- * "finishRegionXXX" methods. Supports updating zookeeper periodically as a
+ * This is a helper class for region servers to update various states in
+ * Zookeeper. The various updates are abstracted out here.
+ *
+ * The "startRegionXXX" methods are to be called first, followed by the
+ * "finishRegionXXX" methods. Supports updating zookeeper periodically as a
* part of the "startRegionXXX". Currently handles the following state updates:
* - Close region
* - Open region
*/
// TODO: make this thread local, in which case it is re-usable per thread
+// TODO: After open/close is direct RPC, move this logic into Handlers
public class RSZookeeperUpdater {
private static final Log LOG = LogFactory.getLog(RSZookeeperUpdater.class);
private final String regionServerName;
@@ -36,59 +39,67 @@ public class RSZookeeperUpdater {
String regionName) {
this(zooKeeper, regionServerName, regionName, 0);
}
-
+
public RSZookeeperUpdater(ZooKeeperWatcher zooKeeper, String regionServerName,
String regionName, int zkVersion) {
this.zooKeeper = zooKeeper;
this.regionServerName = regionServerName;
this.regionName = regionName;
// get the region ZNode we have to create
- this.regionZNode = zooKeeper.getZNode(zooKeeper.assignmentZNode, regionName);
+ this.regionZNode = ZKUtil.joinZNode(zooKeeper.assignmentZNode, regionName);
this.zkVersion = zkVersion;
}
-
+
/**
- * This method updates the various states in ZK to inform the master that the
+ * This method updates the various states in ZK to inform the master that the
* region server has started closing the region.
* @param updatePeriodically - if true, periodically updates the state in ZK
*/
public void startRegionCloseEvent(HMsg hmsg, boolean updatePeriodically) throws IOException {
- // if this ZNode already exists, something is wrong
- if(zooKeeper.exists(regionZNode, true)) {
- String msg = "ZNode " + regionZNode + " already exists in ZooKeeper, will NOT close region.";
- LOG.error(msg);
- throw new IOException(msg);
+ // Try to create the node with a CLOSING state, if already exists,
+ // something is wrong
+ try {
+ if(ZKUtil.createPersistentNodeIfNotExists(zooKeeper, regionZNode,
+ makeZKEventData(HBaseEventType.RS2ZK_REGION_CLOSING, hmsg))) {
+ String msg = "ZNode " + regionZNode + " already exists in ZooKeeper, will NOT close region.";
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ } catch (KeeperException e) {
+ zooKeeper.error("Unexpected exception trying to create unassigned node", e);
+ throw new IOException(e);
}
-
- // create the region node in the unassigned directory first
- zooKeeper.createZNodeIfNotExists(regionZNode, null, CreateMode.PERSISTENT, true);
- // update the data for "regionName" ZNode in unassigned to CLOSING
- updateZKWithEventData(HBaseEventType.RS2ZK_REGION_CLOSING, hmsg);
-
// TODO: implement the updatePeriodically logic here
}
/**
- * This method updates the states in ZK to signal that the region has been
+ * This method updates the states in ZK to signal that the region has been
* closed. This will stop the periodic updater thread if one was started.
* @throws IOException
*/
- public void finishRegionCloseEvent(HMsg hmsg) throws IOException {
+ public void finishRegionCloseEvent(HMsg hmsg) throws IOException {
// TODO: stop the updatePeriodically here
// update the data for "regionName" ZNode in unassigned to CLOSED
updateZKWithEventData(HBaseEventType.RS2ZK_REGION_CLOSED, hmsg);
}
-
+
/**
- * This method updates the various states in ZK to inform the master that the
+ * This method updates the various states in ZK to inform the master that the
* region server has started opening the region.
* @param updatePeriodically - if true, periodically updates the state in ZK
*/
- public void startRegionOpenEvent(HMsg hmsg, boolean updatePeriodically) throws IOException {
+ public void startRegionOpenEvent(HMsg hmsg, boolean updatePeriodically)
+ throws IOException {
Stat stat = new Stat();
- byte[] data = zooKeeper.readZNode(regionZNode, stat);
+ byte[] data = null;
+ try {
+ data = ZKUtil.getDataNoWatch(zooKeeper, regionZNode, stat);
+ } catch (KeeperException e) {
+ zooKeeper.error("ZooKeeper error", e);
+ throw new IOException(e);
+ }
// if there is no ZNode for this region, something is wrong
if(data == null) {
String msg = "ZNode " + regionZNode + " does not exist in ZooKeeper, will NOT open region.";
@@ -108,12 +119,12 @@ public class RSZookeeperUpdater {
// update the data for "regionName" ZNode in unassigned to CLOSING
updateZKWithEventData(HBaseEventType.RS2ZK_REGION_OPENING, hmsg);
-
+
// TODO: implement the updatePeriodically logic here
}
-
+
/**
- * This method updates the states in ZK to signal that the region has been
+ * This method updates the states in ZK to signal that the region has been
* opened. This will stop the periodic updater thread if one was started.
* @throws IOException
*/
@@ -123,7 +134,7 @@ public class RSZookeeperUpdater {
// update the data for "regionName" ZNode in unassigned to CLOSED
updateZKWithEventData(HBaseEventType.RS2ZK_REGION_OPENED, hmsg);
}
-
+
public boolean isClosingRegion() {
return (lastUpdatedState == HBaseEventType.RS2ZK_REGION_CLOSING);
}
@@ -141,19 +152,42 @@ public class RSZookeeperUpdater {
updateZKWithEventData(HBaseEventType.RS2ZK_REGION_CLOSED, hmsg);
}
- private void updateZKWithEventData(HBaseEventType hbEventType, HMsg hmsg) throws IOException {
- // update the data for "regionName" ZNode in unassigned to "hbEventType"
- byte[] data = null;
+ /**
+ * Make the serialized data to put into unassigned znodes for the specified
+ * event type and message.
+ * @param eventType
+ * @param hmsg
+ * @return serialized data
+ */
+ private byte [] makeZKEventData(HBaseEventType eventType, HMsg hmsg)
+ throws IOException {
+ return Writables.getBytes(new RegionTransitionEventData(eventType,
+ regionServerName, hmsg));
+ }
+
+ /**
+ * Update the data for this region to the serialized form of the specified
+ * event type and message.
+ * @param hbEventType
+ * @param hmsg
+ * @throws IOException
+ */
+ private void updateZKWithEventData(HBaseEventType eventType, HMsg hmsg)
+ throws IOException {
+ byte[] data = makeZKEventData(eventType, hmsg);
+ LOG.debug("Updating ZNode " + regionZNode +
+ " with [" + eventType + "]" +
+ " expected version = " + zkVersion);
try {
- data = Writables.getBytes(new RegionTransitionEventData(hbEventType, regionServerName, hmsg));
- } catch (IOException e) {
- LOG.error("Error creating event data for " + hbEventType, e);
+ ZKUtil.updateExistingNodeData(zooKeeper, regionZNode, data, zkVersion);
+ } catch(KeeperException.BadVersionException e) {
+ zooKeeper.error("Version mismatch on unassigned znode when updating", e);
+ throw new IOException(e);
+ } catch(KeeperException e) {
+ zooKeeper.error("Unexpected exception trying to update unassigned node", e);
+ throw new IOException(e);
}
- LOG.debug("Updating ZNode " + regionZNode +
- " with [" + hbEventType + "]" +
- " expected version = " + zkVersion);
- lastUpdatedState = hbEventType;
- zooKeeper.writeZNode(regionZNode, data, zkVersion, true);
+ lastUpdatedState = eventType;
zkVersion++;
}
}
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java Thu Jul 15 20:32:31 2010
@@ -18,6 +18,17 @@
package org.apache.hadoop.hbase.thrift;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@@ -36,6 +47,7 @@ import org.apache.hadoop.hbase.HServerAd
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -73,17 +85,6 @@ import org.apache.thrift.transport.TServ
import org.apache.thrift.transport.TServerTransport;
import org.apache.thrift.transport.TTransportFactory;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
/**
* ThriftServer - this class starts up a Thrift server which implements the
* Hbase API specified in the Hbase.thrift IDL file.
@@ -185,8 +186,10 @@ public class ThriftServer {
* Constructs an HBaseHandler object.
*
* @throws MasterNotRunningException
+ * @throws ZooKeeperConnectionException
*/
- HBaseHandler() throws MasterNotRunningException {
+ HBaseHandler()
+ throws MasterNotRunningException, ZooKeeperConnectionException {
conf = HBaseConfiguration.create();
admin = new HBaseAdmin(conf);
scannerMap = new HashMap<Integer, ResultScanner>();
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java Thu Jul 15 20:32:31 2010
@@ -39,7 +39,7 @@ import org.apache.hadoop.util.StringUtil
*/
public class ZKConfig {
private static final Log LOG = LogFactory.getLog(ZKConfig.class);
-
+
private static final String VARIABLE_START = "${";
private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
private static final String VARIABLE_END = "}";
@@ -49,7 +49,7 @@ public class ZKConfig {
private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
private static final String ZK_CLIENT_PORT_KEY = ZK_CFG_PROPERTY
+ "clientPort";
-
+
/**
* Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
* If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
@@ -178,9 +178,9 @@ public class ZKConfig {
}
return properties;
}
-
+
/**
- * Return the ZK Quorum servers string given zk properties returned by
+ * Return the ZK Quorum servers string given zk properties returned by
* makeZKProps
* @param properties
* @return
@@ -240,7 +240,7 @@ public class ZKConfig {
return hostPortBuilder.toString();
}
-
+
/**
* Return the ZK Quorum servers string given the specified configuration.
* @param properties
Modified: hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java?rev=964570&r1=964569&r2=964570&view=diff
==============================================================================
--- hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java (original)
+++ hbase/branches/0.90_master_rewrite/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java Thu Jul 15 20:32:31 2010
@@ -19,14 +19,21 @@
*/
package org.apache.hadoop.hbase.zookeeper;
+import java.io.BufferedReader;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -65,6 +72,12 @@ public class ZKUtil {
throws IOException {
Properties properties = ZKConfig.makeZKProps(conf);
String quorum = ZKConfig.getZKQuorumServersString(properties);
+ return connect(conf, quorum, watcher);
+ }
+
+ public static ZooKeeper connect(Configuration conf, String quorum,
+ Watcher watcher)
+ throws IOException {
if(quorum == null) {
throw new IOException("Unable to determine ZooKeeper quorum");
}
@@ -91,6 +104,61 @@ public class ZKUtil {
return prefix + ZNODE_PATH_SEPARATOR + suffix;
}
+ /**
+ * Returns the full path of the immediate parent of the specified node.
+ * @param node path to get parent of
+ * @return parent of path, null if passed the root node or an invalid node
+ */
+ public static String getParent(String node) {
+ int idx = node.lastIndexOf(ZNODE_PATH_SEPARATOR);
+ return idx <= 0 ? null : node.substring(0, idx);
+ }
+
+ /**
+ * Get the unique node-name for the specified regionserver.
+ *
+ * Used when a server puts up an ephemeral node for itself and needs to use
+ * a unique name.
+ *
+ * Returns the fully-qualified znode path.
+ *
+ * @param serverInfo server information
+ * @return unique, zookeeper-safe znode path for the server instance
+ */
+ public static String getNodeName(HServerInfo serverInfo) {
+ return serverInfo.getServerName();
+ }
+
+ /**
+ * Get the key to the ZK ensemble for this configuration without
+ * adding a name at the end
+ * @param conf Configuration to use to build the key
+ * @return ensemble key without a name
+ */
+ public static String getZooKeeperClusterKey(Configuration conf) {
+ return getZooKeeperClusterKey(conf, null);
+ }
+
+ /**
+ * Get the key to the ZK ensemble for this configuration and append
+ * a name at the end
+ * @param conf Configuration to use to build the key
+ * @param name Name that should be appended at the end if not empty or null
+ * @return ensemble key with a name (if any)
+ */
+ public static String getZooKeeperClusterKey(Configuration conf, String name) {
+ String quorum = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll(
+ "[\\t\\n\\x0B\\f\\r]", ""));
+ StringBuilder builder = new StringBuilder(quorum);
+ builder.append(":");
+ builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
+ if (name != null && !name.isEmpty()) {
+ builder.append(",");
+ builder.append(name);
+ }
+ return builder.toString();
+ }
+
//
// Existence checks and watches
//
@@ -122,6 +190,33 @@ public class ZKUtil {
}
}
+ /**
+ * Check if the specified node exists. Sets no watches.
+ *
+ * Returns true if node exists, false if not. Returns an exception if there
+ * is an unexpected zookeeper exception.
+ *
+ * @param zkw zk reference
+ * @param znode path of node to watch
+ * @return true if znode exists, false if does not exist or error
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static boolean checkExists(ZooKeeperWatcher zkw, String znode)
+ throws KeeperException {
+ try {
+ Stat s = zkw.getZooKeeper().exists(znode, null);
+ return s != null ? true : false;
+ } catch (KeeperException e) {
+ zkw.warn("Unable to set watcher on znode (" + znode + ")", e);
+ zkw.keeperException(e);
+ return false;
+ } catch (InterruptedException e) {
+ zkw.warn("Unable to set watcher on znode (" + znode + ")", e);
+ zkw.interruptedException(e);
+ return false;
+ }
+ }
+
//
// Znode listings
//
@@ -164,6 +259,69 @@ public class ZKUtil {
}
/**
+ * Lists the children of the specified znode, retrieving the data of each
+ * child as a server address.
+ *
+ * Used to list the currently online regionservers and their addresses.
+ *
+ * Sets no watches at all, this method is best effort.
+ *
+ * Returns an empty list if the node has no children. Returns null if the
+ * parent node itself does not exist.
+ *
+ * @param zkw zookeeper reference
+ * @param znode node to get children of as addresses
+ * @return list of data of children of specified znode, empty if no children,
+ * null if parent does not exist
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static List<HServerAddress> listChildrenAndGetAsAddresses(
+ ZooKeeperWatcher zkw, String znode)
+ throws KeeperException {
+ List<String> children = listChildrenNoWatch(zkw, znode);
+ if(children == null) {
+ return null;
+ }
+ List<HServerAddress> addresses =
+ new ArrayList<HServerAddress>(children.size());
+ for(String child : children) {
+ addresses.add(getDataAsAddress(zkw, joinZNode(znode, child)));
+ }
+ return addresses;
+ }
+
+ /**
+ * Lists the children of the specified znode without setting any watches.
+ *
+ * Used to list the currently online regionservers and their addresses.
+ *
+ * Sets no watches at all, this method is best effort.
+ *
+ * Returns an empty list if the node has no children. Returns null if the
+ * parent node itself does not exist.
+ *
+ * @param zkw zookeeper reference
+ * @param znode node to get children of as addresses
+ * @return list of data of children of specified znode, empty if no children,
+ * null if parent does not exist
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static List<String> listChildrenNoWatch(
+ ZooKeeperWatcher zkw, String znode)
+ throws KeeperException {
+ List<String> children = null;
+ try {
+ // List the children without watching
+ children = zkw.getZooKeeper().getChildren(znode, null);
+ } catch(KeeperException.NoNodeException nne) {
+ return null;
+ } catch(InterruptedException ie) {
+ zkw.interruptedException(ie);
+ }
+ return children;
+ }
+
+ /**
* Checks if the specified znode has any children. Sets no watches.
*
* Returns true if the node exists and has children. Returns false if the
@@ -236,6 +394,44 @@ public class ZKUtil {
}
/**
+ * Get the data at the specified znode without setting a watch.
+ *
+ * Returns the data if the node exists. Returns null if the node does not
+ * exist.
+ *
+ * Sets the stats of the node in the passed Stat object. Pass a null stat if
+ * not interested.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ * @param stat node status to set if node exists
+ * @return data of the specified znode, or null if does not exist
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static byte [] getDataNoWatch(ZooKeeperWatcher zkw, String znode,
+ Stat stat)
+ throws KeeperException {
+ try {
+ byte [] data = zkw.getZooKeeper().getData(znode, zkw, stat);
+ zkw.debug("Retrieved " + data.length + " bytes of data from znode (" +
+ znode + ") and set a watcher");
+ return data;
+ } catch (KeeperException.NoNodeException e) {
+ zkw.debug("Unable to get data of znode (" + znode + ") " +
+ "because node does not exist (not necessarily an error)");
+ return null;
+ } catch (KeeperException e) {
+ zkw.warn("Unable to get data of znode (" + znode + ")", e);
+ zkw.keeperException(e);
+ return null;
+ } catch (InterruptedException e) {
+ zkw.warn("Unable to get data of znode (" + znode + ")", e);
+ zkw.interruptedException(e);
+ return null;
+ }
+ }
+
+ /**
* Get the data at the specified znode, deserialize it as an HServerAddress,
* and set a watch.
*
@@ -261,6 +457,35 @@ public class ZKUtil {
}
/**
+ * Update the data of an existing node with the expected version to have the
+ * specified data.
+ *
+ * Throws an exception if there is a version mismatch or some other problem.
+ *
+ * Sets no watches under any conditions.
+ *
+ * @param zkw zk reference
+ * @param znode
+ * @param data
+ * @param expectedVersion
+ * @throws KeeperException if unexpected zookeeper exception
+ * @throws KeeperException.BadVersionException if version mismatch
+ */
+ public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode,
+ byte [] data, int expectedVersion)
+ throws KeeperException {
+ try {
+ zkw.getZooKeeper().setData(znode, data, expectedVersion);
+ } catch(InterruptedException ie) {
+ zkw.interruptedException(ie);
+ }
+ }
+
+ //
+ // Data setting
+ //
+
+ /**
* Set the specified znode to be an ephemeral node carrying the specified
* server address. Used by masters for their ephemeral node and regionservers
* for their ephemeral node.
@@ -284,6 +509,11 @@ public class ZKUtil {
return createEphemeralNodeAndWatch(zkw, znode,
Bytes.toBytes(address.toString()));
}
+
+ //
+ // Node creation
+ //
+
/**
*
* Set the specified znode to be an ephemeral node carrying the specified
@@ -321,6 +551,42 @@ public class ZKUtil {
}
/**
+ *
+ * Set the specified znode to be a persistent node carrying the specified
+ * data.
+ *
+ * Returns true if the node was successfully created, false if the node
+ * already existed.
+ *
+ * If the node is created successfully, a watcher is also set on the node.
+ *
+ * If the node is not created successfully because it already exists, this
+ * method will also set a watcher on the node.
+ *
+ * If there is another problem, a KeeperException will be thrown.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ * @param data data of node
+ * @return true if node created, false if not, watch set in both cases
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static boolean createPersistentNodeIfNotExists(
+ ZooKeeperWatcher zkw, String znode, byte [] data)
+ throws KeeperException {
+ try {
+ zkw.getZooKeeper().create(znode, data, Ids.OPEN_ACL_UNSAFE,
+ CreateMode.EPHEMERAL);
+ } catch (KeeperException.NodeExistsException nee) {
+ return false;
+ } catch (InterruptedException e) {
+ zkw.interruptedException(e);
+ return false;
+ }
+ return true;
+ }
+
+ /**
* Creates the specified node, if the node does not exist. Does not set a
* watch and fails silently if the node already exists.
*
@@ -341,4 +607,152 @@ public class ZKUtil {
zkw.interruptedException(ie);
}
}
+
+ /**
+ * Creates the specified node and all parent nodes required for it to exist.
+ *
+ * No watches are set and no errors are thrown if the node already exists.
+ *
+ * The nodes created are persistent and open access.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ * @throws KeeperException if unexpected zookeeper exception
+ */
+ public static void createWithParents(ZooKeeperWatcher zkw,
+ String znode)
+ throws KeeperException {
+ try {
+ if(znode == null) {
+ return;
+ }
+ zkw.getZooKeeper().create(znode, new byte[0], Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ } catch(KeeperException.NodeExistsException nee) {
+ return;
+ } catch(KeeperException.NoNodeException nne) {
+ createWithParents(zkw, getParent(znode));
+ createWithParents(zkw, znode);
+ } catch(InterruptedException ie) {
+ zkw.interruptedException(ie);
+ }
+ }
+
+ //
+ // Deletes
+ //
+
+ /**
+ * Delete the specified node. Sets no watches. Throws all exceptions.
+ */
+ public static void deleteNode(ZooKeeperWatcher zkw, String node)
+ throws KeeperException {
+ try {
+ zkw.getZooKeeper().delete(node, -1);
+ } catch(InterruptedException ie) {
+ }
+ }
+
+ /**
+ * Delete the specified node and all of it's children.
+ *
+ * Sets no watches. Throws all exceptions besides dealing with deletion of
+ * children.
+ */
+ public static void deleteNodeRecursively(ZooKeeperWatcher zkw, String node)
+ throws KeeperException {
+ try {
+ List<String> children = ZKUtil.listChildrenNoWatch(zkw, node);
+ if(!children.isEmpty()) {
+ for(String child : children) {
+ deleteNodeRecursively(zkw, joinZNode(node, child));
+ }
+ }
+ zkw.getZooKeeper().delete(node, -1);
+ } catch(InterruptedException ie) {
+ }
+ }
+ //
+ // ZooKeeper cluster information
+ //
+
+ /** @return String dump of everything in ZooKeeper. */
+ public static String dump(ZooKeeperWatcher zkw) {
+ StringBuilder sb = new StringBuilder();
+ try {
+ sb.append("\nHBase tree in ZooKeeper is rooted at ").append(zkw.baseZNode);
+ sb.append("\n Cluster up? ").append(checkExists(zkw, zkw.clusterStateZNode));
+ sb.append("\n Master address: ").append(
+ getDataAsAddress(zkw, zkw.masterAddressZNode));
+ sb.append("\n Region server holding ROOT: ").append(
+ getDataAsAddress(zkw, zkw.rootServerZNode));
+ sb.append("\n Region servers:");
+ for (HServerAddress address : listChildrenAndGetAsAddresses(zkw,
+ zkw.rsZNode)) {
+ sb.append("\n - ").append(address);
+ }
+ sb.append("\n Quorum Server Statistics:");
+ String[] servers = zkw.getQuorum().split(",");
+ for (String server : servers) {
+ sb.append("\n - ").append(server);
+ try {
+ String[] stat = getServerStats(server);
+ for (String s : stat) {
+ sb.append("\n ").append(s);
+ }
+ } catch (Exception e) {
+ sb.append("\n ERROR: ").append(e.getMessage());
+ }
+ }
+ } catch(KeeperException ke) {
+ sb.append("\n FATAL ZooKeeper Exception!\n");
+ sb.append("\n " + ke.getMessage());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Gets the statistics from the given server. Uses a 1 minute timeout.
+ *
+ * @param server The server to get the statistics from.
+ * @return The array of response strings.
+ * @throws IOException When the socket communication fails.
+ */
+ public static String[] getServerStats(String server)
+ throws IOException {
+ return getServerStats(server, 60 * 1000);
+ }
+
+ /**
+ * Gets the statistics from the given server.
+ *
+ * @param server The server to get the statistics from.
+ * @param timeout The socket timeout to use.
+ * @return The array of response strings.
+ * @throws IOException When the socket communication fails.
+ */
+ public static String[] getServerStats(String server, int timeout)
+ throws IOException {
+ String[] sp = server.split(":");
+ Socket socket = new Socket(sp[0],
+ sp.length > 1 ? Integer.parseInt(sp[1]) : 2181);
+ socket.setSoTimeout(timeout);
+ PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
+ BufferedReader in = new BufferedReader(new InputStreamReader(
+ socket.getInputStream()));
+ out.println("stat");
+ out.flush();
+ ArrayList<String> res = new ArrayList<String>();
+ while (true) {
+ String line = in.readLine();
+ if (line != null) {
+ res.add(line);
+ } else {
+ break;
+ }
+ }
+ socket.close();
+ return res.toArray(new String[res.size()]);
+ }
+
}
\ No newline at end of file