You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/07/09 21:48:11 UTC
svn commit: r792654 - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ bin/ conf/
src/java/org/apache/hadoop/hbase/zookeeper/ src/test/org/apache/hadoop/hbase/
Author: apurtell
Date: Thu Jul 9 19:48:10 2009
New Revision: 792654
URL: http://svn.apache.org/viewvc?rev=792654&view=rev
Log:
HBASE-1551,HBASE-1635
Added:
hadoop/hbase/trunk_on_hadoop-0.18.3/bin/zookeepers.sh
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
Removed:
hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase-zookeeper.sh
hadoop/hbase/trunk_on_hadoop-0.18.3/bin/zookeeper.sh
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase-daemons.sh
hadoop/hbase/trunk_on_hadoop-0.18.3/bin/start-hbase.sh
hadoop/hbase/trunk_on_hadoop-0.18.3/bin/stop-hbase.sh
hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-default.xml
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/TestKeyValue.java
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Thu Jul 9 19:48:10 2009
@@ -452,12 +452,14 @@
HBASE-1624 Don't sort Puts if only one in list in HCM#processBatchOfRows
HBASE-1626 Allow emitting Deletes out of new TableReducer
(Lars George via Stack)
+ HBASE-1551 HBase should manage multiple node ZooKeeper quorum
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
HBASE-1535 Add client ability to perform mutations without the WAL
(Jon Gray via Stack)
HBASE-1460 Concurrent LRU Block Cache (Jon Gray via Stack)
+ HBASE-1635 PerformanceEvaluation should use scanner prefetching
Release 0.19.0 - 01/21/2009
INCOMPATIBLE CHANGES
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase-daemons.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase-daemons.sh?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase-daemons.sh (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase-daemons.sh Thu Jul 9 19:48:10 2009
@@ -37,6 +37,20 @@
. $bin/hbase-config.sh
-exec "$bin/regionservers.sh" --config "${HBASE_CONF_DIR}" \
- cd "${HBASE_HOME}" \; \
- "$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@"
+remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@"
+args="--config ${HBASE_CONF_DIR} $remote_cmd"
+
+command=$2
+case $command in
+ (regionserver)
+ exec "$bin/regionservers.sh" $args
+ ;;
+ (zookeeper)
+ exec "$bin/zookeepers.sh" $args
+ ;;
+ (*)
+ echo $usage
+ exit 1
+ ;;
+esac
+
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/bin/start-hbase.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/bin/start-hbase.sh?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/bin/start-hbase.sh (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/bin/start-hbase.sh Thu Jul 9 19:48:10 2009
@@ -38,8 +38,8 @@
then
exit $errCode
fi
-"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
- start zookeeper
+
+"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
"$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master
"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
--hosts "${HBASE_REGIONSERVERS}" start regionserver
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/bin/stop-hbase.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/bin/stop-hbase.sh?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/bin/stop-hbase.sh (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/bin/stop-hbase.sh Thu Jul 9 19:48:10 2009
@@ -30,5 +30,4 @@
. "$bin"/hbase-config.sh
"$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master
-"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
- stop zookeeper
+"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/bin/zookeepers.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/bin/zookeepers.sh?rev=792654&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/bin/zookeepers.sh (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/bin/zookeepers.sh Thu Jul 9 19:48:10 2009
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Copyright 2009 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+# Run a shell command on all zookeeper hosts.
+#
+# Environment Variables
+#
+# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
+# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+# HBASE_SSH_OPTS Options passed to ssh when running remote commands.
+#
+# Modelled after $HADOOP_HOME/bin/slaves.sh.
+
+usage="Usage: zookeepers [--config <hbase-confdir>] command..."
+
+# if no args specified, show usage
+if [ $# -le 0 ]; then
+ echo $usage
+ exit 1
+fi
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/hbase-config.sh
+
+if [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then
+ . "${HBASE_CONF_DIR}/hbase-env.sh"
+fi
+
+if [ "$HBASE_MANAGES_ZK" = "" ]; then
+ HBASE_MANAGES_ZK=true
+fi
+
+if [ "$HBASE_MANAGES_ZK" = "true" ]; then
+ hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool`
+ cmd=$"${@// /\\ }"
+ for zookeeper in $hosts; do
+ ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" &
+ if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
+ sleep $HBASE_SLAVE_SLEEP
+ fi
+ done
+fi
+
+wait
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-default.xml?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-default.xml (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-default.xml Thu Jul 9 19:48:10 2009
@@ -200,6 +200,21 @@
</description>
</property>
<property>
+ <name>hbase.zookeeper.dns.interface</name>
+ <value>default</value>
+ <description>The name of the Network Interface from which a ZooKeeper server
+ should report its IP address.
+ </description>
+ </property>
+ <property>
+ <name>hbase.zookeeper.dns.nameserver</name>
+ <value>default</value>
+ <description>The host name or IP address of the name server (DNS)
+ which a ZooKeeper server should use to determine the host name used by the
+ master for communication and display purposes.
+ </description>
+ </property>
+ <property>
<name>hbase.regionserver.dns.interface</name>
<value>default</value>
<description>The name of the Network Interface from which a region server
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java Thu Jul 9 19:48:10 2009
@@ -19,8 +19,11 @@
*/
package org.apache.hadoop.hbase.zookeeper;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PrintWriter;
+import java.net.UnknownHostException;
import java.util.Properties;
import java.util.Map.Entry;
@@ -28,11 +31,11 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.net.DNS;
import org.apache.zookeeper.server.ServerConfig;
import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
import org.apache.zookeeper.server.quorum.QuorumPeerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
/**
* HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage
@@ -53,28 +56,24 @@
private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
/**
- * Parse ZooKeeper configuration and run a QuorumPeer.
- * While parsing the zoo.cfg, we substitute variables with values from
- * hbase-site.xml.
+ * Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer.
* @param args String[] of command line arguments. Not used.
- * @throws IOException
*/
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) {
HBaseConfiguration conf = new HBaseConfiguration();
- Properties zkProperties = makeZKProps(conf);
-
- QuorumPeerConfig zkConfig = new QuorumPeerConfig();
try {
+ Properties zkProperties = makeZKProps(conf);
+ writeMyID(zkProperties);
+ QuorumPeerConfig zkConfig = new QuorumPeerConfig();
zkConfig.parseProperties(zkProperties);
- } catch (ConfigException e) {
+ runZKServer(zkConfig);
+ } catch (Exception e) {
e.printStackTrace();
System.exit(-1);
}
-
- startZKServer(zkConfig);
}
- private static void startZKServer(QuorumPeerConfig zkConfig) throws IOException {
+ private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException {
if (zkConfig.isDistributed()) {
QuorumPeerMain qp = new QuorumPeerMain();
qp.runFromConfig(zkConfig);
@@ -86,6 +85,48 @@
}
}
+ private static void writeMyID(Properties properties) throws UnknownHostException, IOException {
+ HBaseConfiguration conf = new HBaseConfiguration();
+ String myAddress = DNS.getDefaultHost(
+ conf.get("hbase.zookeeper.dns.interface","default"),
+ conf.get("hbase.zookeeper.dns.nameserver","default"));
+
+ long myId = -1;
+
+ for (Entry<Object, Object> entry : properties.entrySet()) {
+ String key = entry.getKey().toString().trim();
+ String value = entry.getValue().toString().trim();
+ if (key.startsWith("server.")) {
+ int dot = key.indexOf('.');
+ long id = Long.parseLong(key.substring(dot + 1));
+ String[] parts = value.split(":");
+ String address = parts[0];
+ if (myAddress.equals(address)) {
+ myId = id;
+ break;
+ }
+ }
+ }
+
+ if (myId == -1) {
+ throw new IOException("Could not find my address: " + myAddress +
+ " in list of ZooKeeper quorum servers");
+ }
+
+ String dataDirStr = properties.get("dataDir").toString().trim();
+ File dataDir = new File(dataDirStr);
+ if (!dataDir.isDirectory()) {
+ if (!dataDir.mkdirs()) {
+ throw new IOException("Unable to create data dir " + dataDir);
+ }
+ }
+
+ File myIdFile = new File(dataDir, "myid");
+ PrintWriter w = new PrintWriter(myIdFile);
+ w.println(myId);
+ w.close();
+ }
+
/**
* Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
* If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
@@ -193,11 +234,11 @@
if (key.startsWith("server.")) {
if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) &&
value.startsWith("localhost")) {
- String msg = "The server in zoo.cfg cannot be set to localhost " +
+ String msg = "The server in zoo.cfg cannot be set to localhost " +
"in a fully-distributed setup because it won't be reachable. " +
"See \"Getting Started\" for more information.";
- LOG.fatal(msg);
- throw new IOException(msg);
+ LOG.fatal(msg);
+ throw new IOException(msg);
}
}
newValue.append(value.substring(varEnd));
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java?rev=792654&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java Thu Jul 9 19:48:10 2009
@@ -0,0 +1,33 @@
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+
+/**
+ * Tool for reading ZooKeeper servers from HBase XML configuation and producing
+ * a line-by-line list for use by bash scripts.
+ */
+public class ZKServerTool implements HConstants {
+ /**
+ * Run the tool.
+ * @param args Command line arguments. First arg is path to zookeepers file.
+ */
+ public static void main(String args[]) {
+ HBaseConfiguration conf = new HBaseConfiguration();
+ // Note that we do not simply grab the property ZOOKEEPER_QUORUM from
+ // the HBaseConfiguration because the user may be using a zoo.cfg file.
+ Properties zkProps = HQuorumPeer.makeZKProps(conf);
+ for (Entry<Object, Object> entry : zkProps.entrySet()) {
+ String key = entry.getKey().toString().trim();
+ String value = entry.getValue().toString().trim();
+ if (key.startsWith("server.")) {
+ String[] parts = value.split(":");
+ String host = parts[0];
+ System.out.println(host);
+ }
+ }
+ }
+}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java Thu Jul 9 19:48:10 2009
@@ -383,6 +383,7 @@
this.table = new HTable(conf, TABLE_DESCRIPTOR.getName());
this.table.setAutoFlush(false);
this.table.setWriteBufferSize(1024*1024*12);
+ this.table.setScannerCaching(30);
}
void testTakedown() throws IOException {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/TestKeyValue.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/TestKeyValue.java?rev=792654&r1=792653&r2=792654&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/TestKeyValue.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/TestKeyValue.java Thu Jul 9 19:48:10 2009
@@ -276,4 +276,4 @@
// TODO actually write this test!
}
-}
\ No newline at end of file
+}