You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by iv...@apache.org on 2011/08/30 10:34:32 UTC
svn commit: r1163137 - in /zookeeper/bookkeeper/trunk: ./ bookkeeper-server/
bookkeeper-server/bin/ bookkeeper-server/conf/ bookkeeper-server/lib/ conf/
doc/ hedwig-client/conf/ hedwig-server/ hedwig-server/bin/
hedwig-server/conf/ hedwig-server/lib/ s...
Author: ivank
Date: Tue Aug 30 08:34:31 2011
New Revision: 1163137
URL: http://svn.apache.org/viewvc?rev=1163137&view=rev
Log:
BOOKKEEPER-28: Create useful startup scripts for bookkeeper and hedwig (ivank)
Added:
zookeeper/bookkeeper/trunk/bookkeeper-server/bin/
zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper
zookeeper/bookkeeper/trunk/bookkeeper-server/conf/
zookeeper/bookkeeper/trunk/bookkeeper-server/conf/bkenv.sh
zookeeper/bookkeeper/trunk/hedwig-client/conf/
zookeeper/bookkeeper/trunk/hedwig-client/conf/hw_client.conf
zookeeper/bookkeeper/trunk/hedwig-server/bin/
zookeeper/bookkeeper/trunk/hedwig-server/bin/hedwig
zookeeper/bookkeeper/trunk/hedwig-server/conf/
zookeeper/bookkeeper/trunk/hedwig-server/conf/hw_server.conf
zookeeper/bookkeeper/trunk/hedwig-server/conf/hwenv.sh
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/lib/README
zookeeper/bookkeeper/trunk/bookkeeper-server/pom.xml
zookeeper/bookkeeper/trunk/conf/hw_client_sample.conf
zookeeper/bookkeeper/trunk/conf/hw_server_sample.conf
zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile
zookeeper/bookkeeper/trunk/doc/bookkeeperStarted.textile
zookeeper/bookkeeper/trunk/doc/hedwigBuild.textile
zookeeper/bookkeeper/trunk/doc/hedwigDesign.textile
zookeeper/bookkeeper/trunk/doc/hedwigUser.textile
zookeeper/bookkeeper/trunk/hedwig-server/lib/README
zookeeper/bookkeeper/trunk/hedwig-server/pom.xml
zookeeper/bookkeeper/trunk/scripts/README.txt
zookeeper/bookkeeper/trunk/scripts/analyze.py
zookeeper/bookkeeper/trunk/scripts/hw.bash
zookeeper/bookkeeper/trunk/scripts/hwServer.sh
zookeeper/bookkeeper/trunk/scripts/quote
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Tue Aug 30 08:34:31 2011
@@ -42,6 +42,8 @@ BUGFIXES:
IMPROVEMENTS:
+ BOOKKEEPER-28: Create useful startup scripts for bookkeeper and hedwig (ivank)
+
hedwig-client/
BOOKKEEPER-44: Reuse publish channel to default server to avoid too many connect requests to default server when lots of producers came in same time (Sijie Guo via breed)
Added: zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper?rev=1163137&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/bin/bookkeeper Tue Aug 30 08:34:31 2011
@@ -0,0 +1,140 @@
+#!/bin/sh
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd $BINDIR/..;pwd`
+
+DEFAULT_ZOOKEEPER=localhost:2181
+DEFAULT_PORT=3181
+DEFAULT_TXN_DIR=/tmp/bk-txn
+DEFAULT_DATA_DIR=/tmp/bk-data
+
+source $BK_HOME/conf/bkenv.sh
+
+RELEASE_JAR=`ls $BK_HOME/bookkeeper-server-*.jar 2> /dev/null | tail -1`
+if [ $? == 0 ]; then
+ BOOKIE_JAR=$RELEASE_JAR
+fi
+
+BUILT_JAR=`ls $BK_HOME/target/bookkeeper-server-*.jar 2> /dev/null | tail -1`
+if [ $? != 0 ] && [ ! -e "$BOOKIE_JAR" ]; then
+ echo "\nCouldn't find bookkeeper jar.";
+ echo "Make sure you've run 'mvn package'\n";
+ exit 1;
+elif [ -e "$BUILT_JAR" ]; then
+ BOOKIE_JAR=$BUILT_JAR
+fi
+
+bookkeeper_help() {
+ cat <<EOF
+Usage: bookkeeper <command>
+where command is one of:
+ bookie Run a bookie server
+ localbookie <n> Run a test ensemble of <n> bookies locally
+ help This help message
+
+or command is the full name of a class with a defined main() method.
+
+Environment variables:
+ BOOKIE_ZOOKEEPER Zookeeper ensemble (default: $DEFAULT_ZOOKEEPER)
+ BOOKIE_PORT Port to listen on (default: $DEFAULT_PORT)
+ BOOKIE_TXN_LOGDIR Directory for transaction logs (default: $DEFAULT_TXN_DIR)
+ BOOKIE_DATA_DIR Directory for data (default: $DEFAULT_DATA_DIR)
+ BOOKIE_LOG_CONF Log4j configuration file
+ BOOKIE_EXTRA_OPTS Extra options to be passed to the jvm
+
+These variable can also be set in conf/bkenv.sh
+EOF
+}
+
+add_maven_deps_to_classpath() {
+ MVN="mvn"
+ if [ "$MAVEN_HOME" != "" ]; then
+ MVN=${MAVEN_HOME}/bin/mvn
+ fi
+
+ # Need to generate classpath from maven pom. This is costly so generate it
+ # and cache it. Save the file into our target dir so a mvn clean will get
+ # clean it up and force us create a new one.
+ f="${BK_HOME}/target/cached_classpath.txt"
+ if [ ! -f "${f}" ]
+ then
+ ${MVN} -f "${BK_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null
+ fi
+ BOOKIE_CLASSPATH=${CLASSPATH}:`cat "${f}"`
+}
+
+if [ -d "$BK_HOME/lib" ]; then
+ for i in $BK_HOME/lib/*.jar; do
+ BOOKIE_CLASSPATH=$BOOKIE_CLASSPATH:$i
+ done
+else
+ add_maven_deps_to_classpath
+fi
+
+# if no args specified, show usage
+if [ $# = 0 ]; then
+ bookkeeper_help;
+ exit 1;
+fi
+
+# get arguments
+COMMAND=$1
+shift
+
+if [ "$BOOKIE_ZOOKEEPER" == "" ]; then
+ BOOKIE_ZOOKEEPER=$DEFAULT_ZOOKEEPER
+fi
+
+if [ "$BOOKIE_PORT" == "" ]; then
+ BOOKIE_PORT=$DEFAULT_PORT
+fi
+
+if [ "$BOOKIE_TXN_LOGDIR" == "" ]; then
+ BOOKIE_TXN_LOGDIR=$DEFAULT_TXN_DIR
+fi
+
+if [ "$BOOKIE_DATA_DIR" == "" ]; then
+ BOOKIE_DATA_DIR=$DEFAULT_DATA_DIR
+fi
+
+BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH"
+if [ "$BOOKIE_LOG_CONF" != "" ]; then
+ BOOKIE_CLASSPATH="`dirname $BOOKIE_LOG_CONF`:$BOOKIE_CLASSPATH"
+ OPTS="$OPTS -Dlog4j.configuration=`basename $BOOKIE_LOG_CONF`"
+fi
+OPTS="-cp $BOOKIE_CLASSPATH $OPTS $BOOKIE_EXTRA_OPTS"
+
+OPTS="$OPTS $BOOKIE_EXTRA_OPTS"
+
+if [ $COMMAND == "bookie" ]; then
+ exec java $OPTS org.apache.bookkeeper.proto.BookieServer $BOOKIE_PORT $BOOKIE_ZOOKEEPER $BOOKIE_TXN_LOGDIR $BOOKIE_DATA_DIR $@
+elif [ $COMMAND == "localbookie" ]; then
+ NUMBER=$1
+ shift
+ exec java $OPTS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $@
+elif [ $COMMAND == "help" ]; then
+ bookkeeper_help;
+else
+ exec java $OPTS $COMMAND $@
+fi
+
Added: zookeeper/bookkeeper/trunk/bookkeeper-server/conf/bkenv.sh
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/conf/bkenv.sh?rev=1163137&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/conf/bkenv.sh (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/conf/bkenv.sh Tue Aug 30 08:34:31 2011
@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# default settings for starting bookkeeper
+
+# Zookeeper ensemble for bookkeeper to use
+#BOOKIE_ZOOKEEPER=
+
+# Port for bookie to listen on
+#BOOKIE_PORT=
+
+# Directory Bookkeeper outputs its write ahead log
+#BOOKIE_TXN_LOGDIR=
+
+# Directory Bookkeeper outputs ledger snapshots
+#BOOKIE_DATA_DIR=
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/pom.xml?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/pom.xml (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/pom.xml Tue Aug 30 08:34:31 2011
@@ -57,28 +57,6 @@
<forkMode>pertest</forkMode>
</configuration>
</plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- <archive>
- <manifest>
- <mainClass>${mainclass}</mainClass>
- </manifest>
- </archive>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>attached</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
<repositories>
Modified: zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile (original)
+++ zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile Tue Aug 30 08:34:31 2011
@@ -30,15 +30,14 @@ h1. Running bookies
p. To run a bookie, we execute the following command:
- @java -cp .:./zookeeper-<version>-bookkeeper.jar:./zookeeper-<version>.jar\ :../log4j/apache-log4j-1.2.15/log4j-1.2.15.jar -Dlog4j.configuration=log4j.properties\ org.apache.bookkeeper.proto.BookieServer 3181 127.0.0.1:2181 /path_to_log_device/\ /path_to_ledger_device/ @
+ @ bookkeeper-server/bin/bookkeeper bookie
-p. The parameters are:
-
-* Port number that the bookie listens on;
-* Comma separated list of ZooKeeper servers with a hostname:port format;
-* Path for Log Device (stores bookie write-ahead log);
-* Path for Ledger Device (stores ledger entries);
+p. The configuration parameters, which can be set in bookkeeper-server/conf/bkenv.sh
+* BOOKIE_PORT: Port number that the bookie listens on;
+* BOOKIE_ZOOKEEPER: Comma separated list of ZooKeeper servers with a hostname:port format;
+* BOOKIE_TXN_LOGDIR: Path for Log Device (stores bookie write-ahead log);
+* BOOKIE_DATA_DIR: Path for Ledger Device (stores ledger entries);
p. Ideally, @/path_to_log_device/@ and @/path_to_ledger_device/@ are each in a different device.
Modified: zookeeper/bookkeeper/trunk/doc/bookkeeperStarted.textile
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/doc/bookkeeperStarted.textile?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/doc/bookkeeperStarted.textile (original)
+++ zookeeper/bookkeeper/trunk/doc/bookkeeperStarted.textile Tue Aug 30 08:34:31 2011
@@ -20,15 +20,19 @@ p. This document contains information to
h1. Pre-requisites
-p. See "System Requirements":bookkeeperConfig.html#bk_sysReqin the Admin guide.
+p. See "System Requirements":./bookkeeperConfig.html#bk_sysReqin the Admin guide.
h1. Download
-p. BookKeeper is distributed along with ZooKeeper. To get a ZooKeeper distribution, download a recent "stable":http://hadoop.apache.org/zookeeper/releases.htmlrelease from one of the Apache Download Mirrors.
+p. BookKeeper trunk can be downloaded from subversion. See "Version Control:http://zookeeper.apache.org/bookkeeper/svn.html.
h1. LocalBookKeeper
-p. Under org.apache.bookkeeper.util, you'll find a java program called LocalBookKeeper.java that sets you up to run BookKeeper on a single machine. This is far from ideal from a performance perspective, but the program is useful for both test and educational purposes.
+p. BookKeeper provides a utility program to start a standalone ZooKeeper ensemble and a number of bookies on a local machine. As this all runs on a local machine, throughput will be very low. It should only be used for testing.
+
+p. To start a local bookkeeper ensemble with 5 bookies:
+
+@ bookkeeper-server/bin/bookkeeper localbookie 5
h1. Setting up bookies
@@ -36,9 +40,13 @@ p. If you're bold and you want more than
p. For each bookie, we need to execute a command like the following:
- @java -cp .:./zookeeper-<version>-bookkeeper.jar:./zookeeper-<version>.jar\ :lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.15.jar -Dlog4j.configuration=log4j.properties\ org.apache.bookkeeper.proto.BookieServer 3181 127.0.0.1:2181 /path_to_log_device/\ /path_to_ledger_device/ @
+ @ bookkeeper-server/bin/bookkeeper bookie
+
+p. This command will use the default directories for storing ledgers and the write ahead log, and will look for a zookeeper server on localhost:2181. To modify this you must edit bookkeeper-server/conf/bkenv.sh. BOOKIE_TXN_LOGDIR and BOOKIE_DATA_DIR should be configured to used different physical devices for best performance. BOOKIE_ZOOKEEPER should be a comma separated list of zookeeper servers.
-p. "/path_to_log_device/" and "/path_to_ledger_device/" are different paths. Also, port 3181 is the port that a bookie listens on for connection requests from clients. 127.0.0.1:2181 is the hostname:port for the ZooKeeper server. In this example, the standalone ZooKeeper server is running locally on port 2181. If we had multiple ZooKeeper servers, this parameter would be a comma separated list of all the hostname:port values corresponding to them.
+p. To see the default values of these configuration variables, run:
+
+ @ bookkeeper-server/bin/bookkeeper help
h1. Setting up ZooKeeper
@@ -48,7 +56,6 @@ p. ZooKeeper stores metadata on behalf o
# @/ledgers/available @
# For each bookie, we add one znode such that the name of the znode is the concatenation of the machine name and the port number that the bookie is listening on. For example, if a bookie is running on bookie.foo.com an is listening on port 3181, we add a znode @/ledgers/available/bookie.foo.com:3181@ .
-
h1. Example
p. In the following excerpt of code, we:
Modified: zookeeper/bookkeeper/trunk/doc/hedwigBuild.textile
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/doc/hedwigBuild.textile?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/doc/hedwigBuild.textile (original)
+++ zookeeper/bookkeeper/trunk/doc/hedwigBuild.textile Tue Aug 30 08:34:31 2011
@@ -5,54 +5,13 @@ For the core itself:
* JDK 6: "http://java.sun.com/":http://java.sun.com/. Ensure @$JAVA_HOME@ is correctly set.
* Maven 2: "http://maven.apache.org/":http://maven.apache.org/.
* Protocol Buffers 2.3.0: "http://protobuf.googlecode.com/":http://protobuf.googlecode.com/.
-* Zookeeper 3.4.0: "http://hadoop.apache.org/zookeeper/":http://hadoop.apache.org/zookeeper/. See below.
-* Bookkeeper 3.4.0: "http://hadoop.apache.org/zookeeper/":http://hadoop.apache.org/zookeeper/. See below.
+* Zookeeper 3.4.0: "http://zookeeper.apache.org/":http://zookeeper.apache.org/.
Hedwig has been tested on Windows XP, Linux 2.6, and OS X.
-For the deployment and distributed support scripts in @hw.bash@:
-
-* Ant: "http://ant.apache.org/":http://ant.apache.org/, if you want to build Zookeeper.
-* Bash: "http://www.gnu.org/software/bash/":http://www.gnu.org/software/bash/.
-* Coreutils: "http://www.gnu.org/software/coreutils/":http://www.gnu.org/software/coreutils/.
-* Expect: "http://expect.nist.gov/":http://expect.nist.gov/, if you want @unbuffer@.
-* Findutils: "http://www.gnu.org/software/findutils/":http://www.gnu.org/software/findutils/.
-* OpenSSH: "http://www.openssh.com/":http://www.openssh.com/.
-* Python 2.6: "http://python.org/":http://python.org/.
-
-h2. Protocol Buffers
-
-Hedwig requires the use of the Java runtime libraries of Protocol Buffers 2.3.0. These libraries need to be installed into your local maven repository. (Maven allows multiple versions to be installed.) To install protocol buffels to your local repository, you have to download the tarball and follow the README.txt instructions. Note that you must first install the C++ package which contains the compiler (protoc) before you can build the java libraries. That will install the library jar's in the local maven repository where Hedwig is currently configured to point to.
-
-h2. Zookeeper and Bookkeeper
-
-Hedwig currently requires the version of Bookkeeper maintained in Apache's current trunk SVN respository (version 3.4.0). This is not a released version yet but certain features needed for BookKeeper are only available there.
-
-Hedwig also depends on ZK testing code for its own testing code.
-
-Since Hedwig is a Maven project, all these dependencies must be made available as Maven artifacts. However, neither ZK nor BK are currently Mavenized. Hedwig provides some bash scripts to ease the installation of ZK, ZK tests, and BK, all as Maven artifacts.
-
-Currently, we have included the necessary ZooKeeper and BookKeeper jars in the Hedwig source itself in the $HEDWIG_DIR/server/lib directory. There is no need to retrieve them directly from the Apache download site as they are non-released trunk versions.
-
-h1. Not relevant right now since we already have the ZK jars already in the Hedwig source.
-
-To fetch and build ZK 3.4.0 (and its tests) in the current directory, run:
-
-$HEDWIG_DIR/scripts/hw.bash get-zk
-
-h1. Not relevant right now, but when we start using the apache version of BK, to
-
-build the local version of BK:
-
-$HEDWIG_DIR/scripts/hw.bash get-bk
-
-The $HEDWIG_DIR/server/lib directory contains all of the the class and source jars for ZK, ZK tests, and BK. To install these, go to that directory and run the following command to install them into your local maven repository:
-
-$HEDWIG_DIR/scripts/hw.bash install-zk-bk
-
h1. Command-Line Instructions
-From the main Hedwig directory, run @mvn package@. This will produce the executable jars for both the client and server, as well as a server "assembly jar" containing all dependencies as well for easier deployment.
+From the top level bookkeeper directory, run @mvn package@. This will compile and package the jars necessary for running hedwig.
See the User's Guide for instructions on running and usage.
@@ -70,23 +29,3 @@ To check out, build, and develop using E
You are now ready to run and debug the client and server code. See the User's Guide for instructions on running and usage.
-h1. Utilities
-
-h2. Removing Conflicting Files in Jars
-
-The Maven assembly plugin that produces the fat assembly jar may end up putting into the jar files with the same conflicting paths from multiple dependencies. This makes working with the files from certain tools (like @jar@) a bit jarring. In our case, these files are not things like class files, but rather README and LICENSE files, so we can safely remove conflicts by choosing an arbitrary winner. To do so, run:
-
-$HEDWIG_DIR/scripts/hw.bash strip-jar
-
-h2. Adjusting Logging
-
-The logging level is something that is baked into the jar in the @log4j.properties@ resource. However, it would be wasteful to go through a Maven build cycle to update and adjust this. If you're working from a source tree, it's also annoying to have to edit a source file to adjust the logging.
-
-We have a little script for tweaking the logging level. After running @strip-jar@, run:
-
-$HEDWIG_DIR/scripts/hw.bash set-logging WARN
-
-To see what the current logging level is:
-
-$HEDWIG_DIR/scripts/hw.bash get-logging
-
Modified: zookeeper/bookkeeper/trunk/doc/hedwigDesign.textile
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/doc/hedwigDesign.textile?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/doc/hedwigDesign.textile (original)
+++ zookeeper/bookkeeper/trunk/doc/hedwigDesign.textile Tue Aug 30 08:34:31 2011
@@ -20,127 +20,6 @@ h2. Handler Pipeline
A pipeline implements the intercepting filter pattern. A pipeline is a sequence of handlers. Whenever a packet is read from the wire, it travels up the stream, stopping at each handler that can handle upstream events. Vice-versa for writes. Between each filter, control flows back through the centralized pipeline, and a linked list of contexts keeps track of where we are in the pipeline (one context object per handler).
-h1. Distributed Performance Evaluation
-
-We've included some scripts to repeatedly run varying configurations of Hedwig on a distributed testbed and collect the resulting data. The experiments use the @org.apache.hedwig.client.App@ client application and are driven by @scripts/hw.bash@ (via the @app@ command).
-
-Currently, we have two types of experiments: subscription benchmarks and publishing benchmarks.
-
-h2. Subscription Benchmarks
-
-The subscription benchmark compares synchronous and asynchronous subscriptions. Because the synchronicity of subscriptions is a server configuration parameter, the servers must be restarted to change this. The benchmarks varies the maximum number of outstanding subscription requests.
-
-To run the subscription benchmark with wilbur6 as the subscriber and wilbur1 as its default hub:
-
-hosts=wilbur6 scripts/hw.bash sub-exp wilbur1
-
-This produces log files into the @sub@ directory, which may then be analyzed using the analysis scripts.
-
-h2. Publishing Benchmarks
-
-The publishing benchmark measures the throughput and latency of publishing messages within a LAN and across a WAN. It varies the following parameters:
-
-* maximum number of outstanding publish requests
-* number of publishers
-* number of (local) receivers
-
-We vary each dimension separately (and have default settings) to avoid a combinatorial explosion in the number of configurations to explore.
-
-First, start a (standalone) instance:
-
-scripts/hw.bash start-region '' $hwhost $zkhost $bk1host $bk2host $bk3host
-
-To run this over @$host1@ through @$host3@, with the number of publishers/subscribers varying linearly over this set:
-
-npars="20 40 60 80 100" scripts/hw.bash pub-exps "$host1 $host2 $host3" $hwhost $zkhost
-
-This will vary the number of outstanding publish requests as specified in @npars@.
-
-You may also optionally run this experiment with a second subscribing region:
-
-scripts/hw.bash start-zk-bks $zkhost $bk1host $bk2host <span class="math">bk3host npars="..." scripts/hw.bash pub-exps "</math>host1 $host2 $host3" $hwhost $zkhost $rrecv $rhwhost $rzkhost
-
-where the final three extra arguments specify the client receiver, Hedwig, and Zookeeper hosts, in that order.
-
-This command will produce files into @./pub/@, which can then be process using @analyze.py@.
-
-h1. Analysis and Visualization
-
-@scripts/analyze.py@ produces plots from the collected experimental data. It has just a few immediate dependencies. In the following, the indentation signifies nested dependencies, like an upside-down tree:
-
-bc. component AAA that component AA requires
- component AAB that component AA requires
-component AA that component A requires
- component ABA that component AB requires
- component ABB that component AB requires
-component AB that component A requires
-
-
-component A that analysis tools depend on component BAA that component BA requires component BAB that component BA requires component BA that component B requires component BBA that component BB requires component BBB that component BB requires component BB that component B requires component B that analysis tools depend on
-
-The reason the tree is upside-down is so that you can treat this whole thing as a chunk of bash script.
-
-"toast":http://toastball.net/toast/ is a utility that makes it a breeze to install all this software, but you do need to make sure your environment is set up correctly (e.g. @PKG_CONFIG_PATH@ must point to @~/.toast/armed/lib/pkgconfig/@).
-
-Setup:
-
-wget -O- http://toastball.net/toast/toast|perl -x - arm toast
-
-toast arm "http://www.python.org/ftp/python/2.6.2/Python-2.6.2.tar.bz2"
-
-toast arm numpy
-
-<pre>
- toast arm libpng
-
- toast arm pixman
-
- toast arm freetype
-
- toast arm 'ftp://xmlsoft.org/libxml2/libxml2-2.7.3.tar.gz'
-
- toast arm fontconfig
-
- toast arm cairo
-
-toast arm pycairo
-</pre>
-
-hg clone https://yang@bitbucket.org/yang/pycha/ pycha/setup.bash -d -p $path_to_install_to
-
-svn co https://assorted.svn.sourceforge.net/svnroot/assorted/python-commons/trunk/ python-commons/ python-commons/setup.bash -d -p $path_to_install_to
-
-To analyze the publishing experiments, change to the @pub@ data directory and run:
-
-scripts/analyze.py pub
-
-To analyze the subscription experiments, change to the @sub@ data directory and run:
-
-scripts/analyze.py sub
-
-h1. Debugging
-
-You can attach an Eclipse debugger (or any debugger) to a Java process running on a remote host, as long as it has been started with the appropriate JVM flags. (See the Building Hedwig document to set up your Eclipse environment.) To launch something using @hw.bash@ with debugger attachment enabled, prefix the command with @attach=true@, e.g.:
-
-attach=true scripts/hw.bash start-regions myregions.cfg
-
-h1. Profiling
-
-The scripts we have provided include ways for you to launch with YourKit profiling enabled.
-
-To deploy YourKit onto a number of machines:
-
-hosts="..." scripts/hw.bash setup-yjp $path_to_yjp
-
-where the path points to the "YourKit Linux zip archive":http://www.yourkit.com/download/yjp-8.0.15.zip (which is freely available and doesn't require any license to use).
-
-Now when using the scripts to run distributed experiments, to profile anything with YourKit, prefix the command with @use_yjp=true@. E.g.:
-
-use_yjp=true scripts/hw.bash start-regions regions.cfg
-
-Now you may start on your local machine the YourKit GUI and connect to the hosts that you're interested in.
-
-Note that you may want to disable the default set of filters in YourKit.
h1. Pseudocode
Modified: zookeeper/bookkeeper/trunk/doc/hedwigUser.textile
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/doc/hedwigUser.textile?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/doc/hedwigUser.textile (original)
+++ zookeeper/bookkeeper/trunk/doc/hedwigUser.textile Tue Aug 30 08:34:31 2011
@@ -20,138 +20,32 @@ The main class for running the server is
The client is a library intended to be consumed by user applications. It takes a Commons Configuration object, for which the source/documentation is in @org.apache.hedwig.client.conf.ClientConfiguration@.
-We have provided a simple client application, @org.apache.hedwig.client.App@, that can drive a number of benchmarks. This also takes a single configuration file argument, which is fed to the client library.
-
-We've provided a number of scripts to faciliate running servers and clients in a variety of configurations, including over distributed hosts. These are all consolidated in @scripts/hw.bash@. Although the snippets in this documentation run the script from the hedwig main directory, you can run it from any location. Apologies in advance for these being bash scripts; time permitting, a more robust and maintainable support/tooling infrastructure would be ideal.
-
h1. Deployment
-When ssh-ing into a new host, you are requested to verify and accept the host key. In order to automatically accept the host keys for many new hosts (dangerous), use:
-
-hosts="$host1 $host2 ..." scripts/hw.bash warmup
-
-The @hosts@ variable is set here to the list of hosts that you would like to warm up.
-
-To set up JDK6 on some hosts, use:
-
-hosts="..." scripts/hw.bash setup-java $path_to_modified_jdk6
-
-The argument must point to a JDK6 binary self-extracting executable, but with the @more@ command that displays the License agreement replaced with @cat@. Unfortunately, this step must be performed manually. This script will extract the JDK directly into the home directory and update @$PATH@ in @~/.bashrc@ (in an idempotent fashion).
-
-Because the current implementation uses a single socket per subscription, the Hedwig launching scripts all require a high @ulimit@ on the number of open file descriptors. Non-root users can only use up to the limit specified in @/etc/security/limits.conf@; to raise this to 1024^2, run:
-
-hosts="..." scripts/hw.bash setup-limits
-
-This uses @ssh@ so that you need to enter your password for @sudo@ just once.
-
-For most of the commands presented in the next section, you may prefix the command with:
-
-push_jar=true ...
-
-to first push the assembly jar (assumed to be available in @server/target/@) to all hosts.
-
-h1. Running Servers
-
-To start three BK bookies on ports 3181-3183 on localhost (directories must all exist):
-
-scripts/hw.bash bk 3181 $bk1_journal_dir $bk1_ledger_dir & scripts/hw.bash bk 3182 $bk2_journal_dir $bk2_ledger_dir & scripts/hw.bash bk 3183 $bk3_journal_dir $bk3_ledger_dir &
-
-To start a ZK on port 2181 (directory must exist):
-
-scripts/hw.bash zk 2181 /path/for/zk/ &
-
-To register the BKs with the ZK (so that Hedwig knows where to find the bookies):
-
-scripts/hw.bash setup-bk localhost:2181 @hostname@:3181 @hostname@:3182 @hostname@:3183
-
-Everything up to this point may be done using a single command over a set of hosts, with ZK on port 9877 and BK on port 9878. The following function takes 2 arguments. The first is the ZK host. The second is a string list of BK hosts:
-
-scripts/hw.bash start-zk-bks <span class="math">zkhost "</math>bk1host $bk2host $bk3host ..."
-
-Note that the hosts may be SSH profile aliases in your @~/.ssh/config@; the script will parse this file and look up their hostnames where necessary. This applies for the hosts specified in the other commands.
-
-Also, the scripts use the @bk-journal@ and @bk-ledger@ functions in @hw.bash@ to determine where to place the BK journal and ledger, given a hostname.
-
-To start a Hedwig server locally:
-
-scripts/hw.bash hw server.conf &
-
-To start Hedwig servers on some hosts "$hw1host $hw2host $hw3host ..." on port 9876, using $zkhost as the ZK server:
-
-scripts/hw.bash start-hw '' "$hw1host $hw2host $hw3host ..." $zkhost
-
-Above, the first empty string argument is the list of default servers to each of the other regions. You may run multiple connected instances of Hedwig this way.
-E.g., to start three regions each with a single Hedwig hub that talk to each other, and using the hw.bash default server ports of 9875 (non-SSL) and 9876 (SSL):
-
-scripts/hw.bash start-hw "$hw2host:9875:9876 <span class="math">hw3host:9875:9876" "</math>hw1host" <span class="math">zk1host scripts/hw.bash start-hw "</math>hw1host:9875:9876 <span class="math">hw3host:9875:9876" "</math>hw2host" <span class="math">zk2host scripts/hw.bash start-hw "</math>hw1host:9875:9876 <span class="math">hw2host:9875:9876" "</math>hw3host" $zk3host
-
-Everything up to this point may be done using a single command over a set of hosts:
-
-scripts/hw.bash start-region '' "$hw1host $hw2host $hw3host ..." <span class="math">zkhost "</math>bk1host $bk2host $bk3host ..."
-
-The first three arguments are the same as for @start-hw@.
-
-You may start multiple regions as well:
-
-scripts/hw.bash start-regions regions.cfg
-
-"regions.cfg" is a list of all regions, one per line, with each region having the following format:
-
-region=<Region name>, hub=<list of hub servers>, default=<single hub server>, zk=<single ZK server>, bk=<list of BK servers>
-
-This will create all of the regions with an all-to-all topology. Each region is connected to the default hub server of every other region. The ", " delimiter is used to separate out the different parts of a region along with the hard-coded parameter names. There also needs to be a newline after the last region line. Here is an example file specifying three regions:
-
-region=wilbur, hub=wilbur90 wilbur91, default=wilbur90, zk=wilbur93, bk=wilbur93 wilbur94 wilbur95 region=re1, hub=sherpa7 sherpa8, default=sherpa7, zk=sherpa9, bk=sherpa9 sherpa10 sherpa11 region=peanuts, hub=peanuts1 peanuts2, default=peanuts2, zk=peanuts3, bk=peanuts3 peanuts4 peanuts5
-
-h1. Running the Client
-
-To run the test client:
-
-JAVAFLAGS="..." scripts/hw.bash hwc $conf_path
-
-where @$conf_path@ is a client configuration file.
-
-To run the test client on some other hosts:
-
-hosts="..." JAVAFLAGS="..." scripts/hw.bash app $hwhost
-
-This will generate a simple configuration file assuming $hwhost is listening on the default SSL and non-SSL ports which are specified as global variables in hw.bash. Currently these are 9875 for non-SSL and 9876 for SSL.
-
-Client usage is currently documented in the source. To run a subscription benchmark, set @JAVAFLAGS@ to:
-
--Dmode=sub -Dcount=10000 -Dnpar=100 -Dstart=5 -Dnwarmups=30
-
-This will first create 30 warm-up subscriptions to topics "warmup-5" through "warmup-34", then 10,000 benchmarked subscriptions to topics "topic-5" through "topic-10,004". It will have a pipeline depth of 100 requests, meaning that there will be at most 100 outstanding (unresponded) messages in flight at any moment.
-
-To run a publishing benchmark, set @JAVAFLAGS@ to:
-
--Dmode=pub -Dcount=10000 -Dnpar=100 -Dstart=5
-
-This will publish 10,000 messages to topic "topic-5", with a pipeline depth of 100 requests.
+h2. Limits
-At the end, the programs will print throughput and latency information.
+Because the current implementation uses a single socket per subscription, the Hedwig requires a high @ulimit@ on the number of open file descriptors. Non-root users can only use up to the limit specified in @/etc/security/limits.conf@; to raise this to 1024^2, as root, modify the "nofile" line in /etc/security/limits.conf on all hubs.
-h1. Utilities
+h2. Running Servers
-To kill all the user's Java processes running on some machines, use:
+Hedwig requires BookKeeper to run. For BookKeeper setup instructions see "BookKeeper Getting Started":./bookkeeperStarted.html.
-hosts="..." scripts/hw.bash dkill
+To start a Hedwig hub server:
-To check if any processes are running and are using ports of interest (esp. 9876-9878):
+ @ hedwig-server/bin/hedwig server
-hosts="..." scripts/hw.bash dstatus
+Hedwig takes its configuration from hedwig-server/conf/hw_server.conf by default. To change location of the conf file, modify the HEDWIG_SERVER_CONF environment variable.
-Add an argument to @dstatus@ (may be anything) to get a more detailed listing.
+h1. Debugging
-To check if there's anything consuming the CPU on some machines:
+You can attach an Eclipse debugger (or any debugger) to a Java process running on a remote host, as long as it has been started with the appropriate JVM flags. (See the Building Hedwig document to set up your Eclipse environment.) To launch something using @bin/hedwig@ with debugger attachment enabled, prefix the command with @HEDWIG_EXTRA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,address=5000@, e.g.:
-hosts="..." scripts/hw.bash tops
+@ HEDWIG_EXTRA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,address=5000 hedwig-server/bin/hedwig server
-To run an arbitrary command on multiple hosts in parallel:
+h1. Logging
-hosts="..." scripts/hw.bash parssh $command
+To control the logging in Hedwig server, set HEDWIG_LOG_CONF to the location of a log4j.properties file.
-To do this in sequence:
+@ HEDWIG_LOG_CONF=../hw_testing/log4j.properties hedwig-server/bin/hedwig server
-hosts="..." xargs= scripts/hw.bash parssh $command
Added: zookeeper/bookkeeper/trunk/hedwig-client/conf/hw_client.conf
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-client/conf/hw_client.conf?rev=1163137&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-client/conf/hw_client.conf (added)
+++ zookeeper/bookkeeper/trunk/hedwig-client/conf/hw_client.conf Tue Aug 30 08:34:31 2011
@@ -0,0 +1,7 @@
+# The default Hedwig server host to contact (this ideally should be a VIP
+# that fronts all of the Hedwig server hubs).
+default_server_host=localhost:4080:9876
+# This parameter is a boolean flag indicating if communication with the
+# server should be done via SSL for encryption. The Hedwig server hubs also
+# need to be SSL enabled for this to work.
+ssl_enabled=false
Added: zookeeper/bookkeeper/trunk/hedwig-server/bin/hedwig
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/bin/hedwig?rev=1163137&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/bin/hedwig (added)
+++ zookeeper/bookkeeper/trunk/hedwig-server/bin/hedwig Tue Aug 30 08:34:31 2011
@@ -0,0 +1,115 @@
+#!/bin/sh
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+BINDIR=`dirname "$0"`
+HW_HOME=`cd $BINDIR/..;pwd`
+
+DEFAULT_CONF=$HW_HOME/conf/hw_server.conf
+
+RELEASE_JAR=`ls $HW_HOME/hedwig-server-*.jar 2> /dev/null | tail -1`
+if [ $? == 0 ]; then
+ HEDWIG_JAR=$RELEASE_JAR
+fi
+
+BUILT_JAR=`ls $HW_HOME/target/hedwig-server-*.jar 2> /dev/null | tail -1`
+if [ $? != 0 ] && [ ! -e "$HEDWIG_JAR" ]; then
+ echo "\nCouldn't find hedwig jar.";
+ echo "Make sure you've run 'mvn package'\n";
+ exit 1;
+elif [ -e "$BUILT_JAR" ]; then
+ HEDWIG_JAR=$BUILT_JAR
+fi
+
+add_maven_deps_to_classpath() {
+ MVN="mvn"
+ if [ "$MAVEN_HOME" != "" ]; then
+ MVN=${MAVEN_HOME}/bin/mvn
+ fi
+
+ # Need to generate classpath from maven pom. This is costly so generate it
+ # and cache it. Save the file into our target dir so a mvn clean will get
+ # clean it up and force us create a new one.
+ f="${HW_HOME}/target/cached_classpath.txt"
+ if [ ! -f "${f}" ]
+ then
+ ${MVN} -f "${HW_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null
+ fi
+ HEDWIG_CLASSPATH=${CLASSPATH}:`cat "${f}"`
+}
+
+if [ -d "$HW_HOME/lib" ]; then
+ for i in $HW_HOME/lib/*.jar; do
+ HEDWIG_CLASSPATH=$HEDWIG_CLASSPATH:$i
+ done
+else
+ add_maven_deps_to_classpath
+fi
+
+hedwig_help() {
+ cat <<EOF
+Usage: hedwig <command>
+where command is one of:
+ server Run the hedwig server
+ help This help message
+
+or command is the full name of a class with a defined main() method.
+
+Environment variables:
+ HEDWIG_SERVER_CONF Hedwig server configuration file (default $DEFAULT_CONF)
+ HEDWIG_LOG_CONF Log4j configuration file
+ HEDWIG_EXTRA_OPTS Extra options to be passed to the jvm
+
+These variable can also be set in conf/hwenv.sh
+EOF
+}
+
+# if no args specified, show usage
+if [ $# = 0 ]; then
+ hedwig_help;
+ exit 1;
+fi
+
+# get arguments
+COMMAND=$1
+shift
+
+if [ "$HEDWIG_SERVER_CONF" == "" ]; then
+ HEDWIG_SERVER_CONF=$DEFAULT_CONF;
+fi
+
+HEDWIG_CLASSPATH="$HEDWIG_JAR:$HEDWIG_CLASSPATH"
+
+if [ "$HEDWIG_LOG_CONF" != "" ]; then
+ HEDWIG_CLASSPATH="`dirname $HEDWIG_LOG_CONF`:$HEDWIG_CLASSPATH"
+ OPTS="$OPTS -Dlog4j.configuration=`basename $HEDWIG_LOG_CONF`"
+fi
+OPTS="-cp $HEDWIG_CLASSPATH $OPTS $HEDWIG_EXTRA_OPTS"
+
+if [ $COMMAND == "server" ]; then
+ exec java $OPTS org.apache.hedwig.server.netty.PubSubServer $HEDWIG_SERVER_CONF $@
+elif [ $COMMAND == "help" ]; then
+ hedwig_help;
+else
+ exec java $OPTS $COMMAND $@
+fi
+
+
Added: zookeeper/bookkeeper/trunk/hedwig-server/conf/hw_server.conf
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/conf/hw_server.conf?rev=1163137&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/conf/hw_server.conf (added)
+++ zookeeper/bookkeeper/trunk/hedwig-server/conf/hw_server.conf Tue Aug 30 08:34:31 2011
@@ -0,0 +1,10 @@
+# The ZooKeeper server host(s) for the Hedwig Server to use.
+zk_host=localhost:2181
+# The number of milliseconds of each tick in ZooKeeper.
+zk_timeout=2000
+# The port at which the clients will connect.
+server_port=4080
+# The SSL port at which the clients will connect (only if SSL is enabled).
+ssl_server_port=9876
+# Flag indicating if the server should also operate in SSL mode.
+ssl_enabled=false
Added: zookeeper/bookkeeper/trunk/hedwig-server/conf/hwenv.sh
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/conf/hwenv.sh?rev=1163137&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/conf/hwenv.sh (added)
+++ zookeeper/bookkeeper/trunk/hedwig-server/conf/hwenv.sh Tue Aug 30 08:34:31 2011
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# default settings for starting hedwig
+#HEDWIG_SERVER_CONF=
\ No newline at end of file
Modified: zookeeper/bookkeeper/trunk/hedwig-server/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/pom.xml?rev=1163137&r1=1163136&r2=1163137&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/pom.xml (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/pom.xml Tue Aug 30 08:34:31 2011
@@ -72,28 +72,6 @@
</dependencies>
<build>
<plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- <archive>
- <manifest>
- <mainClass>${mainclass}</mainClass>
- </manifest>
- </archive>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>attached</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>