You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@distributedlog.apache.org by fc...@apache.org on 2016/12/16 00:21:14 UTC
incubator-distributedlog git commit: Clean up and fix a few shell
scripts
Repository: incubator-distributedlog
Updated Branches:
refs/heads/master fd63b34bc -> 927f0ca01
Clean up and fix a few shell scripts
This is still in progress.
- remove trailing white spaces in a few places
- fix some scripts (not rely on git); style consistency
- validate that the vagrant configuration works
- one of the Dockerfile was not correct
Author: Franck Cuny <fc...@apache.org>
Reviewers: Sijie Guo <si...@apache.org>
Closes #55 from franckcuny/fcuny/hack-week-cleanup and squashes the following commits:
fe3965d [Franck Cuny] Fix dlog script for core.
f15cefd [Franck Cuny] fix a few more things in the common script.
c32dee2 [Franck Cuny] Remove useless cd
622d3ab [Franck Cuny] Fix bundle for service.
93e9485 [Franck Cuny] A few more fixes for benchmarks' scripts.
a3fc4ef [Franck Cuny] Remove trailing white spaces in the doc
27f90f1 [Franck Cuny] Rename bookkeeper hosts from node to bk.
21d89f8 [Franck Cuny] Do not assume git is available in the shell scripts.
480bc34 [Franck Cuny] Remove comment from Dockerfile
a7ac7c0 [Franck Cuny] Git ignore pid files
179eddc [Franck Cuny] Ignore .vagrant directory created by vagrant.
9bf33fc [Franck Cuny] Create a valid snapshot.
5800829 [Franck Cuny] Remove Dockerfile and docker directory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/927f0ca0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/927f0ca0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/927f0ca0
Branch: refs/heads/master
Commit: 927f0ca0159a6fad9cf9a5618e313e2f033e598b
Parents: fd63b34
Author: Franck Cuny <fc...@apache.org>
Authored: Thu Dec 15 16:20:57 2016 -0800
Committer: Franck Cuny <fc...@apache.org>
Committed: Thu Dec 15 16:20:57 2016 -0800
----------------------------------------------------------------------
.gitignore | 4 +
Dockerfile | 10 +-
Vagrantfile | 5 +-
distributedlog-benchmark/bin/bundle | 7 +-
distributedlog-benchmark/bin/dbench | 15 ++-
distributedlog-benchmark/conf/dlogenv.sh | 2 +-
distributedlog-core/bin/dlog | 9 +-
distributedlog-service/bin/bundle | 7 +-
distributedlog-service/bin/dlog | 10 +-
distributedlog-service/bin/dlog-daemon.sh | 170 ++++++++++++-------------
docker/Dockerfile | 35 -----
docs/admin_guide/vagrant.rst | 4 +-
docs/deployment/cluster.rst | 58 +++++----
docs/deployment/docker.rst | 2 +-
scripts/bundle | 2 -
scripts/common.sh | 6 +-
scripts/runner | 3 +-
scripts/snapshot | 11 +-
vagrant/base.sh | 43 ++++---
vagrant/bk.sh | 64 +++++-----
vagrant/zk.sh | 5 +-
21 files changed, 222 insertions(+), 250 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 3b719cd..914de41 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,7 @@ lib/
*.#*
dist/
logs/
+*.pid
+
+# vagrant
+.vagrant
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/Dockerfile
----------------------------------------------------------------------
diff --git a/Dockerfile b/Dockerfile
index d0c90ae..f45e355 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,13 +24,9 @@ COPY . /opt/distributedlog-trunk/
ENV BROKER_ID 0
ENV ZK_SERVERS zk1:2181
-EXPOSE 3181
-EXPOSE 9001
-EXPOSE 4181
+EXPOSE 3181
+EXPOSE 9001
+EXPOSE 4181
EXPOSE 20001
-#CMD ["/bin/bash", "-c", "/opt/distributedlog-trunk/dist/release/vagrant/bk.sh", "$BROKER_ID", "127.0.0.1", "$ZK_SERVERS" ]
ENTRYPOINT [ "/opt/distributedlog-trunk/dist/release/vagrant/bk_docker_wrapper.sh" ]
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/Vagrantfile
----------------------------------------------------------------------
diff --git a/Vagrantfile b/Vagrantfile
index 1aa733e..ca46785 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -34,7 +34,6 @@ if File.exists?(local_config_file) then
eval(File.read(local_config_file), binding, "Vagrantfile.local")
end
-# TODO(ksweeney): RAM requirements are not empirical and can probably be significantly lowered.
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.hostmanager.enabled = enable_hostmanager
config.hostmanager.manage_host = enable_dns
@@ -92,7 +91,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
}
(1..num_hosts).each { |i|
- name = "node" + i.to_s
+ name = "bk" + i.to_s
config.vm.define name do |broker|
name_node(broker, name)
ip_address = "192.168.50." + (50 + i).to_s
@@ -106,6 +105,4 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
broker.vm.provision "shell", path: "vagrant/bk.sh", :args => [i.to_s, enable_dns ? name : ip_address, zookeeper_connect]
end
}
-
-
end
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-benchmark/bin/bundle
----------------------------------------------------------------------
diff --git a/distributedlog-benchmark/bin/bundle b/distributedlog-benchmark/bin/bundle
index 838f74d..c0fdcd8 100755
--- a/distributedlog-benchmark/bin/bundle
+++ b/distributedlog-benchmark/bin/bundle
@@ -20,6 +20,9 @@
set -e
-DLOG_ROOT=$(git rev-parse --show-toplevel)
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/../.."
-bash "${DLOG_ROOT}"/scripts/bundle benchmark
+cd "${DLOG_ROOT}"
+
+bash scripts/bundle benchmark
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-benchmark/bin/dbench
----------------------------------------------------------------------
diff --git a/distributedlog-benchmark/bin/dbench b/distributedlog-benchmark/bin/dbench
index 42dd5b0..ef58477 100755
--- a/distributedlog-benchmark/bin/dbench
+++ b/distributedlog-benchmark/bin/dbench
@@ -20,7 +20,8 @@
set -e
-DLOG_ROOT=$(git rev-parse --show-toplevel)
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/../.."
DLOG_HOME="${DLOG_ROOT}/distributedlog-benchmark"
@@ -43,8 +44,7 @@ These variable can also be set in conf/dlogenv.sh
EOF
}
-cd "${DLOG_ROOT}"
-source ./scripts/common.sh
+source ${DLOG_ROOT}/scripts/common.sh
# get arguments
COMMAND=$1
@@ -64,7 +64,7 @@ BENCH_ARGS="""
"""
#Change to DLOG_HOME to support relative paths
-cd "$DLOG_HOME"
+#cd "$DLOG_HOME"
case "${COMMAND}" in
bkwrite)
@@ -73,7 +73,7 @@ case "${COMMAND}" in
--rate ${INITIAL_RATE} \\
--max-rate ${MAX_RATE} \\
--change-rate ${CHANGE_RATE} \\
- --change-interval ${CHANGE_RATE_INTERVAL}
+ --change-interval ${CHANGE_RATE_INTERVAL} \\
"""
BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode dlwrite \\"
exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@
@@ -85,6 +85,7 @@ case "${COMMAND}" in
--max-rate ${MAX_RATE} \\
--change-rate ${CHANGE_RATE} \\
--change-interval ${CHANGE_RATE_INTERVAL} \\
+ --finagle-name inet!localhost:8000 \\
"""
BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode write \\"
exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@
@@ -94,6 +95,7 @@ case "${COMMAND}" in
--readers-per-stream ${NUM_READERS_PER_STREAM} \\
--max-stream-id ${MAX_STREAM_ID} \\
--truncation-interval ${TRUNCATION_INTERVAL} \\
+ --finagle-name inet!localhost:8000 \\
"""
BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_READ_ARGS} \\ --mode read \\"
exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@
@@ -102,7 +104,8 @@ case "${COMMAND}" in
usage
;;
*)
- exec java $OPTS $COMMAND $@
+ set -x
+ echo java $OPTS $COMMAND $@
;;
esac
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-benchmark/conf/dlogenv.sh
----------------------------------------------------------------------
diff --git a/distributedlog-benchmark/conf/dlogenv.sh b/distributedlog-benchmark/conf/dlogenv.sh
index 205606b..11abe9f 100644
--- a/distributedlog-benchmark/conf/dlogenv.sh
+++ b/distributedlog-benchmark/conf/dlogenv.sh
@@ -44,7 +44,7 @@ STREAM_NAME_PREFIX=distributedlog-smoketest
# Benchmark Run Duration in minutes
BENCHMARK_DURATION=60
# DL Namespace
-DL_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace
+DL_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/my_namespace
# Benchmark SHARD id
BENCHMARK_SHARD_ID=0
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-core/bin/dlog
----------------------------------------------------------------------
diff --git a/distributedlog-core/bin/dlog b/distributedlog-core/bin/dlog
index c4f9ed2..522e47c 100755
--- a/distributedlog-core/bin/dlog
+++ b/distributedlog-core/bin/dlog
@@ -20,8 +20,10 @@
set -e
-DLOG_ROOT=$(git rev-parse --show-toplevel)
-DLOG_HOME="${DLOG_ROOT}"
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/../.."
+
+DLOG_HOME="${DLOG_ROOT}/distributedlog-core"
usage() {
cat <<EOF
@@ -46,8 +48,7 @@ These variable can also be set in conf/dlogenv.sh
EOF
}
-cd "${DLOG_ROOT}"
-source ./scripts/common.sh
+source "${DLOG_ROOT}"/scripts/common.sh
# get arguments
COMMAND=$1
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-service/bin/bundle
----------------------------------------------------------------------
diff --git a/distributedlog-service/bin/bundle b/distributedlog-service/bin/bundle
index bd5b1a0..1b1904e 100755
--- a/distributedlog-service/bin/bundle
+++ b/distributedlog-service/bin/bundle
@@ -20,6 +20,9 @@
set -e
-DLOG_ROOT=$(git rev-parse --show-toplevel)
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/../.."
-bash "${DLOG_ROOT}"/scripts/bundle service
+cd "${DLOG_ROOT}"
+
+bash scripts/bundle service
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-service/bin/dlog
----------------------------------------------------------------------
diff --git a/distributedlog-service/bin/dlog b/distributedlog-service/bin/dlog
index b9a01e1..5a11611 100755
--- a/distributedlog-service/bin/dlog
+++ b/distributedlog-service/bin/dlog
@@ -20,8 +20,8 @@
set -e
-DLOG_ROOT=$(git rev-parse --show-toplevel)
-
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/.."
DLOG_HOME="${DLOG_ROOT}"
usage() {
@@ -49,16 +49,12 @@ These variable can also be set in conf/dlogenv.sh
EOF
}
-cd "${DLOG_ROOT}"
-source ./scripts/common.sh
+source "${DLOG_ROOT}"/../scripts/common.sh
# get arguments
COMMAND=$1
shift
-#Change to DLOG_HOME to support relative paths
-cd "$DLOG_HOME"
-
case "${COMMAND}" in
local)
exec java $OPTS $JMX_ARGS com.twitter.distributedlog.LocalDLMEmulator $@
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-service/bin/dlog-daemon.sh
----------------------------------------------------------------------
diff --git a/distributedlog-service/bin/dlog-daemon.sh b/distributedlog-service/bin/dlog-daemon.sh
index e683da0..976d4f1 100755
--- a/distributedlog-service/bin/dlog-daemon.sh
+++ b/distributedlog-service/bin/dlog-daemon.sh
@@ -19,26 +19,25 @@
# */
usage() {
- cat <<EOF
+ cat <<EOF
Usage: dlog-daemon.sh (start|stop) <service> <args...>
where service is one of:
- zookeeper Run the zookeeper server
- bookie Run the bookie server
- bookie-rereplicator Run the bookie rereplicator
- writeproxy Run the write proxy server
- writeproxy-monitor Run the write proxy monitor
+ zookeeper Run the zookeeper server
+ bookie Run the bookie server
+ bookie-rereplicator Run the bookie rereplicator
+ writeproxy Run the write proxy server
+ writeproxy-monitor Run the write proxy monitor
where argument is one of:
- -force (accepted only with stop service): Decides whether to stop the process forcefully if not stopped by normal shutdown
+ -force (accepted only with stop service): Decides whether to stop the process forcefully if not stopped by normal shutdown
EOF
}
-BINDIR=`dirname "$0"`
-DL_HOME=`cd $BINDIR/..;pwd`
+BINDIR=$(dirname "$0")
+DL_HOME=$(cd $BINDIR/..;pwd)
-if [ -f $DL_HOME/conf/dlogenv.sh ]
-then
- . $DL_HOME/conf/dlogenv.sh
+if [ -f "${DL_HOME}/conf/dlogenv.sh" ]; then
+ . "${DL_HOME}/conf/dlogenv.sh"
fi
SERVICE_PORT=${SERVICE_PORT:-"0"}
@@ -47,56 +46,57 @@ SERVICE_ARGS=""
# DLOG logging configuration
DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DL_HOME/logs"}
DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-'INFO,R'}
+
# Process Control Parameters
DLOG_STOP_TIMEOUT=${DLOG_STOP_TIMEOUT:-30}
DLOG_PID_DIR=${DLOG_PID_DIR:-$DL_HOME/pids}
-if [ $# -lt 2 ]
-then
- echo "Error: no enough arguments provided."
- usage
- exit 1
+if [ $# -lt 2 ]; then
+ echo "Error: not enough arguments provided."
+ usage
+ exit 1
fi
command=$1
shift
+
service=$1
shift
service_class=$service
case $service in
- (zookeeper)
- service_class="org.apache.zookeeper.server.quorum.QuorumPeerMain"
- DLOG_ROOT_LOGGER=${ZK_ROOT_LOGGER:-'INFO,R'}
- ;;
- (bookie)
- service_class="org.apache.bookkeeper.proto.BookieServer"
- DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'}
- ;;
- (bookie-rereplicator)
- service_class="org.apache.bookkeeper.replication.AutoRecoveryMain"
- DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'}
- ;;
- (writeproxy)
- service_class="com.twitter.distributedlog.service.DistributedLogServerApp"
- DLOG_ROOT_LOGGER=${WP_ROOT_LOGGER:-'INFO,R'}
- WP_CONF_FILE=${WP_CONF_FILE:-"$DL_HOME/conf/write_proxy.conf"}
- WP_SERVICE_PORT=${WP_SERVICE_PORT:-'4181'}
- WP_STATS_PORT=${WP_STATS_PORT:-'9000'}
- WP_STATS_PROVIDER=${WP_STATS_PROVIDER:-'org.apache.bookkeeper.stats.CodahaleMetricsServletProvider'}
- WP_SHARD_ID=${WP_SHARD_ID:-'0'}
- WP_NAMESPACE=${WP_NAMESPACE:-'distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace'}
- SERVICE_PORT=${WP_SERVICE_PORT}
- SERVICE_ARGS="--conf ${WP_CONF_FILE} --uri ${WP_NAMESPACE} --shard-id ${WP_SHARD_ID} --port ${WP_SERVICE_PORT} --stats-port ${WP_STATS_PORT} --stats-provider ${WP_STATS_PROVIDER} --announce --thriftmux"
- DLOG_EXTRA_OPTS="${DLOG_EXTRA_OPTS} -DcodahaleStatsHttpPort=${WP_STATS_PORT} -Dserver_port=${WP_SERVICE_PORT} -Dserver_shard=${WP_SHARD_ID}"
- ;;
- (writeproxy-monitor)
- ;;
- (*)
- echo "Error: unknown service name $service"
- usage
- exit 1
- ;;
+ (zookeeper)
+ service_class="org.apache.zookeeper.server.quorum.QuorumPeerMain"
+ DLOG_ROOT_LOGGER=${ZK_ROOT_LOGGER:-'INFO,R'}
+ ;;
+ (bookie)
+ service_class="org.apache.bookkeeper.proto.BookieServer"
+ DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'}
+ ;;
+ (bookie-rereplicator)
+ service_class="org.apache.bookkeeper.replication.AutoRecoveryMain"
+ DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'}
+ ;;
+ (writeproxy)
+ service_class="com.twitter.distributedlog.service.DistributedLogServerApp"
+ DLOG_ROOT_LOGGER=${WP_ROOT_LOGGER:-'INFO,R'}
+ WP_CONF_FILE=${WP_CONF_FILE:-"$DL_HOME/conf/write_proxy.conf"}
+ WP_SERVICE_PORT=${WP_SERVICE_PORT:-'4181'}
+ WP_STATS_PORT=${WP_STATS_PORT:-'9000'}
+ WP_STATS_PROVIDER=${WP_STATS_PROVIDER:-'org.apache.bookkeeper.stats.CodahaleMetricsServletProvider'}
+ WP_SHARD_ID=${WP_SHARD_ID:-'0'}
+ WP_NAMESPACE=${WP_NAMESPACE:-'distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace'}
+ SERVICE_PORT=${WP_SERVICE_PORT}
+ SERVICE_ARGS="--conf ${WP_CONF_FILE} --uri ${WP_NAMESPACE} --shard-id ${WP_SHARD_ID} --port ${WP_SERVICE_PORT} --stats-port ${WP_STATS_PORT} --stats-provider ${WP_STATS_PROVIDER} --announce --thriftmux"
+ DLOG_EXTRA_OPTS="${DLOG_EXTRA_OPTS} -DcodahaleStatsHttpPort=${WP_STATS_PORT} -Dserver_port=${WP_SERVICE_PORT} -Dserver_shard=${WP_SHARD_ID}"
+ ;;
+ (writeproxy-monitor)
+ ;;
+ (*)
+ echo "Error: unknown service name $service"
+ usage
+ exit 1
+ ;;
esac
echo "doing $command $service ..."
@@ -110,21 +110,20 @@ pid=$DLOG_PID_DIR/dlog-$service-$SERVICE_PORT.pid
out=$DLOG_LOG_DIR/dlog-$service-$HOSTNAME-$SERVICE_PORT.out
logfile=$DLOG_LOG_DIR/$DLOG_LOG_FILE
-rotate_out_log ()
-{
- log=$1;
- num=5;
- if [ -n "$2" ]; then
- num=$2
- fi
- if [ -f "$log" ]; then # rotate logs
- while [ $num -gt 1 ]; do
- prev=`expr $num - 1`
- [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
- num=$prev
- done
- mv "$log" "$log.$num";
- fi
+rotate_out_log () {
+ log=$1;
+ num=5;
+ if [ -n "$2" ]; then
+ num=$2
+ fi
+ if [ -f "$log" ]; then # rotate logs
+ while [ $num -gt 1 ]; do
+ prev=`expr $num - 1`
+ [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
+ num=$prev
+ done
+ mv "$log" "$log.$num";
+ fi
}
mkdir -p "$DLOG_LOG_DIR"
@@ -133,8 +132,8 @@ mkdir -p "$DLOG_PID_DIR"
case $command in
(start)
if [ -f $pid ]; then
- if kill -0 `cat $pid` > /dev/null 2>&1; then
- echo $service running as process `cat $pid`. Stop it first.
+ if kill -0 $(cat $pid) > /dev/null 2>&1; then
+ echo $service running as process $(cat $pid). Stop it first.
exit 1
fi
fi
@@ -153,41 +152,36 @@ case $command in
(stop)
if [ -f $pid ]; then
- TARGET_PID=`cat $pid`
+ TARGET_PID=$(cat $pid)
if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo stopping $service
kill $TARGET_PID
count=0
location=$DLOG_LOG_DIR
- while ps -p $TARGET_PID > /dev/null;
- do
+ while ps -p $TARGET_PID > /dev/null; do
echo "Shutdown is in progress... Please wait..."
sleep 1
- count=`expr $count + 1`
-
+ count=$(expr $count + 1)
if [ "$count" = "$DLOG_STOP_TIMEOUT" ]; then
- break
+ break
fi
- done
-
+ done
if [ "$count" != "$DLOG_STOP_TIMEOUT" ]; then
- echo "Shutdown completed."
+ echo "Shutdown completed."
fi
-
if kill -0 $TARGET_PID > /dev/null 2>&1; then
- fileName=$location/$service.out
- $JAVA_HOME/bin/jstack $TARGET_PID > $fileName
- echo Thread dumps are taken for analysis at $fileName
- if [ "$1" == "-force" ]
- then
- echo forcefully stopping $service
- kill -9 $TARGET_PID >/dev/null 2>&1
- echo Successfully stopped the process
- else
- echo "WARNNING : $service is not stopped completely."
- exit 1
- fi
+ fileName=$location/$service.out
+ $JAVA_HOME/bin/jstack $TARGET_PID > $fileName
+ echo Thread dumps are taken for analysis at $fileName
+ if [ "$1" == "-force" ]; then
+ echo forcefully stopping $service
+ kill -9 $TARGET_PID >/dev/null 2>&1
+ echo Successfully stopped the process
+ else
+ echo "WARNNING : $service is not stopped completely."
+ exit 1
+ fi
fi
else
echo no $service to stop
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/Dockerfile b/docker/Dockerfile
deleted file mode 100644
index f517914..0000000
--- a/docker/Dockerfile
+++ /dev/null
@@ -1,35 +0,0 @@
-#/**
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-FROM java:8
-
-MAINTAINER Arvind Kandhare (arvind.kandhare@emc.com)
-
-COPY .. /opt/distributedlog-trunk/
-
-ENV BROKER_ID 0
-ENV ZK_SERVERS zk1:2181
-
-EXPOSE 3181
-EXPOSE 9001
-EXPOSE 4181
-EXPOSE 20001
-
-CMD ["/opt/distributedlog-trunk/dist/release/vagrant/bk.sh", $BROKER_ID, "127.0.0.1", $ZK_SERVERS ]
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docs/admin_guide/vagrant.rst
----------------------------------------------------------------------
diff --git a/docs/admin_guide/vagrant.rst b/docs/admin_guide/vagrant.rst
index 70e6fb0..e4ed574 100644
--- a/docs/admin_guide/vagrant.rst
+++ b/docs/admin_guide/vagrant.rst
@@ -13,6 +13,6 @@ Steps
1. Create a snapshot using ./scripts/snapshot
2. Run vagrant up from the root directory of the enlistment
3. Vagrant brings up a zookeepers with machine names: zk1,zk2, .... with IP addresses 192.168.50.11,192.168.50.12,....
-4. Vagrant brings the bookies with machine names: node1,node2, ... with IP addresses 192.168.50.51,192.168.50.52,....
-5. The script will also start writeproxies at distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace as the namespace. If you want it to point to a different namespace/port/shard, please update the ./vagrant/bk.sh script.
+4. Vagrant brings the bookies with machine names: bk1,bk2, ... with IP addresses 192.168.50.51,192.168.50.52,....
+5. The script will also start writeproxies at distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace as the namespace. If you want it to point to a different namespace/port/shard, please update the ./vagrant/bk.sh script.
6. If you want to run the client on the host machine, please add these node names and their IP addresses to the /etc/hosts file on the host.
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docs/deployment/cluster.rst
----------------------------------------------------------------------
diff --git a/docs/deployment/cluster.rst b/docs/deployment/cluster.rst
index 8dcd461..250ba3c 100644
--- a/docs/deployment/cluster.rst
+++ b/docs/deployment/cluster.rst
@@ -38,8 +38,9 @@ Or run `./scripts/snapshot` to build the release packages from current source. T
packages contain the binaries for running `distributedlog-service`, `distributedlog-benchmark`
and `distributedlog-tutorials`.
-NOTE: we run following instructions from distributedlog source code after running `mvn clean install`.
-And assume `DL_HOME` is the directory of distributedlog source.
+NOTE: we run the following instructions from distributedlog source code after
+running `mvn clean install`. And assume `DL_HOME` is the directory of
+distributedlog source.
Zookeeper
---------
@@ -118,9 +119,10 @@ First of all, choose the zookeeper cluster that the bookies will use and set `zk
the configuration files.
::
-
+
zkServers=localhost:2181
+
Choose the zookeeper path to store bookkeeper metadata and set `zkLedgersRootPath` in the configuration
files. Let's use `/messaging/bookkeeper/ledgers` in this instruction.
@@ -159,7 +161,7 @@ So using `zkshell` to create the `zkLedgersRootPath`.
If the `zkLedgersRootPath`, run `metaformat` to format the bookkeeper metadata.
::
-
+
$ BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-1.conf ./distributedlog-service/bin/dlog bkshell metaformat
Are you sure to format bookkeeper metadata ? (Y or N) Y
@@ -176,7 +178,7 @@ Configure the ports that used by bookies.
bookie-1:
::
-
+
# Port that bookie server listen on
bookiePort=3181
# Exporting codahale stats
@@ -185,7 +187,7 @@ bookie-1:
bookie-2:
::
-
+
# Port that bookie server listen on
bookiePort=3182
# Exporting codahale stats
@@ -194,7 +196,7 @@ bookie-2:
bookie-3:
::
-
+
# Port that bookie server listen on
bookiePort=3183
# Exporting codahale stats
@@ -206,7 +208,7 @@ Configure Disk Layout
Configure the disk directories used by a bookie server by setting following options.
::
-
+
# Directory Bookkeeper outputs its write ahead log
journalDirectory=/tmp/data/bk/journal
# Directory Bookkeeper outputs ledger snapshots
@@ -219,7 +221,7 @@ As we are configuring a 3-nodes bookkeeper cluster, we modify the following sett
bookie-1:
::
-
+
# Directory Bookkeeper outputs its write ahead log
journalDirectory=/tmp/data/bk-1/journal
# Directory Bookkeeper outputs ledger snapshots
@@ -230,7 +232,7 @@ bookie-1:
bookie-2:
::
-
+
# Directory Bookkeeper outputs its write ahead log
journalDirectory=/tmp/data/bk-2/journal
# Directory Bookkeeper outputs ledger snapshots
@@ -241,7 +243,7 @@ bookie-2:
bookie-3:
::
-
+
# Directory Bookkeeper outputs its write ahead log
journalDirectory=/tmp/data/bk-3/journal
# Directory Bookkeeper outputs ledger snapshots
@@ -256,7 +258,7 @@ Once the disk directories are configured correctly in the configuration file, us
`bkshell bookieformat` to format the bookie.
::
-
+
BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-1.conf ./distributedlog-service/bin/dlog bkshell bookieformat
BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-2.conf ./distributedlog-service/bin/dlog bkshell bookieformat
BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-3.conf ./distributedlog-service/bin/dlog bkshell bookieformat
@@ -268,16 +270,16 @@ Start bookie
Start the bookie using `dlog-daemon.sh`.
::
-
+
SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-1.conf
SERVICE_PORT=3182 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-2.conf
SERVICE_PORT=3183 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-3.conf
-
+
Verify whether the bookie is setup correctly. You could simply check whether the bookie is showed up in
zookeeper `zkLedgersRootPath`/available znode.
::
-
+
$ ./distributedlog-service/bin/dlog zkshell localhost:2181
Connecting to localhost:2181
Welcome to ZooKeeper!
@@ -294,7 +296,7 @@ zookeeper `zkLedgersRootPath`/available znode.
Or check if the bookie is exposing the stats at port `codahaleStatsHttpPort`.
::
-
+
// ping the service
$ curl localhost:9001/ping
pong
@@ -307,7 +309,7 @@ Stop bookie
Stop the bookie using `dlog-daemon.sh`.
::
-
+
$ ./distributedlog-service/bin/dlog-daemon.sh stop bookie
// Example:
$ SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh stop bookie
@@ -322,13 +324,13 @@ Turn bookie to readonly
Start the bookie in `readonly` mode.
::
-
+
$ SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-1.conf --readonly
Verify if the bookie is running in `readonly` mode.
::
-
+
$ ./distributedlog-service/bin/dlog zkshell localhost:2181
Connecting to localhost:2181
Welcome to ZooKeeper!
@@ -376,7 +378,7 @@ Create namespace binding using `dlog tool`. For example, we create a namespace
bookkeeper cluster we just created above.
::
-
+
$ distributedlog-service/bin/dlog admin bind \\
-dlzr 127.0.0.1:2181 \\
-dlzw 127.0.0.1:2181 \\
@@ -425,7 +427,7 @@ Run write proxy
A write proxy could be started using `dlog-daemon.sh` script under `distributedlog-service`.
::
-
+
WP_SHARD_ID=${WP_SHARD_ID} WP_SERVICE_PORT=${WP_SERVICE_PORT} WP_STATS_PORT=${WP_STATS_PORT} ./distributedlog-service/bin/dlog-daemon.sh start writeproxy
- `WP_SHARD_ID`: A non-negative integer. You don't need to guarantee uniqueness of shard id, as it is just an
@@ -442,7 +444,7 @@ Please check `distributedlog-service/conf/dlogenv.sh` for more environment varia
For example, we start 3 write proxies locally and point to the namespace created above.
::
-
+
$ WP_SHARD_ID=1 WP_SERVICE_PORT=4181 WP_STATS_PORT=20001 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy
$ WP_SHARD_ID=2 WP_SERVICE_PORT=4182 WP_STATS_PORT=20002 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy
$ WP_SHARD_ID=3 WP_SERVICE_PORT=4183 WP_STATS_PORT=20003 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy
@@ -452,7 +454,7 @@ The write proxy will announce itself to the zookeeper path `.write_proxy` under
We could verify that the write proxy is running correctly by checking the zookeeper path or checking its stats port.
::
-
+
$ ./distributedlog-service/bin/dlog zkshell localhost:2181
Connecting to localhost:2181
Welcome to ZooKeeper!
@@ -466,7 +468,7 @@ We could verify that the write proxy is running correctly by checking the zookee
::
-
+
$ curl localhost:20001/ping
pong
@@ -477,7 +479,7 @@ Add and Remove Write Proxies
Removing a write proxy is pretty straightforward by just killing the process.
::
-
+
WP_SHARD_ID=1 WP_SERVICE_PORT=4181 WP_STATS_PORT=10001 ./distributedlog-service/bin/dlog-daemon.sh stop writeproxy
@@ -506,14 +508,14 @@ Create 10 streams.
Tail read from the 10 streams.
::
-
+
$ ./distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.MultiReader distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace stream-0,stream-1,stream-2,stream-3,stream-4,stream-5,stream-6,stream-7,stream-8,stream-9,stream-10
Run record generator over some streams
::
-
+
$ ./distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.RecordGenerator 'zk!127.0.0.1:2181!/messaging/distributedlog/mynamespace/.write_proxy' stream-0 100
$ ./distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.RecordGenerator 'zk!127.0.0.1:2181!/messaging/distributedlog/mynamespace/.write_proxy' stream-1 100
@@ -521,7 +523,7 @@ Run record generator over some streams
Check the terminal running `MultiReader`. You will see similar output as below:
::
-
+
"""
Received record DLSN{logSegmentSequenceNo=1, entryId=21044, slotId=0} from stream stream-0
"""
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docs/deployment/docker.rst
----------------------------------------------------------------------
diff --git a/docs/deployment/docker.rst b/docs/deployment/docker.rst
index 8222dd6..d9fd87a 100644
--- a/docs/deployment/docker.rst
+++ b/docs/deployment/docker.rst
@@ -24,7 +24,7 @@ Steps
./scripts/snapshot
-2. Create your own docker image using
+2. Create your own docker image using
.. code-block:: bash
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/bundle
----------------------------------------------------------------------
diff --git a/scripts/bundle b/scripts/bundle
index a0c8a3a..54691ec 100755
--- a/scripts/bundle
+++ b/scripts/bundle
@@ -27,8 +27,6 @@ if [[ -z "${COMPONENT}" ]]; then
exit 1
fi
-cd "${DLOG_ROOT}"
-
rm -rf "distributedlog-${COMPONENT}/lib"
rm -rf "dist/distributedlog-${COMPONENT}*"
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/common.sh
----------------------------------------------------------------------
diff --git a/scripts/common.sh b/scripts/common.sh
index 433f93b..f37ed1b 100755
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -48,6 +48,7 @@ else
echo "JMX disabled by user request" >&2
fi
+echo "DLOG_HOME => ${DLOG_HOME}"
DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties"
[ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh"
@@ -59,7 +60,7 @@ if [ $? == 0 ]; then
fi
# exclude tests jar
-BUILT_JAR=$(ls "${DLOG_HOME}/target/distributedlog-*.jar" 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1)
+BUILT_JAR=$(ls "${DLOG_HOME}"/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1)
if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then
echo "\nCouldn't find dlog jar.";
@@ -93,9 +94,8 @@ else
add_maven_deps_to_classpath
fi
-# if no args specified, show usage
+# if no args specified, exit
if [ $# = 0 ]; then
- usage
exit 1
fi
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/runner
----------------------------------------------------------------------
diff --git a/scripts/runner b/scripts/runner
index cc05946..937829a 100755
--- a/scripts/runner
+++ b/scripts/runner
@@ -22,7 +22,8 @@
set -e
-DLOG_ROOT=$(git rev-parse --show-toplevel)
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/.."
DLOG_HOME="${DLOG_ROOT}/distributedlog-service"
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/snapshot
----------------------------------------------------------------------
diff --git a/scripts/snapshot b/scripts/snapshot
index 2c9e285..2f2f732 100755
--- a/scripts/snapshot
+++ b/scripts/snapshot
@@ -32,7 +32,7 @@ rm -rf dist/*
# create the layout
for module in distributedlog-service distributedlog-benchmark distributedlog-tutorials/distributedlog-basic distributedlog-tutorials/distributedlog-messaging; do
- mkdir -p "dist/release/${module}/{bin,conf,lib}"
+ mkdir -p "dist/release/${module}"/{bin,conf,lib}
done
# build the package
@@ -42,26 +42,27 @@ mvn clean install -Ptwitter-ostrich-provider -DskipTests
cp -rf distributedlog-service/bin/* dist/release/distributedlog-service/bin
cp -rf distributedlog-service/conf/* dist/release/distributedlog-service/conf
cp -rf distributedlog-service/lib/* dist/release/distributedlog-service/lib
-cp -rf distributedlog-service/target/distributedlog-service-*.jar dist/release/distributedlog-service
+cp -rf distributedlog-service/target/distributedlog-service-*.jar dist/release/distributedlog-service/lib
cp -rf vagrant dist/release/
+cp -rf scripts dist/release/
# build the distributedlog-benchmark
cp -rf distributedlog-benchmark/bin/* dist/release/distributedlog-benchmark/bin
cp -rf distributedlog-benchmark/conf/* dist/release/distributedlog-benchmark/conf
cp -rf distributedlog-benchmark/lib/* dist/release/distributedlog-benchmark/lib
-cp -rf distributedlog-benchmark/target/distributedlog-benchmark-*.jar dist/release/distributedlog-benchmark
+cp -rf distributedlog-benchmark/target/distributedlog-benchmark-*.jar dist/release/distributedlog-benchmark/lib
# build the distributedlog-tutorial/basic
cp -rf distributedlog-tutorials/distributedlog-basic/bin/* dist/release/distributedlog-tutorials/distributedlog-basic/bin
cp -rf distributedlog-tutorials/distributedlog-basic/conf/* dist/release/distributedlog-tutorials/distributedlog-basic/conf
cp -rf distributedlog-tutorials/distributedlog-basic/lib/* dist/release/distributedlog-tutorials/distributedlog-basic/lib
-cp -rf distributedlog-tutorials/distributedlog-basic/target/distributedlog-basic-*.jar dist/release/distributedlog-tutorials/distributedlog-basic
+cp -rf distributedlog-tutorials/distributedlog-basic/target/distributedlog-basic-*.jar dist/release/distributedlog-tutorials/distributedlog-basic/lib
# build the distributedlog-tutorial/messaging
cp -rf distributedlog-tutorials/distributedlog-messaging/bin/* dist/release/distributedlog-tutorials/distributedlog-messaging/bin
cp -rf distributedlog-tutorials/distributedlog-messaging/conf/* dist/release/distributedlog-tutorials/distributedlog-messaging/conf
cp -rf distributedlog-tutorials/distributedlog-messaging/lib/* dist/release/distributedlog-tutorials/distributedlog-messaging/lib
-cp -rf distributedlog-tutorials/distributedlog-messaging/target/distributedlog-messaging-*.jar dist/release/distributedlog-tutorials/distributedlog-messaging
+cp -rf distributedlog-tutorials/distributedlog-messaging/target/distributedlog-messaging-*.jar dist/release/distributedlog-tutorials/distributedlog-messaging/lib
# release
cd dist/release
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/vagrant/base.sh
----------------------------------------------------------------------
diff --git a/vagrant/base.sh b/vagrant/base.sh
index 03e348e..21e6167 100755
--- a/vagrant/base.sh
+++ b/vagrant/base.sh
@@ -18,39 +18,40 @@
set -e
if [ -z $(which javac) ]; then
- apt-get -y update
- apt-get install -y --no-install-recommends software-properties-common python-software-properties
- add-apt-repository -y ppa:webupd8team/java
- apt-get -y update
-
- # Try to share cache. See Vagrantfile for details
- mkdir -p /var/cache/oracle-jdk8-installer
- if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then
- find /tmp/oracle-jdk8-installer-cache/ -not -empty -exec cp '{}' /var/cache/oracle-jdk8-installer/ \;
- fi
-
- /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
- apt-get -y install --force-yes --no-install-recommends oracle-java8-installer oracle-java8-set-default
-
- if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then
- cp -R /var/cache/oracle-jdk8-installer/* /tmp/oracle-jdk8-installer-cache
- fi
+ apt-get -y update
+ apt-get install -y --no-install-recommends software-properties-common python-software-properties
+ add-apt-repository -y ppa:webupd8team/java
+ apt-get -y update
+
+ # Try to share cache. See Vagrantfile for details
+ mkdir -p /var/cache/oracle-jdk8-installer
+ if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then
+ find /tmp/oracle-jdk8-installer-cache/ -not -empty -exec cp '{}' /var/cache/oracle-jdk8-installer/ \;
+ fi
+
+ /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
+ apt-get -y install --force-yes --no-install-recommends oracle-java8-installer oracle-java8-set-default maven2
+
+ if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then
+ cp -R /var/cache/oracle-jdk8-installer/* /tmp/oracle-jdk8-installer-cache
+ fi
fi
chmod a+rw /opt
if [ -h /opt/distributedlog-trunk ]; then
- # reset symlink
- rm /opt/distributedlog-trunk
+ # reset symlink
+ rm /opt/distributedlog-trunk
fi
-ln -s /vagrant /opt/distributedlog-trunk
+ln -s /vagrant /opt/distributedlog-trunk
# For EC2 nodes, we want to use /mnt, which should have the local disk. On local
# VMs, we can just create it if it doesn't exist and use it like we'd use
# /tmp. Eventually, we'd like to also support more directories, e.g. when EC2
# instances have multiple local disks.
if [ ! -e /mnt ]; then
- mkdir /mnt
+ mkdir /mnt
fi
+
chmod a+rwx /mnt
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/vagrant/bk.sh
----------------------------------------------------------------------
diff --git a/vagrant/bk.sh b/vagrant/bk.sh
index 3c0009a..1eb5ab1 100755
--- a/vagrant/bk.sh
+++ b/vagrant/bk.sh
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
# Usage: bk.sh <broker ID> <public hostname or IP> <list zookeeper public hostname or IP + port>
set -e
@@ -40,47 +39,54 @@ if [ ! -z $DEPLOY_BOTH ]; then
DEPLOY_WP=true
fi
-
log_dir=/opt/distributedlog-trunk/dist/release
cd $log_dir
#Stop the running bookie
-echo "Killing server"
+echo "Killing bookkeeper server"
./distributedlog-service/bin/dlog-daemon.sh stop bookie
cp ./distributedlog-service/conf/bookie.conf.template ./distributedlog-service/conf/bookie-$BROKER_ID.conf
sed \
- -e 's/zkServers=localhost:2181/'zkServers=$PUBLIC_ZOOKEEPER_ADDRESSES'/' \
- ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/bookie-$BROKER_ID.conf
+ -e 's/zkServers=localhost:2181/'zkServers=$PUBLIC_ZOOKEEPER_ADDRESSES'/' \
+ ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/bookie-$BROKER_ID.conf
-sleep 5
+sleep 5
if [ $DEPLOY_BK == true ]; then
- if [ $BROKER_ID -eq "1" ]; then
- echo "create /messaging" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES
- echo "create /messaging/bookkeeper" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES
- echo "create /messaging/bookkeeper/ledgers" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES
- echo "Metafirmatting bookie"
- export BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf
- echo "Y" | ./distributedlog-service/bin/dlog bkshell metaformat
- fi
-
- echo "Configuring bookkeeper"
- BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf ./distributedlog-service/bin/dlog bkshell bookieformat
-
- echo "Starting server"
- SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie -c $log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf
-
-
- if [ $BROKER_ID -eq "1" ]; then
- ./distributedlog-service/bin/dlog admin bind -dlzr $PUBLIC_ZOOKEEPER_ADDRESSES -dlzw $PUBLIC_ZOOKEEPER_ADDRESSES -s $PUBLIC_ZOOKEEPER_ADDRESSES -bkzr $PUBLIC_ZOOKEEPER_ADDRESSES \
- -l /messaging/bookkeeper/ledgers -i false -r true -c distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace
- fi
-
+ if [ $BROKER_ID -eq "1" ]; then
+ echo "create /messaging" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES
+ echo "create /messaging/bookkeeper" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES
+ echo "create /messaging/bookkeeper/ledgers" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES
+ echo "Metaformatting bookie"
+ export BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf
+ ./distributedlog-service/bin/dlog bkshell metaformat -n -f
+ fi
+
+ echo "Configuring bookkeeper"
+ BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf ./distributedlog-service/bin/dlog bkshell bookieformat
+
+ echo "Starting server"
+ SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie -c $log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf
+
+ if [ $BROKER_ID -eq "1" ]; then
+ ./distributedlog-service/bin/dlog admin bind \
+ -dlzr $PUBLIC_ZOOKEEPER_ADDRESSES \
+ -dlzw $PUBLIC_ZOOKEEPER_ADDRESSES \
+ -s $PUBLIC_ZOOKEEPER_ADDRESSES \
+ -bkzr $PUBLIC_ZOOKEEPER_ADDRESSES \
+ -l /messaging/bookkeeper/ledgers \
+ -i false \
+ -r true \
+ -c distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace
+ fi
fi
-
if [ $DEPLOY_WP == true ]; then
- WP_NAMESPACE=distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace WP_SHARD_ID=$BROKER_ID WP_SERVICE_PORT=4181 WP_STATS_PORT=20001 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy
+ WP_NAMESPACE=distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace \
+ WP_SHARD_ID=$BROKER_ID \
+ WP_SERVICE_PORT=4181 \
+ WP_STATS_PORT=20001 \
+ ./distributedlog-service/bin/dlog-daemon.sh start writeproxy
fi
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/vagrant/zk.sh
----------------------------------------------------------------------
diff --git a/vagrant/zk.sh b/vagrant/zk.sh
index 3d5d5de..c690149 100755
--- a/vagrant/zk.sh
+++ b/vagrant/zk.sh
@@ -30,9 +30,9 @@ echo "initLimit=5" >> $log_dir/distributedlog-service/conf/zookeeper-$ZKID.prope
echo "syncLimit=2" >> $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties
echo "quorumListenOnAllIPs=true" >> $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties
sed '/server.1/d' \
- ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/zookeeper-$ZKID.conf
+ ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/zookeeper-$ZKID.conf
for i in $(seq 1 $NUM_ZK); do
- echo "server.${i}=zk${i}:2888:3888" >>$log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties
+ echo "server.${i}=zk${i}:2888:3888" >>$log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties
done
mkdir -p /tmp/data/zookeeper
@@ -41,6 +41,7 @@ echo "$ZKID" > /tmp/data/zookeeper/myid
echo "Killing ZooKeeper"
./distributedlog-service/bin/dlog-daemon.sh stop zookeeper || true
sleep 5
+
echo "Starting ZooKeeper"
echo "./distributedlog-service/bin/dlog-daemon.sh start zookeeper $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties"
./distributedlog-service/bin/dlog-daemon.sh start zookeeper $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties