You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2018/07/23 14:36:29 UTC
[GitHub] eolivelli closed pull request #1553: [table service] Improve
bin/bookkeeper standalone to start all available service components
eolivelli closed pull request #1553: [table service] Improve bin/bookkeeper standalone to start all available service components
URL: https://github.com/apache/bookkeeper/pull/1553
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bin/bookkeeper b/bin/bookkeeper
index ceb047020a..07e7db3ecd 100755
--- a/bin/bookkeeper
+++ b/bin/bookkeeper
@@ -23,9 +23,13 @@ BK_HOME=`cd ${BINDIR}/..;pwd`
source ${BK_HOME}/bin/common.sh
-BOOKIE_MODULE_PATH=bookkeeper-server
-BOOKIE_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server"
-BOOKIE_MODULE_HOME=${BK_HOME}/${BOOKIE_MODULE_PATH}
+if [ "x$1" == "xstandalone" ]; then
+ BOOKIE_MODULE_PATH=stream/server
+ BOOKIE_MODULE_NAME="(org.apache.bookkeeper-)?stream-storage-server"
+else
+ BOOKIE_MODULE_PATH=bookkeeper-server
+ BOOKIE_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server"
+fi
# find the module jar
BOOKIE_JAR=$(find_module_jar ${BOOKIE_MODULE_PATH} ${BOOKIE_MODULE_NAME})
@@ -44,7 +48,7 @@ where command is one of:
bookie Run a bookie server
autorecovery Run AutoRecovery service daemon
localbookie <n> Run a test ensemble of <n> bookies locally
- standalone <n> Run a standalone cluster of <n> bookies locally
+ standalone Run a standalone cluster (with all service components) locally
upgrade Upgrade bookie filesystem
shell Run shell for admin commands
zookeeper Run zookeeper server
@@ -130,10 +134,12 @@ if [ ${COMMAND} == "bookie" ]; then
exec ${JAVA} ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.server.Main --conf ${BOOKIE_CONF} $@
elif [ ${COMMAND} == "autorecovery" ]; then
exec ${JAVA} ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.replication.AutoRecoveryMain --conf ${BOOKIE_CONF} $@
-elif [ ${COMMAND} == "localbookie" -o ${COMMAND} == "standalone" ]; then
+elif [ ${COMMAND} == "localbookie" ]; then
NUMBER=$1
shift
exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.util.LocalBookKeeper ${NUMBER} ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "standalone" ]; then
+ exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.stream.cluster.StandaloneStarter --conf ${BK_HOME}/conf/standalone.conf $@
elif [ ${COMMAND} == "upgrade" ]; then
exec ${JAVA} ${OPTS} org.apache.bookkeeper.bookie.FileSystemUpgrade --conf ${BOOKIE_CONF} $@
elif [ $COMMAND == "zookeeper" ]; then
diff --git a/bin/bookkeeper-daemon.sh b/bin/bookkeeper-daemon.sh
index 3a8bbdcae3..d64a4489bc 100755
--- a/bin/bookkeeper-daemon.sh
+++ b/bin/bookkeeper-daemon.sh
@@ -69,6 +69,9 @@ case $command in
(autorecovery)
echo "doing $startStop $command ..."
;;
+ (standalone)
+ echo "doing $startStop $command ..."
+ ;;
(*)
echo "Error: unknown service name $command"
usage
@@ -134,7 +137,7 @@ case $startStop in
count=0
location=$BOOKIE_LOG_DIR
- while kill -0 $TARGET_PID > /dev/null;
+ while kill -0 $TARGET_PID > /dev/null 2>&1;
do
echo "Shutdown is in progress... Please wait..."
sleep 1
diff --git a/bin/standalone b/bin/standalone
index d4bc0f5b87..6ed5b0975c 100755
--- a/bin/standalone
+++ b/bin/standalone
@@ -19,13 +19,47 @@
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
+usage() {
+ cat <<EOF
+Usage: standalone <runtime> (up|down)
+where runtime is one of:
+ process Run a standalone cluster in one process
+ docker-compose Run a standalone cluster in a docker-composed cluster
+EOF
+}
BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
-DOCKER_COMPOSE=$(which docker-compose)
-if [ $? != 0 ]; then
- echo "Error: docker-compose is not found in ${PATH}." 1>&2
+if [ $# -lt 2 ]
+then
+ echo "Error: no enough arguments provided."
+ usage
exit 1
-else
- ${BINDIR}/standalone.docker-compose $@
fi
+
+runtime=$1
+shift
+action=$1
+shift
+
+case $runtime in
+ (process)
+ ${BINDIR}/standalone.process $action $@
+ ;;
+ (docker-compose)
+ DOCKER_COMPOSE=$(which docker-compose)
+ if [ $? != 0 ]; then
+ echo "Error: docker-compose is not found in ${PATH}." 1>&2
+ usage
+ exit 1
+ else
+ echo ${BINDIR}/standalone.docker-compose $action $@
+ ${BINDIR}/standalone.docker-compose $action $@
+ fi
+ ;;
+ (*)
+ echo "Error: unknown runtime $command"
+ usage
+ exit 1
+ ;;
+esac
diff --git a/bin/standalone.docker-compose b/bin/standalone.docker-compose
index 4cf1ffd2bc..05808a74da 100755
--- a/bin/standalone.docker-compose
+++ b/bin/standalone.docker-compose
@@ -111,7 +111,7 @@ function generate_docker_compose_file() {
function show_help() {
cat <<EOF
-Usage: standalone [-c <cluster_name>] [-h] <action:[up|down]>
+Usage: standalone.docker-compose [-c <cluster_name>] [-h] <action:[up|down]>
EOF
}
diff --git a/bin/standalone.process b/bin/standalone.process
new file mode 100755
index 0000000000..661995d0c2
--- /dev/null
+++ b/bin/standalone.process
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+#
+# vim:et:ft=sh:sts=2:sw=2
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+function show_help() {
+ cat <<EOF
+Usage: standalone.process [-c <cluster_name>] [-h] <action:[up|down]>
+EOF
+}
+
+BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
+
+source ${BINDIR}/common.sh
+
+DATA_ROOT_DIR=${BK_DATA_DIR:-"${BK_HOME}/data"}
+mkdir -p ${DATA_ROOT_DIR}
+
+# main entrypoint
+
+CLUSTER_NAME="bk-standalone"
+OPTIND=1
+
+while getopts "h:c:" opt; do
+ case "${opt}" in
+ c )
+ CLUSTER_NAME=${OPTARG}
+ echo "use cluster = '${CLUSTER_NAME}'."
+ ;;
+ h|\? )
+ show_help
+ exit 1
+ ;;
+ esac
+done
+
+shift $((OPTIND-1))
+
+[ "${1:-}" = "--" ] && shift
+
+if [ $# -le 0 ]; then
+ show_help
+ exit 1
+fi
+
+ACTION=$1
+DAEMON_ACTION=""
+case "${ACTION}" in
+ up)
+ DAEMON_ACTION="start"
+ ;;
+ down)
+ DAEMON_ACTION="stop"
+ ;;
+ *)
+ echo "Unknown action : ${ACTION}"
+ show_help
+ exit 1
+ ;;
+esac
+
+
+CLUSTER_DATA_ROOT="${DATA_ROOT_DIR}/${CLUSTER_NAME}"
+mkdir -p ${CLUSTER_DATA_ROOT}
+
+${BINDIR}/bookkeeper-daemon.sh ${DAEMON_ACTION} standalone --data-dir ${CLUSTER_DATA_ROOT}
diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml
index 740f16a5b3..ea07705064 100644
--- a/bookkeeper-common/pom.xml
+++ b/bookkeeper-common/pom.xml
@@ -38,6 +38,18 @@
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
diff --git a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
index 7d22443097..8b2759709e 100644
--- a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
+++ b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
@@ -21,9 +21,13 @@
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URL;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import org.apache.bookkeeper.common.util.JsonUtil;
+import org.apache.bookkeeper.common.util.JsonUtil.ParseJsonException;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
@@ -287,4 +291,31 @@ public String getString(String key, String defaultValue) {
public List<Object> getList(String key, List<?> defaultValue) {
return conf.getList(getKeyName(key), defaultValue);
}
+
+ /**
+ * returns the string representation of json format of this config.
+ *
+ * @return
+ * @throws ParseJsonException
+ */
+ public String asJson() {
+ try {
+ return JsonUtil.toJson(toMap());
+ } catch (ParseJsonException e) {
+ throw new RuntimeException("Failed to serialize the configuration as json", e);
+ }
+ }
+
+ private Map<String, Object> toMap() {
+ Map<String, Object> configMap = new HashMap<>();
+ Iterator<String> iterator = this.getKeys();
+ while (iterator.hasNext()) {
+ String key = iterator.next().toString();
+ Object property = this.getProperty(key);
+ if (property != null) {
+ configMap.put(key, property.toString());
+ }
+ }
+ return configMap;
+ }
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/JsonUtil.java b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/JsonUtil.java
similarity index 77%
rename from bookkeeper-server/src/main/java/org/apache/bookkeeper/util/JsonUtil.java
rename to bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/JsonUtil.java
index e5e71427ab..afc90a4f48 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/JsonUtil.java
+++ b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/JsonUtil.java
@@ -1,5 +1,4 @@
-/**
- *
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -8,17 +7,15 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
*
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-package org.apache.bookkeeper.util;
+package org.apache.bookkeeper.common.util;
import com.fasterxml.jackson.databind.ObjectMapper;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
index 2c80f43e12..481c5a5d08 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
@@ -28,14 +28,14 @@
import javax.net.ssl.SSLEngine;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.common.util.JsonUtil;
+import org.apache.bookkeeper.common.util.JsonUtil.ParseJsonException;
import org.apache.bookkeeper.feature.Feature;
import org.apache.bookkeeper.meta.AbstractZkLedgerManagerFactory;
import org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.meta.LongHierarchicalLedgerManagerFactory;
import org.apache.bookkeeper.util.EntryFormatter;
-import org.apache.bookkeeper.util.JsonUtil;
-import org.apache.bookkeeper.util.JsonUtil.ParseJsonException;
import org.apache.bookkeeper.util.LedgerIdFormatter;
import org.apache.bookkeeper.util.ReflectionUtils;
import org.apache.bookkeeper.util.StringEntryFormatter;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
index d386b915c3..711c2326ce 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
@@ -37,6 +37,7 @@
import org.apache.bookkeeper.bookie.ExitCode;
import org.apache.bookkeeper.bookie.ReadOnlyBookie;
import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.common.util.JsonUtil.ParseJsonException;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.processor.RequestProcessor;
@@ -48,7 +49,6 @@
import org.apache.bookkeeper.tls.SecurityException;
import org.apache.bookkeeper.tls.SecurityHandlerFactory;
import org.apache.bookkeeper.tls.SecurityProviderFactoryFactory;
-import org.apache.bookkeeper.util.JsonUtil.ParseJsonException;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java
index 2ac9e8e6a0..bcf55781d3 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java
@@ -23,12 +23,12 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
/**
* HttpEndpointService that handle Bookkeeper Configuration related http request.
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java
index ea05f6e97a..0091ce9ee2 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java
@@ -24,13 +24,13 @@
import java.util.concurrent.ExecutorService;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java
index 2593e94d42..626b89f9dd 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java
@@ -22,13 +22,13 @@
import java.util.Map;
import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java
index 4cbd1dc78c..0ffa9bbfb2 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java
@@ -30,13 +30,13 @@
import org.apache.bookkeeper.bookie.Journal;
import org.apache.bookkeeper.bookie.LedgerDirsManager;
import org.apache.bookkeeper.bookie.LogMark;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.util.DiskChecker;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java
index 5a06fafc06..43b49943f4 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java
@@ -24,6 +24,7 @@
import com.google.common.collect.Maps;
import java.util.Map;
import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
@@ -32,7 +33,6 @@
import org.apache.bookkeeper.meta.LedgerManager;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.proto.BookieServer;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java
index 36afa922f0..14439be86c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java
@@ -29,6 +29,7 @@
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookieInfoReader;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
@@ -36,7 +37,6 @@
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java
index b28c715cf2..f969ff7797 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java
@@ -27,13 +27,13 @@
import java.util.Map;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java
index 260b82dc8c..330ea7f016 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java
@@ -27,12 +27,12 @@
import java.util.List;
import java.util.Map;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
index 758af704fa..efd00e7341 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
@@ -29,6 +29,7 @@
import java.util.Map;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
@@ -38,7 +39,6 @@
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
index 237760c33a..7303be0ae1 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
@@ -26,6 +26,7 @@
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
@@ -34,7 +35,6 @@
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
import org.apache.bookkeeper.proto.BookieServer;
-import org.apache.bookkeeper.util.JsonUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java
index fcc9eff71b..6442daaa56 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java
@@ -23,12 +23,12 @@
import java.util.HashMap;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java
index 9bed45da7c..4c2141dc1b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java
@@ -27,12 +27,12 @@
import java.util.Map;
import org.apache.bookkeeper.client.BookKeeperAdmin;
import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java
index afc5bcd5c1..3a08bafc5b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java
@@ -28,13 +28,13 @@
import org.apache.bookkeeper.bookie.Cookie;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.apache.bookkeeper.versioning.Versioned;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
index 1ed1a22a26..a2a77284a0 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
@@ -35,6 +35,7 @@
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.LedgerHandleAdapter;
import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
import org.apache.bookkeeper.http.HttpServer;
@@ -48,7 +49,6 @@
import org.apache.bookkeeper.replication.AuditorElector;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.apache.bookkeeper.test.TestCallbacks;
-import org.apache.bookkeeper.util.JsonUtil;
import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
import org.apache.zookeeper.ZooKeeper;
import org.junit.Before;
diff --git a/conf/standalone.conf b/conf/standalone.conf
new file mode 100755
index 0000000000..4b52f2bf28
--- /dev/null
+++ b/conf/standalone.conf
@@ -0,0 +1,30 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Standalone configuration
+
+##################################################################
+##################################################################
+# stream/table service
+##################################################################
+##################################################################
+
+### Storage ###
+
+# the cluster controller schedule interval, in milliseconds. default is 30 seconds.
+storage.cluster.controller.schedule.interval.ms=30000
diff --git a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java
index 16d73d2258..a8c94eaa30 100644
--- a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java
+++ b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java
@@ -17,10 +17,16 @@
*/
package org.apache.bookkeeper.stream.cluster;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import java.io.File;
import java.util.concurrent.CountDownLatch;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.common.net.ServiceURI;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
/**
* A starter to start a standalone cluster.
@@ -28,6 +34,78 @@
@Slf4j
public class StandaloneStarter {
+ /**
+ * Args for standalone starter.
+ */
+ private static class StarterArgs {
+
+ @Parameter(
+ names = {
+ "-c", "--conf"
+ },
+ description = "Configuration file path")
+ String configFile = null;
+
+ @Parameter(
+ names = {
+ "--num-bookies"
+ },
+ description = "Num of bookies")
+ int numBookies = 1;
+
+ @Parameter(
+ names = {
+ "--zk-port"
+ },
+ description = "ZooKeeper port")
+ int zkPort = 2181;
+
+ @Parameter(
+ names = {
+ "-u", "--metadata-service-uri"
+ },
+ description = "Uri of the external metadata service."
+ + " If set, the standalone will not start any metadata service itself.")
+ String metadataServiceUri = null;
+
+ @Parameter(
+ names = {
+ "--initial-bookie-port"
+ },
+ description = "Initial bookie port")
+ int initialBookiePort = 3181;
+
+ @Parameter(
+ names = {
+ "--initial-bookie-grpc-port"
+ },
+ description = "Initial bookie grpc port")
+ int initialBookieGrpcPort = 4181;
+
+ @Parameter(
+ names = {
+ "--data-dir"
+ },
+ description = "Location to store standalone data"
+ )
+ String dataDir = "data";
+
+ @Parameter(
+ names = {
+ "--wipe-data"
+ },
+ description = "Clean up previous standalone data")
+ boolean wipeData = false;
+
+ @Parameter(
+ names = {
+ "-h", "--help"
+ },
+ description = "Show this help message")
+ boolean help = false;
+
+ }
+
public static void main(String[] args) throws Exception {
int retCode = doMain(args);
@@ -35,16 +113,57 @@ public static void main(String[] args) throws Exception {
}
static int doMain(String[] args) throws Exception {
- Path rootDir = Files.createTempDirectory("data");
- rootDir.toFile().deleteOnExit();
-
- StreamClusterSpec spec = StreamClusterSpec.builder()
- .numServers(3)
- .initialBookiePort(3181)
- .initialGrpcPort(4181)
- .serveReadOnlyTable(true)
- .shouldStartZooKeeper(true)
- .storageRootDir(rootDir.toFile())
+ StarterArgs starterArgs = new StarterArgs();
+
+ JCommander commander = new JCommander();
+ try {
+ commander.setProgramName("standalone-starter");
+ commander.addObject(starterArgs);
+ commander.parse(args);
+ if (starterArgs.help) {
+ commander.usage();
+ return 0;
+ }
+ } catch (Exception e) {
+ commander.usage();
+ return -1;
+ }
+
+ StreamClusterSpec.StreamClusterSpecBuilder specBuilder = StreamClusterSpec.builder();
+ if (starterArgs.metadataServiceUri == null) {
+ specBuilder = specBuilder
+ .zkPort(starterArgs.zkPort)
+ .shouldStartZooKeeper(true);
+ } else {
+ ServiceURI serviceURI = ServiceURI.create(starterArgs.metadataServiceUri);
+ specBuilder = specBuilder
+ .metadataServiceUri(serviceURI)
+ .shouldStartZooKeeper(false);
+ }
+
+ CompositeConfiguration conf = new CompositeConfiguration();
+ if (null != starterArgs.configFile) {
+ PropertiesConfiguration propsConf = new PropertiesConfiguration(starterArgs.configFile);
+ conf.addConfiguration(propsConf);
+ }
+
+ checkArgument(starterArgs.numBookies > 0, "Invalid number of bookies : " + starterArgs.numBookies);
+ if (starterArgs.numBookies == 1) {
+ conf.setProperty("dlog.bkcEnsembleSize", 1);
+ conf.setProperty("dlog.bkcWriteQuorumSize", 1);
+ conf.setProperty("dlog.bkcAckQuorumSize", 1);
+ } else {
+ conf.setProperty("dlog.bkcEnsembleSize", starterArgs.numBookies);
+ conf.setProperty("dlog.bkcWriteQuorumSize", starterArgs.numBookies);
+ conf.setProperty("dlog.bkcAckQuorumSize", starterArgs.numBookies - 1);
+ }
+
+ StreamClusterSpec spec = specBuilder
+ .baseConf(conf)
+ .numServers(starterArgs.numBookies)
+ .initialBookiePort(starterArgs.initialBookiePort)
+ .initialGrpcPort(starterArgs.initialBookieGrpcPort)
+ .storageRootDir(new File(starterArgs.dataDir))
.build();
CountDownLatch liveLatch = new CountDownLatch(1);
diff --git a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java
index e5a6df43ad..407b27fbe2 100644
--- a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java
+++ b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java
@@ -14,17 +14,15 @@
package org.apache.bookkeeper.stream.cluster;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.bookkeeper.common.concurrent.FutureUtils.result;
import static org.apache.bookkeeper.stream.protocol.ProtocolConstants.DEFAULT_STREAM_CONF;
-import static org.apache.bookkeeper.stream.storage.StorageConstants.ZK_METADATA_ROOT_PATH;
-import static org.apache.bookkeeper.stream.storage.StorageConstants.getSegmentsRootPath;
-import static org.apache.bookkeeper.stream.storage.StorageConstants.getStoragePath;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
import java.net.BindException;
-import java.net.URI;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -34,10 +32,12 @@
import org.apache.bookkeeper.clients.StorageClientBuilder;
import org.apache.bookkeeper.clients.admin.StorageAdminClient;
import org.apache.bookkeeper.clients.config.StorageClientSettings;
-import org.apache.bookkeeper.clients.exceptions.NamespaceExistsException;
+import org.apache.bookkeeper.clients.exceptions.ClientException;
+import org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException;
import org.apache.bookkeeper.clients.utils.NetUtils;
import org.apache.bookkeeper.common.component.AbstractLifecycleComponent;
import org.apache.bookkeeper.common.component.LifecycleComponent;
+import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.MetadataDrivers;
import org.apache.bookkeeper.shims.zk.ZooKeeperServerShim;
@@ -46,12 +46,11 @@
import org.apache.bookkeeper.stream.proto.NamespaceProperties;
import org.apache.bookkeeper.stream.proto.common.Endpoint;
import org.apache.bookkeeper.stream.server.StorageServer;
-import org.apache.bookkeeper.stream.storage.StorageConstants;
import org.apache.bookkeeper.stream.storage.conf.StorageConfiguration;
import org.apache.bookkeeper.stream.storage.exceptions.StorageRuntimeException;
import org.apache.bookkeeper.stream.storage.impl.cluster.ZkClusterInitializer;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.distributedlog.DistributedLogConfiguration;
import org.apache.distributedlog.LocalDLMEmulator;
/**
@@ -73,10 +72,9 @@ public static StreamCluster build(StreamClusterSpec spec) {
return new StreamCluster(spec);
}
- private static ServerConfiguration newBookieConfiguration(String zkEnsemble) {
+ private static ServerConfiguration newBookieConfiguration(ServiceURI serviceURI) {
ServerConfiguration serverConf = new ServerConfiguration();
- serverConf.setMetadataServiceUri(
- "zk://" + zkEnsemble + getSegmentsRootPath(StorageConstants.ZK_METADATA_ROOT_PATH));
+ serverConf.setMetadataServiceUri(serviceURI.getUri().toString());
serverConf.setAllowLoopback(true);
serverConf.setGcWaitTime(300000);
serverConf.setDiskUsageWarnThreshold(0.9999f);
@@ -86,7 +84,7 @@ private static ServerConfiguration newBookieConfiguration(String zkEnsemble) {
private final StreamClusterSpec spec;
private final List<Endpoint> rpcEndpoints;
- private String zkEnsemble;
+ private ServiceURI metadataServiceUri;
private int zkPort;
private ZooKeeperServerShim zks;
private List<LifecycleComponent> servers;
@@ -109,24 +107,21 @@ private StreamCluster(StreamClusterSpec spec) {
return rpcEndpoints;
}
- public URI getDefaultBackendUri() {
- return URI.create("distributedlog://" + zkEnsemble + getStoragePath(ZK_METADATA_ROOT_PATH));
- }
-
private void startZooKeeper() throws Exception {
if (!spec.shouldStartZooKeeper()) {
- zkPort = spec.zkPort();
- zkEnsemble = spec.zkServers() + ":" + zkPort;
+ metadataServiceUri = checkNotNull(spec.metadataServiceUri,
+ "No metadata service uri is configured while configuring not to start zookeeper");
return;
}
File zkDir = new File(spec.storageRootDir(), "zookeeper");
Pair<ZooKeeperServerShim, Integer> zkServerAndPort =
- LocalDLMEmulator.runZookeeperOnAnyPort(zkDir);
+ LocalDLMEmulator.runZookeeperOnAnyPort(spec.zkPort(), zkDir);
zks = zkServerAndPort.getLeft();
zkPort = zkServerAndPort.getRight();
log.info("Started zookeeper at port {}.", zkPort);
- zkEnsemble = "127.0.0.1:" + zkPort;
+ metadataServiceUri = ServiceURI.create(
+ "zk://127.0.0.1:" + zkPort + "/ledgers");
}
private void stopZooKeeper() {
@@ -137,12 +132,17 @@ private void stopZooKeeper() {
}
private void initializeCluster() throws Exception {
- new ZkClusterInitializer(zkEnsemble).initializeCluster(
- URI.create("zk://" + zkEnsemble),
+ checkArgument(ServiceURI.SERVICE_ZK.equals(metadataServiceUri.getServiceName()),
+ "Only support zookeeper based metadata service now");
+ String[] serviceHosts = metadataServiceUri.getServiceHosts();
+ String metadataServers = StringUtils.join(serviceHosts, ',');
+
+ new ZkClusterInitializer(metadataServers).initializeCluster(
+ metadataServiceUri.getUri(),
spec.numServers() * 2);
// format the bookkeeper cluster
- MetadataDrivers.runFunctionWithMetadataBookieDriver(newBookieConfiguration(zkEnsemble), driver -> {
+ MetadataDrivers.runFunctionWithMetadataBookieDriver(newBookieConfiguration(metadataServiceUri), driver -> {
try {
boolean initialized = driver.getRegistrationManager().initNewCluster();
if (initialized) {
@@ -170,23 +170,20 @@ private LifecycleComponent startServer() throws Exception {
}
LifecycleComponent server = null;
try {
- ServerConfiguration serverConf = newBookieConfiguration(zkEnsemble);
+ ServerConfiguration serverConf = newBookieConfiguration(metadataServiceUri);
+ serverConf.loadConf(spec.baseConf());
serverConf.setBookiePort(bookiePort);
File bkDir = new File(spec.storageRootDir(), "bookie_" + bookiePort);
serverConf.setJournalDirName(bkDir.getPath());
serverConf.setLedgerDirNames(new String[]{bkDir.getPath()});
- DistributedLogConfiguration dlConf = new DistributedLogConfiguration();
- dlConf.loadConf(serverConf);
-
File rangesStoreDir = new File(spec.storageRootDir(), "ranges_" + grpcPort);
StorageConfiguration storageConf = new StorageConfiguration(serverConf);
storageConf.setRangeStoreDirNames(new String[]{rangesStoreDir.getPath()});
- storageConf.setServeReadOnlyTables(spec.serveReadOnlyTable);
log.info("Attempting to start storage server at (bookie port = {}, grpc port = {})"
- + " : bkDir = {}, rangesStoreDir = {}, serveReadOnlyTables = {}",
- bookiePort, grpcPort, bkDir, rangesStoreDir, spec.serveReadOnlyTable);
+ + " : bkDir = {}, rangesStoreDir = {}",
+ bookiePort, grpcPort, bkDir, rangesStoreDir);
server = StorageServer.buildStorageServer(
serverConf,
grpcPort);
@@ -245,18 +242,27 @@ private void createDefaultNamespaces() throws Exception {
.withSettings(settings)
.buildAdmin()) {
- System.out.println("Creating namespace '" + namespaceName + "' ...");
- try {
- NamespaceProperties nsProps = result(
- admin.createNamespace(
- namespaceName,
- NamespaceConfiguration.newBuilder()
- .setDefaultStreamConf(DEFAULT_STREAM_CONF)
- .build()));
- System.out.println("Successfully created namespace '" + namespaceName + "':");
- System.out.println(nsProps);
- } catch (NamespaceExistsException nee) {
- System.out.println("Namespace '" + namespaceName + "' already exists.");
+ boolean created = false;
+ while (!created) {
+ try {
+ result(admin.getNamespace(namespaceName));
+ created = true;
+ } catch (NamespaceNotFoundException nnfe) {
+ log.info("Namespace '{}' is not found.");
+ log.info("Creating namespace '{}' ...", namespaceName);
+ try {
+ NamespaceProperties nsProps = result(
+ admin.createNamespace(
+ namespaceName,
+ NamespaceConfiguration.newBuilder()
+ .setDefaultStreamConf(DEFAULT_STREAM_CONF)
+ .build()));
+ log.info("Successfully created namespace '{}':", namespaceName);
+ log.info("{}", nsProps);
+ } catch (ClientException ce) {
+ // encountered exception, try to fetch the namespace again
+ }
+ }
}
}
}
diff --git a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java
index 41608bc641..76fd97dcf4 100644
--- a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java
+++ b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java
@@ -20,6 +20,7 @@
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
+import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.commons.configuration.CompositeConfiguration;
/**
@@ -47,12 +48,12 @@
CompositeConfiguration baseConf;
/**
- * Returns the zookeeper servers used in this cluster.
+ * Returns the metadata service uri that the servers will connect to.
*
- * @return the zookeeper servers used in this cluster.
+ * @return the metadata service uri that the servers will connect to.
*/
@Default
- String zkServers = "127.0.0.1";
+ ServiceURI metadataServiceUri = null;
/**
* Returns if should start zookeeper.
@@ -94,7 +95,4 @@
@Default
File storageRootDir = new File("data/bookkeeper");
- @Default
- boolean serveReadOnlyTable = false;
-
}
diff --git a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java
index ad8491e6b2..97ff66ad2b 100644
--- a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java
+++ b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java
@@ -206,11 +206,17 @@ public static LifecycleComponent buildStorageServer(CompositeConfiguration conf,
StatsProviderService statsProviderService = new StatsProviderService(bkConf);
rootStatsLogger = statsProviderService.getStatsProvider().getStatsLogger("");
serverBuilder.addComponent(statsProviderService);
+ log.info("Bookie configuration : {}", bkConf.asJson());
} else {
rootStatsLogger = checkNotNull(externalStatsLogger,
"External stats logger is not provided while not starting stats provider");
}
+ // dump configurations
+ log.info("Dlog configuration : {}", dlConf.asJson());
+ log.info("Storage configuration : {}", storageConf.asJson());
+ log.info("Server configuration : {}", serverConf.asJson());
+
// Create the bookie service
ServerConfiguration bkServerConf;
if (startBookieAndStartProvider) {
diff --git a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java
index 5a645d3894..4a3741f9f0 100644
--- a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java
+++ b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java
@@ -83,7 +83,7 @@ private static void waitingForNumBookies(RegistrationClient client, int minNumBo
bookies = FutureUtils.result(client.getWritableBookies()).getValue();
log.info("Only {} bookies are live since {} seconds elapsed, "
+ "wait for another {} bookies for another 1 second",
- bookies.size(), minNumBookies - bookies.size(), stopwatch.elapsed(TimeUnit.SECONDS));
+ bookies.size(), stopwatch.elapsed(TimeUnit.SECONDS), minNumBookies - bookies.size());
}
}
diff --git a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java
index 5f1e4382e9..7beb2912a9 100644
--- a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java
+++ b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java
@@ -23,7 +23,7 @@
import java.time.Duration;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
-import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
+import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy;
/**
* Test Container for Bookies.
@@ -33,6 +33,7 @@
private static final int ZK_PORT = 2181;
private static final int BOOKIE_BASE_PORT = 3181;
+ private static final int BOOKIE_GRPC_BASE_PORT = 4181;
private static final String IMAGE_NAME = "apachebookkeeper/bookkeeper-current:latest";
@@ -60,14 +61,19 @@ protected void configure() {
}
setCommand(
"standalone",
- "" + numBookies);
+ "--num-bookies",
+ String.valueOf(numBookies),
+ "--zk-port",
+ String.valueOf(ZK_PORT),
+ "--initial-bookie-port",
+ String.valueOf(BOOKIE_BASE_PORT),
+ "--initial-bookie-grpc-port",
+ String.valueOf(BOOKIE_GRPC_BASE_PORT));
}
@Override
public void start() {
- this.waitStrategy = new LogMessageWaitStrategy()
- .withRegEx(".*ForceWrite Thread started.*\\s")
- .withTimes(numBookies)
+ this.waitStrategy = new HostPortWaitStrategy()
.withStartupTimeout(Duration.of(60, SECONDS));
this.withCreateContainerCmdModifier(createContainerCmd -> {
createContainerCmd.withHostName(STANDALONE_HOST_NAME);
diff --git a/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java b/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java
index df082d4b75..f7b7077810 100644
--- a/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java
+++ b/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java
@@ -23,7 +23,9 @@
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.tests.containers.BKStandaloneContainer;
import org.junit.ClassRule;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.testcontainers.containers.Container.ExecResult;
/**
@@ -32,6 +34,9 @@
@Slf4j
public class StandaloneTest {
+ @Rule
+ public final TestName testName = new TestName();
+
@ClassRule
public static BKStandaloneContainer bkContainer = new BKStandaloneContainer<>("integrationtest", 3);
@@ -50,4 +55,42 @@ public void runSimpleTest() throws Exception {
result.getStdout().contains("100 entries written to ledger"));
}
+ //
+ // `namespace` commands
+ //
+
+ @Test
+ public void createNamespace() throws Exception {
+ String nsName = testName.getMethodName();
+ ExecResult result = bkContainer.execCmd(
+ "/opt/bookkeeper/bin/bkctl",
+ "-u bk://localhost:4181",
+ "namespace",
+ "create",
+ nsName
+ );
+ assertTrue(
+ result.getStdout(),
+ result.getStdout().contains("Successfully created namespace '" + nsName + "'"));
+ }
+
+ //
+ // `tables` commands
+ //
+
+ @Test
+ public void createTable() throws Exception {
+ String tableName = testName.getMethodName();
+ ExecResult result = bkContainer.execCmd(
+ "/opt/bookkeeper/bin/bkctl",
+ "-u bk://localhost:4181",
+ "tables",
+ "create",
+ tableName
+ );
+ assertTrue(
+ result.getStdout(),
+ result.getStdout().contains("Successfully created stream '" + tableName + "'"));
+ }
+
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services