You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by mm...@apache.org on 2021/03/28 01:21:17 UTC
[bookkeeper] branch master updated: ISSUE-2640: BP-43: Run
integration tests with current source code. Build BKCTL. (#2663)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 6f6b17e ISSUE-2640: BP-43: Run integration tests with current source code. Build BKCTL. (#2663)
6f6b17e is described below
commit 6f6b17e5da7fae874febe18fb1524a12f315d043
Author: Prashant Kumar <65...@users.noreply.github.com>
AuthorDate: Sat Mar 27 18:21:10 2021 -0700
ISSUE-2640: BP-43: Run integration tests with current source code. Build BKCTL. (#2663)
Co-authored-by: Prashant <pr...@splunk.com>
---
dependencies.gradle | 14 +--
settings.gradle | 16 ++-
stream/api/build.gradle | 4 +
stream/clients/java/all/build.gradle | 4 +
stream/common/build.gradle | 3 +
stream/server/build.gradle | 29 ++++-
stream/storage/api/build.gradle | 4 +
tests/docker-images/statestore-image/Dockerfile | 70 ++++++++++++
tests/docker-images/statestore-image/bin/bkctl | 59 ++++++++++
tests/docker-images/statestore-image/bin/bkperf | 58 ++++++++++
.../docker-images/statestore-image/bin/bookkeeper | 119 +++++++++++++++++++++
.../statestore-image/conf/log4j.cli.properties | 41 +++++++
.../statestore-image/conf/log4j.properties | 40 +++++++
.../statestore-image/conf/log4j.shell.properties | 41 +++++++
.../statestore-image/image_builder.sh | 39 +++++++
tests/integration/cluster/build.gradle | 19 ++++
tests/integration/smoke/build.gradle | 5 +
tests/integration/standalone/build.gradle | 14 +++
tools/framework/build.gradle | 4 +
tools/{framework => ledger}/build.gradle | 18 ++++
....apache.bookkeeper.tools.framework.CommandGroup | 24 +++++
{stream/server => tools/perf}/build.gradle | 54 ++++++----
{stream/server => tools/stream}/build.gradle | 45 ++++----
....apache.bookkeeper.tools.framework.CommandGroup | 22 ++++
24 files changed, 697 insertions(+), 49 deletions(-)
diff --git a/dependencies.gradle b/dependencies.gradle
index 04f8513..58a7eb0 100644
--- a/dependencies.gradle
+++ b/dependencies.gradle
@@ -35,7 +35,7 @@ depVersions = [
commonsLang2: "2.6",
commonsLang3: "3.6",
commonsBeanutils: "1.9.3",
- curator: "4.0.1",
+ curator: "5.1.0",
dockerJava: "3.2.5",
errorprone: "2.1.2",
freebuilder: "1.14.9",
@@ -44,11 +44,12 @@ depVersions = [
groovy: "2.5.8",
guava: "30.1-jre",
hamcrest: "1.3",
+ hdrhistogram: "2.1.4",
httpclient: "4.5.13",
- jackson: "2.9.7",
+ jackson: "2.11.1",
jcommander: "1.78",
jctools: "2.1.2",
- jetty: "9.4.5.v20170502",
+ jetty: "9.4.31.v20200723",
jmock: "2.8.2",
jna: "3.2.7",
jsr305: "3.0.2",
@@ -74,10 +75,10 @@ depVersions = [
shrinkwrap:"3.1.4",
snappy: "1.1.7.7",
thrift: "0.12.0",
- testcontainers: "1.14.3",
+ testcontainers: "1.15.1",
vertx: "3.9.2",
yahooDatasketches: "0.8.3",
- zookeeper: "3.4.13",
+ zookeeper: "3.6.2",
]
depLibs = [
@@ -105,6 +106,7 @@ depLibs = [
grpc: "io.grpc:grpc-all:${depVersions.grpc}",
groovy: "org.codehaus.groovy:groovy-all:${depVersions.groovy}",
guava: "com.google.guava:guava:${depVersions.guava}",
+ hdrHistogram: "org.hdrhistogram:HdrHistogram:${depVersions.hdrhistogram}",
hamcrest: "org.hamcrest:hamcrest-all:${depVersions.hamcrest}",
httpclient: "org.apache.httpcomponents:httpclient:${depVersions.httpclient}",
jacksonAnnotations: "com.fasterxml.jackson.core:jackson-annotations:${depVersions.jackson}",
@@ -153,5 +155,5 @@ depLibs = [
vertxWeb: "io.vertx:vertx-web:${depVersions.vertx}",
yahooDatasketches: "com.yahoo.datasketches:sketches-core:${depVersions.yahooDatasketches}",
zookeeper: "org.apache.zookeeper:zookeeper:${depVersions.zookeeper}",
- zookeeperTest: "org.apache.zookeeper:zookeeper:${depVersions.zookeeper}:tests",
+ zookeeperTest: "org.apache.zookeeper:zookeeper:${depVersions.zookeeper}:tests"
]
diff --git a/settings.gradle b/settings.gradle
index daa1432..ed45ca3 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -24,9 +24,14 @@ pluginManagement {
rootProject.name = 'bookkeeper'
-include 'bookkeeper-benchmark',
+include(':bookkeeper-benchmark',
+ ':bookkeeper-tools-framework',
+ ':bookkeeper-tools-ledger',
+ ':bookkeeper-tools-perf',
+ ':bookkeeper-tools-stream',
'bookkeeper-common',
'bookkeeper-common-allocator',
+ 'bookkeeper-dist:storage-bench',
'bookkeeper-http:http-server',
'bookkeeper-http:vertx-http-server',
'bookkeeper-proto',
@@ -58,5 +63,12 @@ include 'bookkeeper-benchmark',
'tests:integration:cluster',
'tests:integration:smoke',
'tests:integration:standalone',
- 'tools:framework'
+ 'tools:framework')
+
+project(':bookkeeper-tools-framework').projectDir = file('tools/framework')
+project(':bookkeeper-tools-ledger').projectDir = file('tools/ledger')
+project(':bookkeeper-tools-stream').projectDir = file('tools/stream')
+project(':bookkeeper-tools-perf').projectDir = file('tools/perf')
+include 'stream:perf'
+findProject(':stream:perf')?.name = 'perf'
diff --git a/stream/api/build.gradle b/stream/api/build.gradle
index 9be37a7..ad9eaa5 100644
--- a/stream/api/build.gradle
+++ b/stream/api/build.gradle
@@ -28,6 +28,10 @@ dependencies {
annotationProcessor depLibs.lombok
}
+jar {
+ archiveBaseName = 'stream-storage-api'
+}
+
publishing {
publications {
maven(MavenPublication) {
diff --git a/stream/clients/java/all/build.gradle b/stream/clients/java/all/build.gradle
index 9b19abc..54c47ed 100644
--- a/stream/clients/java/all/build.gradle
+++ b/stream/clients/java/all/build.gradle
@@ -36,6 +36,10 @@ dependencies {
annotationProcessor depLibs.lombok
}
+jar {
+ archiveBaseName = 'stream-storage-java-client'
+}
+
publishing {
publications {
maven(MavenPublication) {
diff --git a/stream/common/build.gradle b/stream/common/build.gradle
index daa2b31..cbd0406 100644
--- a/stream/common/build.gradle
+++ b/stream/common/build.gradle
@@ -31,6 +31,9 @@ dependencies {
testAnnotationProcessor depLibs.lombok
}
+jar {
+ archiveBaseName = 'stream-storage-common'
+}
publishing {
publications {
diff --git a/stream/server/build.gradle b/stream/server/build.gradle
index af36a7c..08f2403 100644
--- a/stream/server/build.gradle
+++ b/stream/server/build.gradle
@@ -16,7 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
+plugins {
+ id 'application'
+ id 'distribution'
+}
+
dependencies {
+ implementation project(':bookkeeper-common-allocator')
implementation project(':bookkeeper-common')
implementation project(':bookkeeper-server')
implementation project(':bookkeeper-stats')
@@ -30,19 +36,38 @@ dependencies {
implementation project(':stream:storage:api')
implementation project(':stream:storage:impl')
+ implementation project(':bookkeeper-stats-providers:prometheus-metrics-provider')
+ runtimeOnly project(':bookkeeper-tools-ledger')
+ runtimeOnly project(':bookkeeper-tools-stream')
+ runtimeOnly project(':bookkeeper-http:vertx-http-server')
implementation depLibs.commonsConfiguration
- implementation depLibs.commonsLang2
+ implementation depLibs.commonsLang3
implementation depLibs.curatorFramework
implementation depLibs.grpc
implementation depLibs.guava
implementation depLibs.jcommander
implementation depLibs.lombok
- implementation depLibs.slf4j
+ implementation depLibs.zookeeper
+ runtimeOnly depLibs.metricsCore
+ runtimeOnly depLibs.snappy
+ runtimeOnly depLibs.slf4jLog4j
+ runtimeOnly depLibs.commonsBeanutils
+ runtimeOnly depLibs.vertxCore
+ runtimeOnly depLibs.vertxWeb
testImplementation depLibs.mockito
annotationProcessor depLibs.lombok
}
+application {
+ mainClassName = "org.apache.bookkeeper.stream.cluster.StandaloneStarter"
+}
+
+task writeClasspath {
+ buildDir.mkdirs()
+ new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n"
+}
+
publishing {
publications {
maven(MavenPublication) {
diff --git a/stream/storage/api/build.gradle b/stream/storage/api/build.gradle
index 71cbe41..b4ff1ff 100644
--- a/stream/storage/api/build.gradle
+++ b/stream/storage/api/build.gradle
@@ -24,6 +24,10 @@ dependencies {
implementation depLibs.grpc
}
+jar {
+ archiveBaseName = 'stream-storage-service-api'
+}
+
publishing {
publications {
maven(MavenPublication) {
diff --git a/tests/docker-images/statestore-image/Dockerfile b/tests/docker-images/statestore-image/Dockerfile
new file mode 100644
index 0000000..15cfa9c
--- /dev/null
+++ b/tests/docker-images/statestore-image/Dockerfile
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM centos:7
+#ENV OPTS="$OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
+ENV BOOKIE_PORT=3181
+ENV BOOKIE_HTTP_PORT=8080
+ENV BOOKIE_GRPC_PORT=4181
+EXPOSE ${BOOKIE_PORT} ${BOOKIE_HTTP_PORT} ${BOOKIE_GRPC_PORT}
+ENV BK_USER=bookkeeper
+ENV BK_HOME=/opt/bookkeeper
+ENV JAVA_HOME=/usr/lib/jvm/jre-1.8.0
+
+
+RUN set -x \
+ && adduser "${BK_USER}" \
+ && yum install -y java-1.8.0-openjdk-headless wget bash sudo \
+ && wget -q https://bootstrap.pypa.io/pip/2.7/get-pip.py \
+ && python get-pip.py \
+ && pip install zk-shell \
+ && rm -rf get-pip.py \
+ && mkdir -pv /opt \
+ && cd /opt \
+ && yum clean all
+
+WORKDIR /opt/bookkeeper
+RUN mkdir /opt/bookkeeper/lib
+RUN mkdir /opt/bookkeeper/bin
+RUN mkdir /opt/bookkeeper/conf
+RUN mkdir /opt/bookkeeper/scripts
+
+### -----Copy Jars------###
+ADD ./dist/server.tar /opt/
+RUN mv /opt/server/lib/*.jar /opt/bookkeeper/lib/
+### --------------------###
+
+### ----Copy scripts----------###
+COPY ./scripts/* /opt/bookkeeper/scripts/
+COPY ./temp_bin/* /opt/bookkeeper/bin/
+COPY ./bin/* /opt/bookkeeper/bin/
+### ----Copy scripts----------###
+
+### ----Copy conf-------------###
+COPY ./temp_conf/* /opt/bookkeeper/conf/
+COPY ./conf/* /opt/bookkeeper/conf/
+### ----Copy conf-------------###
+
+RUN chmod +x -R /opt/bookkeeper/scripts
+RUN chmod +x -R /opt/bookkeeper/bin
+
+ENTRYPOINT [ "/bin/bash", "/opt/bookkeeper/scripts/entrypoint.sh" ]
+CMD ["bookie"]
+
+#HEALTHCHECK --interval=10s --timeout=50s CMD /bin/bash /opt/bookkeeper/scripts/healthcheck.sh
diff --git a/tests/docker-images/statestore-image/bin/bkctl b/tests/docker-images/statestore-image/bin/bkctl
new file mode 100755
index 0000000..139186a
--- /dev/null
+++ b/tests/docker-images/statestore-image/bin/bkctl
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# BookKeeper CLI (experimental)
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+
+source ${BK_HOME}/bin/common.sh
+source ${BK_HOME}/conf/bk_cli_env.sh
+
+# set up the classpath
+CLI_CLASSPATH=${BK_HOME}/lib/*
+
+DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf
+if [ -z "${CLI_CONF}" ]; then
+ CLI_CONF=${DEFAULT_CONF}
+fi
+
+DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.cli.properties
+if [ -z "${CLI_LOG_CONF}" ]; then
+ CLI_LOG_CONF=${DEFAULT_LOG_CONF}
+fi
+CLI_LOG_DIR=${CLI_LOG_DIR:-"$BK_HOME/logs"}
+CLI_LOG_FILE=${CLI_LOG_FILE:-"bkctl.log"}
+CLI_ROOT_LOGGER=${CLI_ROOT_LOGGER:-"INFO,ROLLINGFILE"}
+
+# Configure the classpath
+CLI_CLASSPATH="$CLI_JAR:$CLI_CLASSPATH:$CLI_EXTRA_CLASSPATH"
+CLI_CLASSPATH="`dirname $CLI_LOG_CONF`:$CLI_CLASSPATH"
+
+# Build the OPTs
+BOOKIE_OPTS=$(build_bookie_opts)
+GC_OPTS=$(build_cli_jvm_opts ${CLI_LOG_DIR} "bkctl-gc.log")
+NETTY_OPTS=$(build_netty_opts)
+LOGGING_OPTS=$(build_cli_logging_opts ${CLI_LOG_CONF} ${CLI_LOG_DIR} ${CLI_LOG_FILE} ${CLI_ROOT_LOGGER})
+
+OPTS="${OPTS} -cp ${CLI_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${CLI_EXTRA_OPTS}"
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+exec ${JAVA} ${OPTS} org.apache.bookkeeper.tools.cli.BKCtl --conf ${CLI_CONF} $@
diff --git a/tests/docker-images/statestore-image/bin/bkperf b/tests/docker-images/statestore-image/bin/bkperf
new file mode 100755
index 0000000..0bfba83
--- /dev/null
+++ b/tests/docker-images/statestore-image/bin/bkperf
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# BookKeeper Perf Tool (experimental)
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+
+source ${BK_HOME}/bin/common.sh
+source ${BK_HOME}/conf/bk_cli_env.sh
+
+CLI_CLASSPATH=${BK_HOME}/lib/*
+
+DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf
+if [ -z "${CLI_CONF}" ]; then
+ CLI_CONF=${DEFAULT_CONF}
+fi
+
+DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.cli.properties
+if [ -z "${CLI_LOG_CONF}" ]; then
+ CLI_LOG_CONF=${DEFAULT_LOG_CONF}
+fi
+CLI_LOG_DIR=${CLI_LOG_DIR:-"$BK_HOME/logs"}
+CLI_LOG_FILE=${CLI_LOG_FILE:-"bkperf.log"}
+CLI_ROOT_LOGGER=${CLI_ROOT_LOGGER:-"INFO,CONSOLE"}
+
+# Configure the classpath
+CLI_CLASSPATH="$CLI_CLASSPATH:$CLI_EXTRA_CLASSPATH"
+CLI_CLASSPATH="`dirname $CLI_LOG_CONF`:$CLI_CLASSPATH"
+
+# Build the OPTs
+BOOKIE_OPTS=$(build_bookie_opts)
+GC_OPTS=$(build_cli_jvm_opts ${CLI_LOG_DIR} "bkperf-gc.log")
+NETTY_OPTS=$(build_netty_opts)
+LOGGING_OPTS=$(build_cli_logging_opts ${CLI_LOG_CONF} ${CLI_LOG_DIR} ${CLI_LOG_FILE} ${CLI_ROOT_LOGGER})
+
+OPTS="${OPTS} -cp ${CLI_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${CLI_EXTRA_OPTS}"
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+exec ${JAVA} ${OPTS} org.apache.bookkeeper.tools.perf.BKPerf --conf ${CLI_CONF} $@
diff --git a/tests/docker-images/statestore-image/bin/bookkeeper b/tests/docker-images/statestore-image/bin/bookkeeper
new file mode 100755
index 0000000..55fcd72
--- /dev/null
+++ b/tests/docker-images/statestore-image/bin/bookkeeper
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -e
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+
+source ${BK_HOME}/bin/common.sh
+
+# default variables
+DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf
+DEFAULT_ZK_CONF=${BK_HOME}/conf/zookeeper.conf
+
+if [ -z "$BOOKIE_CONF" ]; then
+ BOOKIE_CONF_TO_CHECK=${DEFAULT_CONF}
+else
+ BOOKIE_CONF_TO_CHECK=${BOOKIE_CONF}
+fi
+
+# set up the classpath
+BOOKIE_CLASSPATH=${BK_HOME}/lib/*
+
+
+# get arguments
+COMMAND=$1
+shift
+
+LOCALBOOKIES_CONFIG_DIR="${LOCALBOOKIES_CONFIG_DIR:-/tmp/localbookies-config}"
+if [ ${COMMAND} == "shell" ]; then
+ DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.shell.properties
+ if [[ $1 == "-localbookie" ]]; then
+ if [[ $2 == *:* ]];
+ then
+ BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/$2.conf
+ shift 2
+ else
+ BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/baseconf.conf
+ shift
+ fi
+ fi
+fi
+
+if [ -z "$BOOKIE_ZK_CONF" ]; then
+ BOOKIE_ZK_CONF=$DEFAULT_ZK_CONF
+fi
+
+if [ -z "$BOOKIE_CONF" ]; then
+ BOOKIE_CONF=${DEFAULT_CONF}
+fi
+
+# Configure logging
+if [ -z "$BOOKIE_LOG_CONF" ]; then
+ BOOKIE_LOG_CONF=${DEFAULT_LOG_CONF}
+fi
+BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"$BK_HOME/logs"}
+BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"bookkeeper-server.log"}
+BOOKIE_ROOT_LOGGER=${BOOKIE_ROOT_LOGGER:-"INFO,CONSOLE"}
+
+# Configure the classpath
+BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH:$BOOKIE_EXTRA_CLASSPATH"
+BOOKIE_CLASSPATH="`dirname $BOOKIE_LOG_CONF`:$BOOKIE_CLASSPATH"
+
+echo ${BOOKIE_LOG_CONF}
+
+# Build the OPTS
+BOOKIE_OPTS=$(build_bookie_opts)
+GC_OPTS=$(build_bookie_jvm_opts ${BOOKIE_LOG_DIR} "gc_%p.log")
+NETTY_OPTS=$(build_netty_opts)
+LOGGING_OPTS=$(build_logging_opts ${BOOKIE_LOG_CONF} ${BOOKIE_LOG_DIR} ${BOOKIE_LOG_FILE} ${BOOKIE_ROOT_LOGGER})
+
+OPTS="${OPTS} -cp ${BOOKIE_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${BOOKIE_EXTRA_OPTS}"
+
+# Create log dir if it doesn't exist
+if [ ! -d ${BOOKIE_LOG_DIR} ]; then
+ mkdir ${BOOKIE_LOG_DIR}
+fi
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+if [ ${COMMAND} == "bookie" ]; then
+ exec ${JAVA} ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.server.Main --conf ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "autorecovery" ]; then
+ exec ${JAVA} ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.replication.AutoRecoveryMain --conf ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "localbookie" ]; then
+ NUMBER=$1
+ shift
+ exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.util.LocalBookKeeper ${NUMBER} ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "standalone" ]; then
+ exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.stream.cluster.StandaloneStarter --conf ${BK_HOME}/conf/standalone.conf $@
+elif [ ${COMMAND} == "upgrade" ]; then
+ exec ${JAVA} ${OPTS} org.apache.bookkeeper.bookie.FileSystemUpgrade --conf ${BOOKIE_CONF} $@
+elif [ $COMMAND == "zookeeper" ]; then
+ BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"zookeeper.log"}
+ exec $JAVA $OPTS -Dbookkeeper.log.file=$BOOKIE_LOG_FILE -Dzookeeper.4lw.commands.whitelist='*' org.apache.zookeeper.server.quorum.QuorumPeerMain $BOOKIE_ZK_CONF $@
+elif [ ${COMMAND} == "shell" ]; then
+ ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}"
+ exec ${JAVA} ${OPTS} ${ENTRY_FORMATTER_ARG} org.apache.bookkeeper.bookie.BookieShell -conf ${BOOKIE_CONF} $@
+else
+ exec ${JAVA} ${OPTS} ${COMMAND} $@
+fi
+
diff --git a/tests/docker-images/statestore-image/conf/log4j.cli.properties b/tests/docker-images/statestore-image/conf/log4j.cli.properties
new file mode 100644
index 0000000..6dcfe4c
--- /dev/null
+++ b/tests/docker-images/statestore-image/conf/log4j.cli.properties
@@ -0,0 +1,41 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+#
+# Log INFO level and above messages to the console
+#
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.Threshold=DEBUG
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ABSOLUTE} %-5p %m%n
+
+# verbose console logging
+log4j.appender.VERBOSECONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.VERBOSECONSOLE.Threshold=INFO
+log4j.appender.VERBOSECONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.VERBOSECONSOLE.layout.ConversionPattern=%m%n
+
+log4j.logger.verbose=INFO,VERBOSECONSOLE
+log4j.logger.org.apache.zookeeper=INFO
+log4j.logger.org.apache.bookkeeper=INFO
\ No newline at end of file
diff --git a/tests/docker-images/statestore-image/conf/log4j.properties b/tests/docker-images/statestore-image/conf/log4j.properties
new file mode 100644
index 0000000..cb1c53e
--- /dev/null
+++ b/tests/docker-images/statestore-image/conf/log4j.properties
@@ -0,0 +1,40 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+#
+# Log INFO level and above messages to the console
+#
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.Threshold=DEBUG
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ABSOLUTE} %-5p %m%n
+
+# verbose console logging
+log4j.appender.VERBOSECONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.VERBOSECONSOLE.Threshold=INFO
+log4j.appender.VERBOSECONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.VERBOSECONSOLE.layout.ConversionPattern=%m%n
+
+log4j.logger.verbose=INFO,VERBOSECONSOLE
+log4j.logger.org.apache.zookeeper=INFO
+log4j.logger.org.apache.bookkeeper=INFO
\ No newline at end of file
diff --git a/tests/docker-images/statestore-image/conf/log4j.shell.properties b/tests/docker-images/statestore-image/conf/log4j.shell.properties
new file mode 100644
index 0000000..6dcfe4c
--- /dev/null
+++ b/tests/docker-images/statestore-image/conf/log4j.shell.properties
@@ -0,0 +1,41 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+#
+# Log INFO level and above messages to the console
+#
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.Threshold=DEBUG
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ABSOLUTE} %-5p %m%n
+
+# verbose console logging
+log4j.appender.VERBOSECONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.VERBOSECONSOLE.Threshold=INFO
+log4j.appender.VERBOSECONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.VERBOSECONSOLE.layout.ConversionPattern=%m%n
+
+log4j.logger.verbose=INFO,VERBOSECONSOLE
+log4j.logger.org.apache.zookeeper=INFO
+log4j.logger.org.apache.bookkeeper=INFO
\ No newline at end of file
diff --git a/tests/docker-images/statestore-image/image_builder.sh b/tests/docker-images/statestore-image/image_builder.sh
new file mode 100755
index 0000000..91513bc
--- /dev/null
+++ b/tests/docker-images/statestore-image/image_builder.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+SCRIPT_DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+
+## BASE_DIR will be ./bookkeeper/
+BASE_DIR=${SCRIPT_DIR}/../../../
+mkdir "${BASE_DIR}"/tests/docker-images/statestore-image/dist
+mkdir "${BASE_DIR}"/tests/docker-images/statestore-image/scripts
+mkdir "${BASE_DIR}"/tests/docker-images/statestore-image/temp_conf
+mkdir "${BASE_DIR}"/tests/docker-images/statestore-image/temp_bin
+
+cp "${BASE_DIR}"/stream/server/build/distributions/server.tar "${BASE_DIR}"/tests/docker-images/statestore-image/dist
+cp "${BASE_DIR}"/docker/scripts/* "${BASE_DIR}"/tests/docker-images/statestore-image/scripts
+cp "${BASE_DIR}"/conf/* "${BASE_DIR}"/tests/docker-images/statestore-image/temp_conf
+cp "${BASE_DIR}"/bin/* "${BASE_DIR}"/tests/docker-images/statestore-image/temp_bin
+docker build -t apachebookkeeper/bookkeeper-current:latest "${BASE_DIR}"/tests/docker-images/statestore-image
+
+rm -rf "${BASE_DIR}"/tests/docker-images/statestore-image/dist
+rm -rf "${BASE_DIR}"/tests/docker-images/statestore-image/scripts
+rm -rf "${BASE_DIR}"/tests/docker-images/statestore-image/temp_conf
+rm -rf "${BASE_DIR}"/tests/docker-images/statestore-image/temp_bin
\ No newline at end of file
diff --git a/tests/integration/cluster/build.gradle b/tests/integration/cluster/build.gradle
index 417e6f3..f90ff20 100644
--- a/tests/integration/cluster/build.gradle
+++ b/tests/integration/cluster/build.gradle
@@ -16,6 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
+plugins {
+ id 'java'
+ id 'com.adarshr.test-logger' version '2.0.0'
+}
+
dependencies {
implementation depLibs.grpc
testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts')
@@ -29,6 +34,7 @@ dependencies {
testImplementation project(':bookkeeper-server')
testImplementation project(':bookkeeper-stats')
testImplementation project(':tests:integration-tests-topologies')
+ testImplementation project(':stream:clients:java:kv')
testCompileOnly depLibs.lombok
testImplementation depLibs.slf4jLog4j
@@ -37,6 +43,19 @@ dependencies {
testAnnotationProcessor depLibs.lombok
}
+task buildImage(type:Exec) {
+ workingDir '../../..'
+ commandLine './tests/docker-images/statestore-image/image_builder.sh'
+}
+
+test {
+ dependsOn buildImage
+ jvmArgs("-Djunit.timeout.test=600000",
+ "-Djunit.max.retry=3",
+ "-Djava.net.preferIPv4Stack=true",
+ "-Dio.netty.leakDetection.level=paranoid")
+}
+
publishing {
publications {
maven(MavenPublication) {
diff --git a/tests/integration/smoke/build.gradle b/tests/integration/smoke/build.gradle
index f19a72c..742f5dc 100644
--- a/tests/integration/smoke/build.gradle
+++ b/tests/integration/smoke/build.gradle
@@ -16,6 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
+plugins {
+ id 'java'
+ id 'com.adarshr.test-logger' version '2.0.0'
+}
+
dependencies {
testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts')
testImplementation project(':bookkeeper-server')
diff --git a/tests/integration/standalone/build.gradle b/tests/integration/standalone/build.gradle
index 0058d7e..f8f0751 100644
--- a/tests/integration/standalone/build.gradle
+++ b/tests/integration/standalone/build.gradle
@@ -16,6 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
+plugins {
+ id 'java'
+ id 'com.adarshr.test-logger' version '2.0.0'
+}
+
dependencies {
testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts')
testImplementation project(':bookkeeper-server')
@@ -29,6 +34,15 @@ dependencies {
testAnnotationProcessor depLibs.lombok
}
+task buildImage(type:Exec) {
+ workingDir '../../..'
+ commandLine './tests/docker-images/statestore-image/image_builder.sh'
+}
+
+test {
+ dependsOn buildImage
+}
+
publishing {
publications {
maven(MavenPublication) {
diff --git a/tools/framework/build.gradle b/tools/framework/build.gradle
index 28e637a..c188a1c 100644
--- a/tools/framework/build.gradle
+++ b/tools/framework/build.gradle
@@ -32,3 +32,7 @@ dependencies {
annotationProcessor depLibs.lombok
}
+
+jar {
+ archiveBaseName = 'bookkeeper-tools-framework'
+}
\ No newline at end of file
diff --git a/tools/framework/build.gradle b/tools/ledger/build.gradle
similarity index 68%
copy from tools/framework/build.gradle
copy to tools/ledger/build.gradle
index 28e637a..23334dd 100644
--- a/tools/framework/build.gradle
+++ b/tools/ledger/build.gradle
@@ -22,13 +22,31 @@ plugins {
dependencies {
implementation project(":bookkeeper-common")
+ implementation project(':bookkeeper-tools-framework')
+ implementation project(':bookkeeper-server')
+
compileOnly depLibs.lombok
compileOnly depLibs.spotbugsAnnotations
+
implementation depLibs.commonsConfiguration
implementation depLibs.guava
implementation depLibs.jcommander
implementation depLibs.slf4j
+ implementation depLibs.zookeeper
+
+ testImplementation project(":bookkeeper-stats")
+ testCompileOnly depLibs.lombok
testImplementation depLibs.junit
+ testImplementation depLibs.commonsConfiguration
+ testImplementation depLibs.mockito
+ testImplementation depLibs.powermockJunit
+ testImplementation depLibs.powermockMockito
+ testImplementation depLibs.zookeeperTest
annotationProcessor depLibs.lombok
+ testAnnotationProcessor depLibs.lombok
+}
+
+jar {
+ archiveBaseName = 'bookkeeper-tools-ledger'
}
diff --git a/tools/ledger/src/main/resources/META-INF/services/org.apache.bookkeeper.tools.framework.CommandGroup b/tools/ledger/src/main/resources/META-INF/services/org.apache.bookkeeper.tools.framework.CommandGroup
new file mode 100644
index 0000000..d81ba5a
--- /dev/null
+++ b/tools/ledger/src/main/resources/META-INF/services/org.apache.bookkeeper.tools.framework.CommandGroup
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.bookkeeper.tools.cli.commands.BookieCommandGroup
+org.apache.bookkeeper.tools.cli.commands.BookieIdCommandGroup
+org.apache.bookkeeper.tools.cli.commands.BookiesCommandGroup
+org.apache.bookkeeper.tools.cli.commands.CookieCommandGroup
+org.apache.bookkeeper.tools.cli.commands.LedgerCommandGroup
+org.apache.bookkeeper.tools.cli.commands.AutoRecoveryCommandGroup
diff --git a/stream/server/build.gradle b/tools/perf/build.gradle
similarity index 56%
copy from stream/server/build.gradle
copy to tools/perf/build.gradle
index af36a7c..b3a68c3 100644
--- a/stream/server/build.gradle
+++ b/tools/perf/build.gradle
@@ -16,37 +16,55 @@
* specific language governing permissions and limitations
* under the License.
*/
+plugins {
+ id 'java'
+}
+
dependencies {
- implementation project(':bookkeeper-common')
+
+ implementation project(":bookkeeper-common")
+ implementation project(':bookkeeper-tools-framework')
implementation project(':bookkeeper-server')
implementation project(':bookkeeper-stats')
- implementation project(':stream:clients:java:all')
- implementation project(':stream:clients:java:base')
+ implementation project(':bookkeeper-stats-providers:prometheus-metrics-provider')
+ implementation project(':bookkeeper-common-allocator')
+ implementation project(':stream:api')
implementation project(':stream:common')
+ implementation project(':stream:clients:java:base')
+ implementation project(':stream:clients:java:all')
+ implementation project(':stream:storage:api')
+ implementation project(':stream:storage:impl')
+ implementation project(':stream:distributedlog:common')
implementation project(':stream:distributedlog:core')
implementation project(':stream:distributedlog:protocol')
implementation project(':stream:proto')
- implementation project(':stream:statelib')
- implementation project(':stream:storage:api')
- implementation project(':stream:storage:impl')
+ compileOnly depLibs.lombok
implementation depLibs.commonsConfiguration
- implementation depLibs.commonsLang2
- implementation depLibs.curatorFramework
- implementation depLibs.grpc
implementation depLibs.guava
implementation depLibs.jcommander
- implementation depLibs.lombok
- implementation depLibs.slf4j
- testImplementation depLibs.mockito
+ implementation depLibs.slf4jLog4j
+ implementation depLibs.zookeeper
+ implementation depLibs.curatorFramework
+ implementation depLibs.protobuf
+ implementation depLibs.jacksonDatabind
+ implementation depLibs.hdrHistogram
+ implementation depLibs.nettyBuffer
+ implementation depLibs.commonsLang3
+ compileOnly depLibs.spotbugsAnnotations
+ testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts')
+ testImplementation project(':bookkeeper-http:vertx-http-server')
+ testImplementation depLibs.junit
+ testImplementation depLibs.commonsConfiguration
+ testImplementation depLibs.mockito
+ testImplementation depLibs.powermockJunit
+ testImplementation depLibs.powermockMockito
+ testImplementation depLibs.zookeeperTest
annotationProcessor depLibs.lombok
+ testAnnotationProcessor depLibs.lombok
}
-publishing {
- publications {
- maven(MavenPublication) {
- artifactId = 'stream-storage-server'
- }
- }
+jar {
+ archiveBaseName = 'bookkeeper-tools-perf'
}
diff --git a/stream/server/build.gradle b/tools/stream/build.gradle
similarity index 71%
copy from stream/server/build.gradle
copy to tools/stream/build.gradle
index af36a7c..2f7276c 100644
--- a/stream/server/build.gradle
+++ b/tools/stream/build.gradle
@@ -16,37 +16,40 @@
* specific language governing permissions and limitations
* under the License.
*/
+plugins {
+ id 'java'
+}
+
dependencies {
- implementation project(':bookkeeper-common')
+ implementation project(":bookkeeper-common")
+ implementation project(':bookkeeper-tools-framework')
implementation project(':bookkeeper-server')
- implementation project(':bookkeeper-stats')
- implementation project(':stream:clients:java:all')
- implementation project(':stream:clients:java:base')
+ implementation project(':stream:api')
implementation project(':stream:common')
+ implementation project(':stream:clients:java:base')
+ implementation project(':stream:clients:java:kv')
+ implementation project(':stream:clients:java:all')
+ implementation project(':stream:storage:api')
+ implementation project(':stream:storage:impl')
implementation project(':stream:distributedlog:core')
implementation project(':stream:distributedlog:protocol')
+ implementation project(':stream:distributedlog:core')
implementation project(':stream:proto')
- implementation project(':stream:statelib')
- implementation project(':stream:storage:api')
- implementation project(':stream:storage:impl')
+ compileOnly depLibs.lombok
+ compileOnly depLibs.spotbugsAnnotations
implementation depLibs.commonsConfiguration
- implementation depLibs.commonsLang2
- implementation depLibs.curatorFramework
- implementation depLibs.grpc
implementation depLibs.guava
implementation depLibs.jcommander
- implementation depLibs.lombok
- implementation depLibs.slf4j
- testImplementation depLibs.mockito
-
+ implementation depLibs.slf4jLog4j
+ implementation depLibs.zookeeper
+ implementation depLibs.curatorFramework
+ implementation depLibs.protobuf
+ implementation depLibs.commonsLang3
+ implementation depLibs.nettyBuffer
annotationProcessor depLibs.lombok
}
-publishing {
- publications {
- maven(MavenPublication) {
- artifactId = 'stream-storage-server'
- }
- }
-}
+jar {
+ archiveBaseName = 'bookkeeper-tools-stream'
+}
\ No newline at end of file
diff --git a/tools/stream/src/main/resources/META-INF/services/org.apache.bookkeeper.tools.framework.CommandGroup b/tools/stream/src/main/resources/META-INF/services/org.apache.bookkeeper.tools.framework.CommandGroup
new file mode 100644
index 0000000..070efbf
--- /dev/null
+++ b/tools/stream/src/main/resources/META-INF/services/org.apache.bookkeeper.tools.framework.CommandGroup
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.bookkeeper.stream.cli.ClusterCommandGroup
+org.apache.bookkeeper.stream.cli.NamespaceCommandGroup
+org.apache.bookkeeper.stream.cli.TableAdminCommandGroup
+org.apache.bookkeeper.stream.cli.TableCommandGroup