You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2017/01/21 02:19:12 UTC
kudu git commit: [kudu-jepsen] added Jenkins script
Repository: kudu
Updated Branches:
refs/heads/master fb3bbbc8c -> c8ca20fa7
[kudu-jepsen] added Jenkins script
Added a shell script to build Kudu and run kudu-jepsen tests from a
Jenkins job.
Currently, the set of nodes to run Kudu cluster should be provisioned
prior to running the test. The test uses the Jenkins slave as the
Jepsen control node, running the control logic and the freshly built
Kudu Java client there. Since the clients are run there too, the
bandwidth and latency between the Jepsen and the DB nodes
should be taken into consideration while running the tests.
All Jepsen control operations on the DB nodes (i.e. Kudu master and
tserver nodes) are run via SSH. Appropriate SSH key should be deployed
prior to running the test.
Restriction:
The Kudu jepsen cluster and the jenkins slave must run recent
Debian-based distros. The kudu-jepsen test currently requires Debian
and the jenkins slave must be able to build compatible binaries.
Change-Id: I36a7e890baabb5427b22daa3aeee58ed894b83d6
Reviewed-on: http://gerrit.cloudera.org:8080/5624
Reviewed-by: David Ribeiro Alves <dr...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c8ca20fa
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c8ca20fa
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c8ca20fa
Branch: refs/heads/master
Commit: c8ca20fa7be5e2a934f10df9f748a5556a8ad45b
Parents: fb3bbbc
Author: Alexey Serbin <as...@cloudera.com>
Authored: Wed Jan 4 19:08:59 2017 -0800
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Sat Jan 21 01:42:04 2017 +0000
----------------------------------------------------------------------
src/kudu/scripts/jepsen.sh | 140 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 140 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/c8ca20fa/src/kudu/scripts/jepsen.sh
----------------------------------------------------------------------
diff --git a/src/kudu/scripts/jepsen.sh b/src/kudu/scripts/jepsen.sh
new file mode 100755
index 0000000..f731eb6
--- /dev/null
+++ b/src/kudu/scripts/jepsen.sh
@@ -0,0 +1,140 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This script is invoked from a Jenkins job to build Kudu C++ components,
+# Kudu Java client, kudu-jepsen, and run kudu-jepsen consistency tests
+# against the freshly built Kudu binaries.
+#
+# The following environment variables may be used to customize the operation:
+# JAVA8_HOME Default: /usr/lib/jvm/java-8-openjdk-amd64
+# Path to the JDK8 installation root. Expecting to find 'java'
+# in the 'bin' sub-directory. Java 8 or higher is required for the
+# maven-clojure-plugin.
+#
+# M2_HOME Default: /usr/share/maven
+# Path to the maven (3.3.6 and newer is required).
+#
+# MVN_FLAGS Default: ""
+# Extra flags which are passed to 'mvn' when building and running Java
+# tests. This can be useful, for example, to choose a different maven
+# repository location.
+#
+# KUDU_MASTER_NODES Default: ""
+# Set of master nodes for the Kudu cluster to run the Jepsen consistency
+# test. Those should be the same architecture and OS type as the
+# Jenkins machine where this build script is run.
+# NOTE: currently, the Jepsen test can run only at Debian nodes.
+#
+# KUDU_TSERVER_NODES Default: ""
+# Set of tablet server nodes for the Kudu cluster to run the Jepsen
+# consistency test. Those should be the same architecture and OS type
+# as the Jenkins machine where this build script is run.
+# NOTE: currently, the Jepsen test can run only at Debian nodes.
+#
+# SSH_KEY Default: ""
+# The absolute path to the file containing the private SSH key
+# for the root user at the master and tablet server nodes
+# (DB nodes in Jepsen terminology).
+# NOTE: the key should be in plain (i.e. non-encrypted) format.
+#
+
+#############################################################################
+# Constants
+#############################################################################
+
+# Require Kudu binaries to be statically linked with the Kudu-specific libs:
+# the kudu-jepsen does not distribute Kudu libraries.
+KUDU_LINK=static
+
+SRC_ROOT=$(readlink -f $(dirname $0)/../../..)
+THIRDPARTY_BIN=$SRC_ROOT/thirdparty/installed/common/bin
+NUM_PROCS=$(getconf _NPROCESSORS_ONLN)
+
+#############################################################################
+# Customizable parameters
+#############################################################################
+
+BUILD_TYPE=${BUILD_TYPE:-debug}
+JAVA8_HOME=${JAVA8_HOME:-/usr/lib/jvm/java-8-openjdk-amd64}
+M2_HOME=${M2_HOME:-/usr/share/maven}
+MVN_FLAGS=${MVN_FLAGS:-}
+KUDU_MASTER_NODES=${KUDU_MASTER_NODES:-}
+KUDU_TSERVER_NODES=${KUDU_TSERVER_NODES:-}
+SSH_KEY=${SSH_KEY:-}
+ITER_NUM=${ITER_NUM:-1}
+
+#############################################################################
+# Main
+#############################################################################
+
+set -e
+set -o pipefail
+ulimit -c unlimited
+
+cd $SRC_ROOT
+
+echo
+echo "Removing logs from prior runs"
+echo ------------------------------------------------------------
+rm -rf java/kudu-jepsen/store
+
+echo
+echo "Building third-party components"
+echo ------------------------------------------------------------
+$SRC_ROOT/build-support/enable_devtoolset.sh \
+ $SRC_ROOT/thirdparty/build-if-necessary.sh
+
+mkdir -p build/$BUILD_TYPE
+pushd build
+ln -sf $SRC_ROOT/build/$BUILD_TYPE latest
+pushd $BUILD_TYPE
+
+echo
+echo "Building Kudu C++ components (no tests)"
+echo ------------------------------------------------------------
+$SRC_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake \
+ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+ -DKUDU_LINK=$KUDU_LINK \
+ -DNO_TESTS=1 \
+ ../..
+make -j$NUM_PROCS 2>&1 | tee build.log
+
+export JAVA_HOME=$JAVA8_HOME
+export PATH=$JAVA_HOME/bin:$M2_HOME/bin:$PATH
+set -x
+
+pushd $SRC_ROOT/java
+echo
+echo "Building Kudu Java client and kudu-jepsen"
+echo ------------------------------------------------------------
+mvn $MVN_FLAGS -Pjepsen clean compile test-compile
+pushd kudu-jepsen
+
+echo
+echo "Building kudu-jepsen and running the consistency tests"
+echo ------------------------------------------------------------
+mvn $MVN_FLAGS clojure:run \
+ -DmasterNodes="$KUDU_MASTER_NODES" \
+ -DtserverNodes="$KUDU_TSERVER_NODES" \
+ -DsshKeyPath="$SSH_KEY" \
+ -DiterNum="$ITER_NUM"
+popd
+popd
+
+set +x