You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2014/02/15 01:53:29 UTC
git commit: Added an example java log test.
Repository: mesos
Updated Branches:
refs/heads/master 765c938f5 -> 8a4ec2e09
Added an example java log test.
Review: https://reviews.apache.org/r/17780
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8a4ec2e0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8a4ec2e0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8a4ec2e0
Branch: refs/heads/master
Commit: 8a4ec2e09097b856715242d7eca44ada5f145eed
Parents: 765c938
Author: Vinod Kone <vi...@twitter.com>
Authored: Sun Feb 2 13:18:34 2014 -0800
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Feb 14 16:53:07 2014 -0800
----------------------------------------------------------------------
configure.ac | 2 +
src/Makefile.am | 10 +-
src/examples/java/TestLog.java | 234 ++++++++++++++++++++++++++++++++++++
src/examples/java/test-log.in | 44 +++++++
src/tests/examples_tests.cpp | 1 +
src/tests/java_log_test.sh | 34 ++++++
6 files changed, 322 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8a4ec2e0/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index bb81279..c21e681 100644
--- a/configure.ac
+++ b/configure.ac
@@ -408,6 +408,8 @@ __EOF__
[chmod +x src/examples/java/test-framework])
AC_CONFIG_FILES([src/examples/java/test-multiple-executors-framework],
[chmod +x src/examples/java/test-multiple-executors-framework])
+ AC_CONFIG_FILES([src/examples/java/test-log],
+ [chmod +x src/examples/java/test-log])
AC_CONFIG_FILES([src/java/mesos.pom])
AC_DEFINE([MESOS_HAS_JAVA])
http://git-wip-us.apache.org/repos/asf/mesos/blob/8a4ec2e0/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index aa8bb2b..9a33f21 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -27,6 +27,7 @@ include ../3rdparty/libprocess/3rdparty/versions.am
DISTRIBUTE = 3rdparty/distribute-$(DISTRIBUTE_VERSION)
LEVELDB = 3rdparty/leveldb
ZOOKEEPER = 3rdparty/zookeeper-$(ZOOKEEPER_VERSION)/src/c
+ZOOKEEPER_JAR = 3rdparty/zookeeper-$(ZOOKEEPER_VERSION)/zookeeper-$(ZOOKEEPER_VERSION).jar
LIBPROCESS = 3rdparty/libprocess
STOUT = $(LIBPROCESS)/3rdparty/stout
BOOST = $(LIBPROCESS)/3rdparty/boost-$(BOOST_VERSION)
@@ -555,7 +556,8 @@ EXAMPLES_SOURCE = \
$(srcdir)/examples/java/TestExceptionFramework.java \
$(srcdir)/examples/java/TestExecutor.java \
$(srcdir)/examples/java/TestFramework.java \
- $(srcdir)/examples/java/TestMultipleExecutorsFramework.java
+ $(srcdir)/examples/java/TestMultipleExecutorsFramework.java \
+ $(srcdir)/examples/java/TestLog.java
EXTRA_DIST += $(EXAMPLES_SOURCE)
@@ -696,7 +698,7 @@ $(EXAMPLES_JAR): $(EXAMPLES_SOURCE)
@echo "Building examples.jar ..."
$(MKDIR_P) examples/java
$(JAVA_HOME)/bin/javac -source 1.6 -target 1.6 \
- -cp $(PROTOBUF_JAR):$(MESOS_JAR):$(srcdir)/examples/java \
+ -cp ../$(ZOOKEEPER_JAR):$(PROTOBUF_JAR):$(MESOS_JAR):$(srcdir)/examples/java \
-sourcepath $(srcdir)/examples/java -d examples/java \
$(srcdir)/examples/java/*.java
$(JAVA_HOME)/bin/jar cf $@ -C examples/java .
@@ -905,7 +907,8 @@ if HAS_JAVA
EXAMPLESSCRIPTSJAVA = examples/java/test-framework \
examples/java/test-executor \
examples/java/test-exception-framework \
- examples/java/test-multiple-executors-framework
+ examples/java/test-multiple-executors-framework \
+ examples/java/test-log
check_SCRIPTS += $(EXAMPLESCRIPTSJAVA)
mesos_tests_DEPENDENCIES += $(EXAMPLESCRIPTSJAVA)
@@ -933,6 +936,7 @@ dist_check_SCRIPTS += \
tests/no_executor_framework_test.sh \
tests/java_exception_test.sh \
tests/java_framework_test.sh \
+ tests/java_log_test.sh \
tests/python_framework_test.sh
# We use a check-local target for now to avoid the parallel test
http://git-wip-us.apache.org/repos/asf/mesos/blob/8a4ec2e0/src/examples/java/TestLog.java
----------------------------------------------------------------------
diff --git a/src/examples/java/TestLog.java b/src/examples/java/TestLog.java
new file mode 100644
index 0000000..9dd4630
--- /dev/null
+++ b/src/examples/java/TestLog.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+import org.apache.mesos.*;
+import org.apache.mesos.Protos.*;
+
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.ZooKeeperServer.BasicDataTreeBuilder;
+import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
+
+
+// This could be used both in local mode (to include in unit tests)
+// and in distributed mode (as a benchmark test) based on the command
+// line flags passed to it.
+//
+// Local: <zk_url> = "local"
+// In this mode a ZooKeeper server and *all* the replicas
+// are spawned locally in process.
+//
+// Distributed: <zk_url> != "local"
+// In this mode only one replica is spawned locally. The
+// ZooKeeper URL is used to discover other replicas.
+//
+// If "<load_file>" is provided the replica will continously append
+// as many random chunks of data as there are lines in the file. The
+// format of the file is one number per line where the number
+// represents the size of the chunk in bytes.
+public class TestLog {
+
+ private static final Logger LOG = Logger.getLogger(TestLog.class.getName());
+
+ private static ZooKeeperTestServer zkserver = null;
+
+ static class ZooKeeperTestServer {
+ private ZooKeeperServer server = null;
+ private NIOServerCnxnFactory connection = null;
+ private String logdir;
+
+ ZooKeeperTestServer(String dir) {
+ logdir = dir;
+ }
+
+ private InetSocketAddress start() throws IOException, InterruptedException {
+ int port = 0; // Ephemeral port.
+ int maxconnections = 1000;
+
+ File dataFile = new File(logdir, "zookeeper_data").getAbsoluteFile();
+ File snapFile = new File(logdir, "zookeeper_snap").getAbsoluteFile();
+
+ server = new ZooKeeperServer(
+ new FileTxnSnapLog(dataFile, snapFile),
+ new BasicDataTreeBuilder());
+
+ connection = new NIOServerCnxnFactory();
+ connection.configure(new InetSocketAddress(port), maxconnections);
+ connection.startup(server);
+
+ return connection.getLocalAddress();
+ }
+
+ private void stop() {
+ if (connection != null) {
+ connection.shutdown();
+ }
+ }
+ }
+
+ private static void usage() {
+ String name = TestLog.class.getName();
+ LOG.severe("Usage: " + name + " <zk> <quorum_size> <log_dir> <load_file>");
+ }
+
+ private static int system(String command)
+ throws IOException, InterruptedException {
+ Runtime r = Runtime.getRuntime();
+ Process p = r.exec(command);
+ p.waitFor();
+ return p.exitValue();
+ }
+
+ private static void exit(int status) {
+ if (zkserver != null) {
+ zkserver.stop();
+ }
+ System.exit(status);
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 3) {
+ usage();
+ exit(1);
+ }
+
+ // Get command line parameters.
+ String zkurl = args[0];
+ int quorum = Integer.parseInt(args[1]);
+ String logdir = args[2];
+ String loadfile = args.length > 3 ? args[3] : null;
+
+ String servers = null;
+ String znode = null;
+ int local_replicas = 0;
+ if (zkurl.equals("local")) {
+ LOG.info("Starting a local ZooKeeper server");
+
+ zkserver = new ZooKeeperTestServer(logdir);
+ InetSocketAddress address = zkserver.start();
+ servers = address.getHostName() + ":" + address.getPort();
+ znode = "/log";
+ local_replicas = (2*quorum) - 1; // Start all replicas locally.
+ } else {
+ servers = zkurl.substring(0, zkurl.indexOf("/"));
+ znode = zkurl.substring(zkurl.indexOf("/"), zkurl.length());
+ local_replicas = 1;
+
+ LOG.info("Connecting to ZooKeeper server " + servers + znode);
+ }
+
+ long timeout = 3;
+ List<Log> logs = new ArrayList<Log>();
+ for (int replica = 1; replica <= local_replicas; replica++) {
+ String log = logdir + "/log" + replica;
+
+ String logtool = System.getenv("MESOS_LOG_TOOL");
+ if (logtool != null) {
+ // Initialize the log.
+ // TODO(vinod): Do this via Java API once log tool has one.
+ LOG.info("Initializing log " + log + " with " + logtool);
+ int status = system(logtool + " initialize --path=" + log);
+ if (status != 0) {
+ LOG.severe("Error initializing log '" + log + "': " + status);
+ exit(1);
+ }
+ } else {
+ // TODO(vinod): Kill this once we don't care about log
+ // version < 0.17.0.
+ LOG.warning("Not initializing log file");
+ }
+
+ logs.add(new Log(quorum, log, servers, timeout, TimeUnit.SECONDS, znode));
+ }
+
+ // Write some data.
+ if (loadfile != null) {
+ LOG.info("Initializing writer");
+
+ // First read the sizes.
+ ArrayList<Integer> sizes = new ArrayList<Integer>(10000);
+ try {
+ BufferedReader in = new BufferedReader(new FileReader(loadfile));
+ String str;
+ while ((str = in.readLine()) != null) {
+ sizes.add(Integer.parseInt(str));
+ }
+ in.close();
+ } catch (IOException e) {
+ LOG.severe("Error while reading from file: " + e.getMessage());
+ exit(1);
+ } catch (NumberFormatException e) {
+ LOG.severe("Error while reading size from file: " + e.getMessage());
+ exit(1);
+ }
+
+ // Now write the data of given sizes.
+ int retries = 3;
+ Log.Writer writer = new Log.Writer(
+ logs.get(0), timeout, TimeUnit.SECONDS, retries);
+ try {
+ for(int size : sizes) {
+ byte[] data = new byte[size];
+ new Random().nextBytes(data); // Write random bytes.
+
+ long startTime = System.nanoTime();
+ writer.append(data, timeout, TimeUnit.SECONDS);
+ long duration = System.nanoTime() - startTime;
+
+ LOG.info("Time: " + System.currentTimeMillis() +
+ " Appended " + size + " bytes in " + duration + " ns");
+ }
+ } catch (TimeoutException e) {
+ LOG.severe("Timed out writing to log: " + e.getMessage());
+ exit(1);
+ } catch (Log.WriterFailedException e) {
+ LOG.severe("Error writing to log: " + e.getMessage());
+ exit(1);
+ }
+ } else if (!zkurl.equals("local")) {
+ // We are here if this is a remote replica that is not a writer.
+ while(true) { // Wait until interrupted.
+ try {
+ LOG.info("Sleeping...Press Ctrl+C to exit");
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ exit(1);
+ }
+ }
+ } else {
+ LOG.severe("Expecting load file in local mode");
+ exit(1);
+ }
+
+ exit(0); // Success.
+ }
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/8a4ec2e0/src/examples/java/test-log.in
----------------------------------------------------------------------
diff --git a/src/examples/java/test-log.in b/src/examples/java/test-log.in
new file mode 100644
index 0000000..b7e69e4
--- /dev/null
+++ b/src/examples/java/test-log.in
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# This script uses MESOS_SOURCE_DIR and MESOS_BUILD_DIR which come
+# from configuration substitutions.
+MESOS_SOURCE_DIR=@abs_top_srcdir@
+MESOS_BUILD_DIR=@abs_top_builddir@
+
+# Locate Java from environment or use configure discovered location.
+JAVA_HOME=${JAVA_HOME-@JAVA_HOME@}
+JAVA=${JAVA-${JAVA_HOME}/bin/java}
+
+# Use colors for errors.
+. ${MESOS_SOURCE_DIR}/support/colors.sh
+
+# TODO(vinod): Deduce the protobuf version.
+PROTOBUF_JAR=${MESOS_BUILD_DIR}/protobuf-2.5.0.jar
+
+test ! -e ${PROTOBUF_JAR} && \
+ echo "${RED}Failed to find ${PROTOBUF_JAR}${NORMAL}" && \
+ exit 1
+
+# TODO(vinod): Deduce the zookeeper version.
+ZOOKEEPER=${MESOS_BUILD_DIR}/3rdparty/zookeeper-3.4.5
+ZOOKEEPER_JAR=${ZOOKEEPER}/zookeeper-3.4.5.jar
+
+test ! -e ${ZOOKEEPER_JAR} && \
+ echo "${RED}Failed to find ${ZOOKEEPER_JAR}${NORMAL}" && \
+ exit 1
+
+MESOS_JAR=${MESOS_BUILD_DIR}/src/mesos-@PACKAGE_VERSION@.jar
+
+test ! -e ${MESOS_JAR} && \
+ echo "${RED}Failed to find ${MESOS_JAR}${NORMAL}" && \
+ exit 1
+
+EXAMPLES_JAR=${MESOS_BUILD_DIR}/src/examples.jar
+
+test ! -e ${EXAMPLES_JAR} && \
+ echo "${RED}Failed to find ${EXAMPLES_JAR}${NORMAL}" && \
+ exit 1
+
+exec ${JAVA} -cp "${PROTOBUF_JAR}:${MESOS_JAR}:${EXAMPLES_JAR}:${ZOOKEEPER_JAR}:${ZOOKEEPER}/lib/*" \
+ -Djava.library.path=${MESOS_BUILD_DIR}/src/.libs \
+ TestLog "${@}"
http://git-wip-us.apache.org/repos/asf/mesos/blob/8a4ec2e0/src/tests/examples_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/examples_tests.cpp b/src/tests/examples_tests.cpp
index 28ff0f3..34f0233 100644
--- a/src/tests/examples_tests.cpp
+++ b/src/tests/examples_tests.cpp
@@ -28,6 +28,7 @@ TEST_SCRIPT(ExamplesTest, NoExecutorFramework, "no_executor_framework_test.sh")
#ifdef MESOS_HAS_JAVA
TEST_SCRIPT(ExamplesTest, JavaFramework, "java_framework_test.sh")
TEST_SCRIPT(ExamplesTest, JavaException, "java_exception_test.sh")
+TEST_SCRIPT(ExamplesTest, JavaLog, "java_log_test.sh")
#endif
#ifdef MESOS_HAS_PYTHON
http://git-wip-us.apache.org/repos/asf/mesos/blob/8a4ec2e0/src/tests/java_log_test.sh
----------------------------------------------------------------------
diff --git a/src/tests/java_log_test.sh b/src/tests/java_log_test.sh
new file mode 100755
index 0000000..5451725
--- /dev/null
+++ b/src/tests/java_log_test.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Expecting MESOS_SOURCE_DIR and MESOS_BUILD_DIR to be in environment.
+
+env | grep MESOS_SOURCE_DIR >/dev/null
+
+test $? != 0 && \
+ echo "Failed to find MESOS_SOURCE_DIR in environment" && \
+ exit 1
+
+env | grep MESOS_BUILD_DIR >/dev/null
+
+test $? != 0 && \
+ echo "Failed to find MESOS_BUILD_DIR in environment" && \
+ exit 1
+
+source ${MESOS_SOURCE_DIR}/support/atexit.sh
+
+ZK_URL="local"
+QUORUM=2
+
+LOG_DIR=`mktemp -d -t mesos-XXXXXX`
+atexit "rm -rf ${LOG_DIR}"
+
+LOAD_FILE="${LOG_DIR}/load"
+touch ${LOAD_FILE}
+echo "1024" >> ${LOAD_FILE}
+echo "10240" >> ${LOAD_FILE}
+echo "102400" >> ${LOAD_FILE}
+
+export MESOS_LOG_TOOL=${MESOS_BUILD_DIR}/src/mesos-log
+
+# Check that the Java log executes without crashing (returns 0).
+exec $MESOS_BUILD_DIR/src/examples/java/test-log ${ZK_URL} ${QUORUM} ${LOG_DIR} ${LOAD_FILE}