You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/09 22:12:05 UTC
[1/2] accumulo-testing git commit: ACCUMULO-4510 Created
'accumulo-testing' command
Repository: accumulo-testing
Updated Branches:
refs/heads/master ac5b271ca -> 4f8220640
ACCUMULO-4510 Created 'accumulo-testing' command
* Runs randomwalk locally or builds shaded jar that is submitted to YARN
* Added documentation for random walk
Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/efaa7377
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/efaa7377
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/efaa7377
Branch: refs/heads/master
Commit: efaa7377c88a4716a7abf8f30e025ee7d68d6499
Parents: ac5b271
Author: Mike Walch <mw...@apache.org>
Authored: Wed Dec 28 14:09:53 2016 -0500
Committer: Mike Walch <mw...@apache.org>
Committed: Mon Jan 9 16:21:03 2017 -0500
----------------------------------------------------------------------
.travis.yml | 22 +++
README.md | 53 ++++++
bin/accumulo-testing | 101 +++++++++++
conf/.gitignore | 2 +
conf/accumulo-testing-env.sh.example | 25 +++
conf/accumulo-testing.properties.example | 44 +++++
conf/log4j.properties.example | 27 +++
core/pom.xml | 4 -
.../apache/accumulo/testing/core/TestProps.java | 53 ++++++
.../testing/core/randomwalk/Environment.java | 49 +-----
.../testing/core/randomwalk/Framework.java | 22 +--
.../testing/core/randomwalk/Module.java | 16 +-
.../core/randomwalk/multitable/CopyTable.java | 3 +-
.../randomwalk/sequential/MapRedVerify.java | 3 +-
.../main/resources/randomwalk/modules/Bulk.xml | 2 +-
.../resources/randomwalk/modules/Concurrent.xml | 2 +-
.../randomwalk/modules/Conditional.xml | 3 +-
.../main/resources/randomwalk/modules/Image.xml | 2 +-
.../resources/randomwalk/modules/MultiTable.xml | 2 +-
.../resources/randomwalk/modules/Security.xml | 2 +-
.../resources/randomwalk/modules/Sequential.xml | 2 +-
.../main/resources/randomwalk/modules/Shard.xml | 2 +-
.../resources/randomwalk/modules/unit/Basic.xml | 2 +-
.../randomwalk/modules/unit/Simple.xml | 2 +-
pom.xml | 47 ++++++
yarn/.gitignore | 2 +
yarn/pom.xml | 110 ++++++++++++
.../testing/yarn/YarnAccumuloTestRunner.java | 169 +++++++++++++++++++
yarn/src/main/resources/logback.xml | 31 ++++
29 files changed, 724 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..3688f4a
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+language: java
+cache:
+ directories:
+ - $HOME/.m2
+jdk:
+ - oraclejdk8
+install: true
+script: mvn clean verify
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 9b70451..49b38a9 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,54 @@
# Apache Accumulo Testing Suite
+
+The Apache Accumulo testing suite contains applications that test and verify the
+correctness of Accumulo.
+
+## Installation
+
+In order to run the Apache Accumulo testing suite, you will need Java 8 and Maven installed
+on your machine as well as an Accumulo instance to use for testing.
+
+1. First clone this repository.
+
+ git clone git@github.com:apache/accumulo-testing.git
+ cd accumulo-testing
+
+2. All configuation files for the test suite are in `conf/`. Only the `accumulo-testing.properties`
+ configuration file needs to be created and edited as all other configuration files are optional.
+ In `accumulo-testing.properites`, review the properties with `test.common.*` prefix as these are
+ used by all tests.
+
+ cd conf/
+ cp accumulo-testing.properties.example accumulo-testing.properties
+ vim accumulo-testing.properties
+
+3. Tests are run using the `accumulo-testing` command which is located in the `bin/`
+ directory. Run this command without any arguments to view its usage and see available tests.
+
+ ./bin/accumulo-testing
+
+## Random walk test
+
+The random walk test generates client behavior on an Apache Accumulo instance by randomly walking a
+graph of client operations.
+
+Before running random walk, review the `test.common.*` and `test.randomwalk.*` properties in
+`accumulo-testing.properties` file. A test module must also be specified. See the [modules][modules]
+directory for a list of available ones.
+
+The command below will start a single random walker in a local process using the [Image.xml][image]
+module.
+
+ ./bin/accumulo-testing rw-local Image.xml
+
+If you would like to run multiple, distributed random walkers, run the command below to start random
+walkers in 5 containers in YARN using the Image.xml module.
+
+ ./bin/accumulo-testing rw-yarn 5 Image.xml
+
+This command will create an application in YARN and exit when test is completed. While its
+running, you can view logs for each random walker using the YARN resource manager. The YARN
+application can be killed at any time using `ctrl-c` or via the resource manager.
+
+[modules]: core/src/main/resources/randomwalk/modules
+[image]: core/src/main/resources/randomwalk/modules/Image.xml
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/bin/accumulo-testing
----------------------------------------------------------------------
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
new file mode 100755
index 0000000..e5c1063
--- /dev/null
+++ b/bin/accumulo-testing
@@ -0,0 +1,101 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+at_home=$( cd "$( dirname "$bin_dir" )" && pwd )
+at_version=2.0.0-SNAPSHOT
+
+function print_usage() {
+ cat <<EOF
+
+Usage: accumulo-testing <command> (<argument>)
+
+Possible commands:
+ rw-local <module> Runs randomwalk <module> in local java process
+ rw-yarn <num> <module> Runs randomwalk <module> in <num> containers on YARN
+EOF
+}
+
+if [ -f "$at_home/conf/accumulo-testing-env.sh" ]; then
+ . "$at_home"/conf/accumulo-testing-env.sh
+else
+ . "$at_home"/conf/accumulo-testing-env.sh.example
+fi
+
+if [ -z "$ACCUMULO_VERSION" ]; then
+ echo "ERROR: ACCUMULO_VERSION must be set conf/accumulo-testing-env.sh"
+ exit 1
+fi
+if [ -z "$HADOOP_VERSION" ]; then
+ echo "ERROR: HADOOP_VERSION must be set conf/accumulo-testing-env.sh"
+ exit 1
+fi
+
+at_props="$at_home/conf/accumulo-testing.properties"
+if [ ! -f "$at_props" ]; then
+ echo "Please create and edit accumulo-testing.properties in $at_home/conf"
+ exit 1
+fi
+
+log4j_config="$at_home/conf/log4j.properties"
+if [ ! -f "$log4j_config" ]; then
+ log4j_config="$at_home/conf/log4j.properties.example"
+ if [ ! -f "$log4j_config" ]; then
+ echo "Could not find logj4.properties or log4j.properties.example in $at_home/conf"
+ exit 1
+ fi
+fi
+
+function build_shade_jar() {
+ export at_shaded_jar="$at_home/core/target/accumulo-testing-core-$at_version-shaded.jar"
+ if [ ! -f "$at_shaded_jar" ]; then
+ echo "Building $at_shaded_jar"
+ cd "$at_home" || exit 1
+ mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION"
+ fi
+}
+
+randomwalk_main="org.apache.accumulo.testing.core.randomwalk.Framework"
+case "$1" in
+rw-local)
+ if [ -z "$2" ]; then
+ echo "ERROR: <module> needs to be set"
+ print_usage
+ exit 1
+ fi
+ build_shade_jar
+ java -Dlog4j.configuration="file:$log4j_config" -cp "$at_shaded_jar" "$randomwalk_main" "$at_props" "$2"
+ ;;
+rw-yarn)
+ if [ -z "$2" ]; then
+ echo "ERROR: <num> needs to be set"
+ print_usage
+ exit 1
+ fi
+ if [ -z "$3" ]; then
+ echo "ERROR: <module> needs to be set"
+ print_usage
+ exit 1
+ fi
+ build_shade_jar
+ mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRandomWalkTest -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties $3"
+ ;;
+*)
+ echo "Unknown command: $1"
+ print_usage
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/.gitignore
----------------------------------------------------------------------
diff --git a/conf/.gitignore b/conf/.gitignore
new file mode 100644
index 0000000..336f891
--- /dev/null
+++ b/conf/.gitignore
@@ -0,0 +1,2 @@
+/accumulo-testing.properties
+/accumulo-testing-env.sh
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/accumulo-testing-env.sh.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
new file mode 100644
index 0000000..24f8571
--- /dev/null
+++ b/conf/accumulo-testing-env.sh.example
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Hadoop
+# ======
+test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop
+
+# Accumulo
+# ========
+# Set the Accumulo version that should be included in the shaded jar
+export ACCUMULO_VERSION=`accumulo version`
+export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/accumulo-testing.properties.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example
new file mode 100644
index 0000000..05cbaf5
--- /dev/null
+++ b/conf/accumulo-testing.properties.example
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###################
+# Common properties
+###################
+
+# Accumulo instance name
+test.common.accumulo.instance=instance
+# Accumulo username
+test.common.accumulo.username=root
+# Accumulo password
+test.common.accumulo.password=secret
+# Accumulo keytab
+#test.common.accumulo.keytab=
+# Zookeeper connection string
+test.common.zookeepers=localhost:2181
+# Memory (in MB) given to each container (if running in YARN)
+test.common.yarn.container.memory.mb=1024
+# Number of cores given to each container (if running in YARN)
+test.common.yarn.container.cores=1
+
+########################
+# Random walk properties
+########################
+
+# Max memory for multi-table batch writer
+test.randomwalk.bw.max.mem=100000000
+# Max latency in milliseconds for multi-table batch writer
+test.randomwalk.bw.max.latency=600000
+# Number of write thread for multi-table batch writer
+test.randomwalk.bw.num.threads=4
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/log4j.properties.example
----------------------------------------------------------------------
diff --git a/conf/log4j.properties.example b/conf/log4j.properties.example
new file mode 100644
index 0000000..4cbfd34
--- /dev/null
+++ b/conf/log4j.properties.example
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, CA
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c] %-5p: %m%n
+
+log4j.logger.org.apache.accumulo=WARN
+log4j.logger.org.apache.accumulo.testing=DEBUG
+log4j.logger.org.apache.curator=ERROR
+log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.hadoop.mapreduce=ERROR
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.zookeeper=ERROR
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 83998d5..742ff89 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -31,10 +31,6 @@
<dependencies>
<dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
new file mode 100644
index 0000000..f8ce9ca
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core;
+
+public class TestProps {
+
+ private static final String PREFIX = "test.";
+ private static final String RANDOMWALK = PREFIX + "randomwalk.";
+ private static final String COMMON = PREFIX + "common.";
+
+ /** Common properties **/
+ // Zookeeper connection string
+ public static final String ZOOKEEPERS = COMMON + "zookeepers";
+ // Accumulo instance name
+ public static final String ACCUMULO_INSTANCE = COMMON + "accumulo.instance";
+ // Accumulo username
+ public static final String ACCUMULO_USERNAME = COMMON + "accumulo.username";
+ // Accumulo password
+ public static final String ACCUMULO_PASSWORD = COMMON + "accumulo.password";
+ // Accumulo keytab
+ public static final String ACCUMULO_KEYTAB = COMMON + "accumulo.keytab";
+ // Memory (in MB) given to each YARN container
+ public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb";
+ // Number of cores given to each YARN container
+ public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores";
+
+
+ /** Random walk properties **/
+ // Number of random walker (if running in YARN)
+ public static final String RW_NUM_WALKERS = RANDOMWALK + "num.walkers";
+ // Max memory for multi-table batch writer
+ public static final String RW_BW_MAX_MEM = RANDOMWALK + "bw.max.mem";
+ // Max latency in milliseconds for multi-table batch writer
+ public static final String RW_BW_MAX_LATENCY = RANDOMWALK + "bw.max.latency";
+ // Number of write thread for multi-table batch writer
+ public static final String RW_BW_NUM_THREADS = RANDOMWALK + "bw.num.threads";
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
index 5684353..92a5de0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
@@ -35,6 +35,7 @@ import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,38 +45,6 @@ import org.slf4j.LoggerFactory;
* for creating client-side objects. This class is not thread-safe.
*/
public class Environment {
- /**
- * The configuration property key for a username.
- */
- public static final String KEY_USERNAME = "USERNAME";
- /**
- * The configuration property key for a password.
- */
- public static final String KEY_PASSWORD = "PASSWORD";
- /**
- * The configuration property key for a keytab
- */
- public static final String KEY_KEYTAB = "KEYTAB";
- /**
- * The configuration property key for the instance name.
- */
- public static final String KEY_INSTANCE = "INSTANCE";
- /**
- * The configuration property key for the comma-separated list of ZooKeepers.
- */
- public static final String KEY_ZOOKEEPERS = "ZOOKEEPERS";
- /**
- * The configuration property key for the maximum memory for the multi-table batch writer.
- */
- public static final String KEY_MAX_MEM = "MAX_MEM";
- /**
- * The configuration property key for the maximum latency, in milliseconds, for the multi-table batch writer.
- */
- public static final String KEY_MAX_LATENCY = "MAX_LATENCY";
- /**
- * The configuration property key for the number of write threads for the multi-table batch writer.
- */
- public static final String KEY_NUM_THREADS = "NUM_THREADS";
private static final Logger log = LoggerFactory.getLogger(Environment.class);
@@ -123,7 +92,7 @@ public class Environment {
* @return username
*/
public String getUserName() {
- return p.getProperty(KEY_USERNAME);
+ return p.getProperty(TestProps.ACCUMULO_USERNAME);
}
/**
@@ -132,7 +101,7 @@ public class Environment {
* @return password
*/
public String getPassword() {
- return p.getProperty(KEY_PASSWORD);
+ return p.getProperty(TestProps.ACCUMULO_PASSWORD);
}
/**
@@ -141,7 +110,7 @@ public class Environment {
* @return path to keytab
*/
public String getKeytab() {
- return p.getProperty(KEY_KEYTAB);
+ return p.getProperty(TestProps.ACCUMULO_KEYTAB);
}
/**
@@ -186,8 +155,8 @@ public class Environment {
*/
public Instance getInstance() {
if (instance == null) {
- String instance = p.getProperty(KEY_INSTANCE);
- String zookeepers = p.getProperty(KEY_ZOOKEEPERS);
+ String instance = p.getProperty(TestProps.ACCUMULO_INSTANCE);
+ String zookeepers = p.getProperty(TestProps.ZOOKEEPERS);
this.instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers));
}
return instance;
@@ -216,9 +185,9 @@ public class Environment {
*/
public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException {
if (mtbw == null) {
- long maxMem = Long.parseLong(p.getProperty(KEY_MAX_MEM));
- long maxLatency = Long.parseLong(p.getProperty(KEY_MAX_LATENCY));
- int numThreads = Integer.parseInt(p.getProperty(KEY_NUM_THREADS));
+ long maxMem = Long.parseLong(p.getProperty(TestProps.RW_BW_MAX_MEM));
+ long maxLatency = Long.parseLong(p.getProperty(TestProps.RW_BW_MAX_LATENCY));
+ int numThreads = Integer.parseInt(p.getProperty(TestProps.RW_BW_NUM_THREADS));
mtbw = getConnector().createMultiTableBatchWriter(
new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(numThreads));
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
index 1a5700e..a9f1d68 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
@@ -16,15 +16,12 @@
*/
package org.apache.accumulo.testing.core.randomwalk;
-import java.io.File;
import java.io.FileInputStream;
import java.util.HashMap;
import java.util.Properties;
import org.apache.log4j.Logger;
-import com.beust.jcommander.Parameter;
-
public class Framework {
private static final Logger log = Logger.getLogger(Framework.class);
@@ -63,7 +60,7 @@ public class Framework {
* Name of node
* @return Node specified by id
*/
- public Node getNode(String id) throws Exception {
+ Node getNode(String id) throws Exception {
// check for node in nodes
if (nodes.containsKey(id)) {
@@ -71,9 +68,9 @@ public class Framework {
}
// otherwise create and put in nodes
- Node node = null;
+ Node node;
if (id.endsWith(".xml")) {
- node = new Module(new File("/randomwalk/modules/" + id));
+ node = new Module(id);
} else {
node = (Test) Class.forName(id).newInstance();
}
@@ -81,13 +78,6 @@ public class Framework {
return node;
}
- static class Opts extends org.apache.accumulo.core.cli.Help {
- @Parameter(names = "--configDir", required = true, description = "directory containing the test configuration")
- String configDir;
- @Parameter(names = "--module", required = true, description = "the name of the module to run")
- String module;
- }
-
public static void main(String[] args) throws Exception {
if (args.length != 2) {
@@ -100,10 +90,12 @@ public class Framework {
props.load(fis);
fis.close();
+ log.info("Running random walk test with module: " + args[1]);
+
State state = new State();
Environment env = new Environment(props);
- int retval = getInstance().run(args[1], state, env);
+ getInstance().run(args[1], state, env);
- System.exit(retval);
+ log.info("Test finished");
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
index 1a3d059..3206906 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
@@ -18,7 +18,6 @@ package org.apache.accumulo.testing.core.randomwalk;
import static java.nio.charset.StandardCharsets.UTF_8;
-import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -170,12 +169,12 @@ public class Module extends Node {
private HashMap<String,String> prefixes = new HashMap<>();
private HashMap<String,AdjList> adjMap = new HashMap<>();
private HashMap<String,Set<String>> aliasMap = new HashMap<>();
- private final File xmlFile;
+ private final String id;
private String initNodeId;
private Fixture fixture = null;
- public Module(File xmlFile) throws Exception {
- this.xmlFile = xmlFile;
+ public Module(String id) throws Exception {
+ this.id = id;
loadFromXml();
}
@@ -414,7 +413,7 @@ public class Module extends Node {
@Override
public String toString() {
- return xmlFile.toString();
+ return id;
}
private String getFullName(String name) {
@@ -497,12 +496,13 @@ public class Module extends Node {
dbf.setSchema(moduleSchema);
// parse the document
+
try {
docbuilder = dbf.newDocumentBuilder();
- d = docbuilder.parse(xmlFile);
+ d = docbuilder.parse(this.getClass().getResourceAsStream("/randomwalk/modules/" + id));
} catch (Exception e) {
- log.error("Failed to parse: " + xmlFile, e);
- throw new Exception("Failed to parse: " + xmlFile);
+ log.error("Failed to parse xml at randomwalk/modules/" + id, e);
+ throw new Exception("Failed to parse xml at randomwalk/modules/" + id);
}
// parse packages
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
index 6552161..b67f3d7 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
@@ -23,6 +23,7 @@ import java.util.TreeSet;
import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
import org.apache.accumulo.testing.core.randomwalk.Test;
@@ -64,7 +65,7 @@ public class CopyTable extends Test {
}
args[4] = srcTableName;
args[5] = env.getInstance().getInstanceName();
- args[6] = env.getConfigProperty("ZOOKEEPERS");
+ args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
args[7] = dstTableName;
log.debug("copying " + srcTableName + " to " + dstTableName);
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
index 58d44d4..5113973 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
@@ -26,6 +26,7 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
import org.apache.accumulo.testing.core.randomwalk.Test;
@@ -46,7 +47,7 @@ public class MapRedVerify extends Test {
}
args[4] = state.getString("seqTableName");
args[5] = env.getInstance().getInstanceName();
- args[6] = env.getConfigProperty("ZOOKEEPERS");
+ args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
args[7] = args[4] + "_MR";
if (ToolRunner.run(CachedConfiguration.getInstance(), new MapRedVerifyTool(), args) != 0) {
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Bulk.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Bulk.xml b/core/src/main/resources/randomwalk/modules/Bulk.xml
index 35e2a67..9ffa76a 100644
--- a/core/src/main/resources/randomwalk/modules/Bulk.xml
+++ b/core/src/main/resources/randomwalk/modules/Bulk.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="bulk" value="org.apache.accumulo.test.randomwalk.bulk"/>
+<package prefix="bulk" value="org.apache.accumulo.testing.core.randomwalk.bulk"/>
<init id="bulk.Setup"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Concurrent.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml
index 36ea53c..ea96b27 100644
--- a/core/src/main/resources/randomwalk/modules/Concurrent.xml
+++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="ct" value="org.apache.accumulo.test.randomwalk.concurrent"/>
+<package prefix="ct" value="org.apache.accumulo.testing.core.randomwalk.concurrent"/>
<fixture id="ct.ConcurrentFixture"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Conditional.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Conditional.xml b/core/src/main/resources/randomwalk/modules/Conditional.xml
index 54ff7ab..3c380dc 100644
--- a/core/src/main/resources/randomwalk/modules/Conditional.xml
+++ b/core/src/main/resources/randomwalk/modules/Conditional.xml
@@ -17,8 +17,7 @@
-->
<module>
-<package prefix="ct" value="org.apache.accumulo.test.randomwalk.conditional"/>
-
+<package prefix="ct" value="org.apache.accumulo.testing.core.randomwalk.conditional"/>
<init id="ct.Setup"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Image.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Image.xml b/core/src/main/resources/randomwalk/modules/Image.xml
index 7561895..0b41d14 100644
--- a/core/src/main/resources/randomwalk/modules/Image.xml
+++ b/core/src/main/resources/randomwalk/modules/Image.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="image" value="org.apache.accumulo.test.randomwalk.image"/>
+<package prefix="image" value="org.apache.accumulo.testing.core.randomwalk.image"/>
<fixture id="image.ImageFixture"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/MultiTable.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/MultiTable.xml b/core/src/main/resources/randomwalk/modules/MultiTable.xml
index 55f6590..fa42c3f 100644
--- a/core/src/main/resources/randomwalk/modules/MultiTable.xml
+++ b/core/src/main/resources/randomwalk/modules/MultiTable.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="mt" value="org.apache.accumulo.test.randomwalk.multitable"/>
+<package prefix="mt" value="org.apache.accumulo.testing.core.randomwalk.multitable"/>
<fixture id="mt.MultiTableFixture"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Security.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Security.xml b/core/src/main/resources/randomwalk/modules/Security.xml
index 9e9ef9f..422b63e 100644
--- a/core/src/main/resources/randomwalk/modules/Security.xml
+++ b/core/src/main/resources/randomwalk/modules/Security.xml
@@ -12,7 +12,7 @@
<module>
<package prefix="security"
- value="org.apache.accumulo.test.randomwalk.security" />
+ value="org.apache.accumulo.testing.core.randomwalk.security" />
<fixture id="security.SecurityFixture" />
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Sequential.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Sequential.xml b/core/src/main/resources/randomwalk/modules/Sequential.xml
index 454e75f..ce4abd0 100644
--- a/core/src/main/resources/randomwalk/modules/Sequential.xml
+++ b/core/src/main/resources/randomwalk/modules/Sequential.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="seq" value="org.apache.accumulo.test.randomwalk.sequential"/>
+<package prefix="seq" value="org.apache.accumulo.testing.core.randomwalk.sequential"/>
<fixture id="seq.SequentialFixture"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Shard.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Shard.xml b/core/src/main/resources/randomwalk/modules/Shard.xml
index eb23f37..e8e8654 100644
--- a/core/src/main/resources/randomwalk/modules/Shard.xml
+++ b/core/src/main/resources/randomwalk/modules/Shard.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="shard" value="org.apache.accumulo.test.randomwalk.shard"/>
+<package prefix="shard" value="org.apache.accumulo.testing.core.randomwalk.shard"/>
<fixture id="shard.ShardFixture"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/unit/Basic.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/unit/Basic.xml b/core/src/main/resources/randomwalk/modules/unit/Basic.xml
index 2dead02..f81f132 100644
--- a/core/src/main/resources/randomwalk/modules/unit/Basic.xml
+++ b/core/src/main/resources/randomwalk/modules/unit/Basic.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+<package prefix="test" value="org.apache.accumulo.testing.core.randomwalk.unit"/>
<init id="test.CreateTable"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/unit/Simple.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/unit/Simple.xml b/core/src/main/resources/randomwalk/modules/unit/Simple.xml
index cad940e..6eb685a 100644
--- a/core/src/main/resources/randomwalk/modules/unit/Simple.xml
+++ b/core/src/main/resources/randomwalk/modules/unit/Simple.xml
@@ -17,7 +17,7 @@
-->
<module>
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+<package prefix="test" value="org.apache.accumulo.testing.core.randomwalk.unit"/>
<init id="dummy.all"/>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a2efdf4..2c72faf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,6 +34,7 @@
<modules>
<module>core</module>
+ <module>yarn</module>
</modules>
<properties>
@@ -41,6 +42,7 @@
<hadoop.version>2.6.4</hadoop.version>
<zookeeper.version>3.4.6</zookeeper.version>
<slf4j.version>1.7.21</slf4j.version>
+ <twill.version>0.9.0</twill.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
@@ -58,6 +60,11 @@
<version>1.48</version>
</dependency>
<dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-testing-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.6</version>
@@ -93,6 +100,46 @@
<version>${hadoop.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-api</artifactId>
+ <version>${twill.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-ext</artifactId>
+ <version>${twill.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-yarn</artifactId>
+ <version>${twill.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/.gitignore
----------------------------------------------------------------------
diff --git a/yarn/.gitignore b/yarn/.gitignore
new file mode 100644
index 0000000..17bb010
--- /dev/null
+++ b/yarn/.gitignore
@@ -0,0 +1,2 @@
+/target/
+/*.iml
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/yarn/pom.xml b/yarn/pom.xml
new file mode 100644
index 0000000..650c3b9
--- /dev/null
+++ b/yarn/pom.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-testing</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>accumulo-testing-yarn</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Accumulo Testing YARN</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <!-- Twill requires version 13.0.1 -->
+ <version>13.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-testing-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-ext</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-yarn</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>yarn-test-runner</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>java</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <mainClass>org.apache.accumulo.testing.yarn.YarnAccumuloTestRunner</mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
new file mode 100644
index 0000000..e50dbb4
--- /dev/null
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.yarn;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.google.common.base.Preconditions;
+import org.apache.accumulo.testing.core.TestProps;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunnerService;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.ext.BundledJarRunnable;
+import org.apache.twill.ext.BundledJarRunner;
+import org.apache.twill.yarn.YarnTwillRunnerService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class YarnAccumuloTestRunner {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YarnAccumuloTestRunner.class);
+
+ private static class YarnTestApp implements TwillApplication {
+
+ private TestRunnerOpts opts;
+ private Properties props;
+
+ YarnTestApp(TestRunnerOpts opts, Properties props) {
+ this.opts = opts;
+ this.props = props;
+ }
+
+ @Override
+ public TwillSpecification configure() {
+
+ int numCores = Integer.valueOf(props.getProperty(TestProps.YARN_CONTAINER_CORES));
+ int memory = Integer.valueOf(props.getProperty(TestProps.YARN_CONTAINER_MEMORY_MB));
+
+ ResourceSpecification resourceSpec = ResourceSpecification.Builder.with()
+ .setVirtualCores(numCores).setMemory(memory, ResourceSpecification.SizeUnit.MEGA)
+ .setInstances(opts.numContainers).build();
+
+ File jarFile = new File(opts.jarPath);
+ File testProps = new File(opts.testProps);
+ File log4jProps = new File(opts.logProps);
+
+ return TwillSpecification.Builder.with()
+ .setName(opts.testName)
+ .withRunnable()
+ .add("BundledJarRunnable", new BundledJarRunnable(), resourceSpec)
+ .withLocalFiles()
+ .add(jarFile.getName(), jarFile.toURI(), false)
+ .add(testProps.getName(), testProps.toURI())
+ .add(log4jProps.getName(), log4jProps.toURI())
+ .apply()
+ .anyOrder()
+ .build();
+ }
+ }
+
+ private static class TestRunnerOpts {
+
+ @Parameter(names={"--testName", "-t"}, required = true, description = "Test name")
+ String testName;
+
+ @Parameter(names={"--numContainers", "-n"}, required = true, description = "Test name")
+ int numContainers;
+
+ @Parameter(names={"--jar", "-j"}, required = true, description = "Bundled jar path")
+ String jarPath;
+
+ @Parameter(names={"--main", "-m"}, required = true, description = "Main class")
+ String mainClass;
+
+ @Parameter(names={"--testProps", "-p"}, required = true, description = "Test properties path")
+ String testProps;
+
+ @Parameter(names={"--logProps", "-l"}, required = true, description = "Log properties path")
+ String logProps;
+
+ @Parameter(names={"--args", "-a"}, variableArity = true, description = "Main class args")
+ List<String> mainArgs = new ArrayList<>();
+ }
+
+ private static void verifyPath(String path) {
+ File f = new File(path);
+ Preconditions.checkState(f.exists());
+ Preconditions.checkState(f.canRead());
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ TestRunnerOpts opts = new TestRunnerOpts();
+ new JCommander(opts, args);
+
+ verifyPath(opts.jarPath);
+ verifyPath(opts.testProps);
+ verifyPath(opts.logProps);
+
+ String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new);
+ BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(opts.jarPath, "/lib",
+ opts.mainClass, mainArgs);
+
+ Properties props = new Properties();
+ FileInputStream fis = new FileInputStream(opts.testProps);
+ props.load(fis);
+ fis.close();
+ String zookeepers = props.getProperty(TestProps.ZOOKEEPERS);
+
+ final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(),
+ zookeepers);
+ twillRunner.start();
+
+ final TwillController controller = twillRunner.prepare(
+ new YarnTestApp(opts, props))
+ .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName())
+ .withArguments("BundledJarRunnable", arguments.toArray())
+ .start();
+
+ final AtomicBoolean done = new AtomicBoolean(false);
+
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ if (!done.get()) {
+ controller.kill();
+ }
+ } finally {
+ twillRunner.stop();
+ }
+ }));
+
+ LOG.info("Waiting for {} to finish in YARN...", opts.testName);
+ LOG.info("Press ctrl-c to kill {} in YARN", opts.testName);
+
+ try {
+ controller.awaitTerminated();
+ done.set(true);
+ } catch (ExecutionException e) {
+ LOG.error("Exception during execution", e);
+ throw e;
+ }
+ LOG.info("{} finished", opts.testName);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/yarn/src/main/resources/logback.xml b/yarn/src/main/resources/logback.xml
new file mode 100644
index 0000000..7a8e3b8
--- /dev/null
+++ b/yarn/src/main/resources/logback.xml
@@ -0,0 +1,31 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ agreements. See the NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance with the License. You may obtain a
+ copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+-->
+<configuration>
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <!-- encoders are assigned the type
+ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="org.apache.twill" level="warn"/>
+ <logger name="org.apache.twill.yarn.YarnTwillRunnerService" level="info"/>
+
+ <root level="info">
+ <appender-ref ref="STDOUT" />
+ </root>
+</configuration>
[2/2] accumulo-testing git commit: ACCUMULO-4510 Updates to make old
RW tests run in YARN
Posted by mw...@apache.org.
ACCUMULO-4510 Updates to make old RW tests run in YARN
* Changed how MapReduce jobs are launched in tests and fixed a configuration bug
* Removed test code that expects Accumulo to be installed locally as it cannot be
expected to be on YARN node where test process is run
* Zookeeper version is now configurable in accumulo-testing-env.sh
* YarnAccumuloTestRunner now exits after application was successfully created in YARN
Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/4f822064
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/4f822064
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/4f822064
Branch: refs/heads/master
Commit: 4f8220640419d158e03625e69170a771b9a9c536
Parents: efaa737
Author: Mike Walch <mw...@apache.org>
Authored: Fri Jan 6 16:20:25 2017 -0500
Committer: Mike Walch <mw...@apache.org>
Committed: Mon Jan 9 16:27:34 2017 -0500
----------------------------------------------------------------------
README.md | 6 +-
bin/accumulo-testing | 2 +-
conf/accumulo-testing-env.sh.example | 1 +
conf/log4j.properties.example | 2 +-
core/pom.xml | 4 +
.../testing/core/randomwalk/Environment.java | 12 +++
.../accumulo/testing/core/randomwalk/Node.java | 35 --------
.../testing/core/randomwalk/bulk/Setup.java | 3 +-
.../core/randomwalk/concurrent/Apocalypse.java | 34 --------
.../core/randomwalk/concurrent/BulkImport.java | 5 +-
.../randomwalk/concurrent/OfflineTable.java | 4 +-
.../core/randomwalk/concurrent/Replication.java | 3 +-
.../core/randomwalk/concurrent/Shutdown.java | 63 ---------------
.../core/randomwalk/concurrent/StartAll.java | 58 --------------
.../randomwalk/concurrent/StopTabletServer.java | 84 --------------------
.../core/randomwalk/conditional/Init.java | 6 +-
.../core/randomwalk/multitable/CopyTable.java | 23 +++---
.../randomwalk/sequential/MapRedVerify.java | 27 +++----
.../core/randomwalk/shard/BulkInsert.java | 8 +-
.../core/randomwalk/shard/ExportIndex.java | 4 +-
.../resources/randomwalk/modules/Concurrent.xml | 26 +-----
pom.xml | 7 +-
.../testing/yarn/YarnAccumuloTestRunner.java | 50 ++++++------
23 files changed, 91 insertions(+), 376 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 49b38a9..f19b91b 100644
--- a/README.md
+++ b/README.md
@@ -46,9 +46,9 @@ walkers in 5 containers in YARN using the Image.xml module.
./bin/accumulo-testing rw-yarn 5 Image.xml
-This command will create an application in YARN and exit when test is completed. While its
-running, you can view logs for each random walker using the YARN resource manager. The YARN
-application can be killed at any time using `ctrl-c` or via the resource manager.
+This command will create an application in YARN and exit when all containers for the test have started.
+While its running, you can view logs for each random walker using the YARN resource manager. The YARN
+application can be killed at any time using the YARN resource manager or command line tool.
[modules]: core/src/main/resources/randomwalk/modules
[image]: core/src/main/resources/randomwalk/modules/Image.xml
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/bin/accumulo-testing
----------------------------------------------------------------------
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
index e5c1063..dc6f5da 100755
--- a/bin/accumulo-testing
+++ b/bin/accumulo-testing
@@ -65,7 +65,7 @@ function build_shade_jar() {
if [ ! -f "$at_shaded_jar" ]; then
echo "Building $at_shaded_jar"
cd "$at_home" || exit 1
- mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION"
+ mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION" -D zookeeper.version="$ZOOKEEPER_VERSION"
fi
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/conf/accumulo-testing-env.sh.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
index 24f8571..2e3554e 100644
--- a/conf/accumulo-testing-env.sh.example
+++ b/conf/accumulo-testing-env.sh.example
@@ -23,3 +23,4 @@ test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop
# Set the Accumulo version that should be included in the shaded jar
export ACCUMULO_VERSION=`accumulo version`
export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`
+export ZOOKEEPER_VERSION=3.4.9
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/conf/log4j.properties.example
----------------------------------------------------------------------
diff --git a/conf/log4j.properties.example b/conf/log4j.properties.example
index 4cbfd34..b03805e 100644
--- a/conf/log4j.properties.example
+++ b/conf/log4j.properties.example
@@ -19,7 +19,7 @@ log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c] %-5p: %m%n
log4j.logger.org.apache.accumulo=WARN
-log4j.logger.org.apache.accumulo.testing=DEBUG
+log4j.logger.org.apache.accumulo.testing=INFO
log4j.logger.org.apache.curator=ERROR
log4j.logger.org.apache.hadoop=WARN
log4j.logger.org.apache.hadoop.mapreduce=ERROR
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 742ff89..d5d9230 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -63,6 +63,10 @@
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
index 92a5de0..09d235e 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.testing.core.TestProps;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -122,6 +123,17 @@ public class Environment {
return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
}
+
+ public Configuration getHadoopConfiguration() {
+ Configuration config = new Configuration();
+ config.set("mapreduce.framework.name", "yarn");
+ // Setting below are required due to bundled jar breaking default config.
+ // See http://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file
+ config.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
+ config.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
+ return config;
+ }
+
/**
* Gets an authentication token based on the configured password.
*
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
index b2c2f97..296e974 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
@@ -16,7 +16,6 @@
*/
package org.apache.accumulo.testing.core.randomwalk;
-import java.io.File;
import java.util.Properties;
import org.apache.log4j.Logger;
@@ -63,38 +62,4 @@ public abstract class Node {
synchronized public long lastProgress() {
return progress;
}
-
- protected String getMapReduceJars() {
-
- String acuHome = System.getenv("ACCUMULO_HOME");
- String zkHome = System.getenv("ZOOKEEPER_HOME");
-
- if (acuHome == null || zkHome == null) {
- throw new RuntimeException("ACCUMULO or ZOOKEEPER home not set!");
- }
-
- String retval = null;
-
- File zkLib = new File(zkHome);
- String[] files = zkLib.list();
- if (files != null) {
- for (int i = 0; i < files.length; i++) {
- String f = files[i];
- if (f.matches("^zookeeper-.+jar$")) {
- if (retval == null) {
- retval = String.format("%s/%s", zkLib.getAbsolutePath(), f);
- } else {
- retval += String.format(",%s/%s", zkLib.getAbsolutePath(), f);
- }
- }
- }
- }
-
- File libdir = new File(acuHome + "/lib");
- for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math3 libthrift htrace-core".split(" ")) {
- retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
- }
-
- return retval;
- }
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
index f3c3fdf..635618f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
@@ -26,7 +26,6 @@ import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
@@ -59,7 +58,7 @@ public class Setup extends Test {
// expected if there are multiple walkers
}
state.set("rand", rand);
- state.set("fs", FileSystem.get(CachedConfiguration.getInstance()));
+ state.set("fs", FileSystem.get(env.getHadoopConfiguration()));
state.set("bulkImportSuccess", "true");
BulkPlusOne.counter.set(0l);
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
deleted file mode 100644
index cebc146..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import java.util.Properties;
-
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-
-public class Apocalypse extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- Process exec = Runtime.getRuntime().exec(new String[] {System.getenv("ACCUMULO_HOME") + "/test/system/randomwalk/bin/apocalypse.sh"});
- if (exec.waitFor() != 0)
- throw new RuntimeException("apocalypse.sh returned a non-zero response: " + exec.exitValue());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
index 55fa8d6..9c8eeb4 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
@@ -105,8 +105,7 @@ public class BulkImport extends Test {
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
- Configuration conf = CachedConfiguration.getInstance();
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = FileSystem.get(env.getHadoopConfiguration());
String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl);
@@ -114,7 +113,7 @@ public class BulkImport extends Test {
fs.mkdirs(new Path(bulkDir + "_f"));
try {
- BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf");
+ BatchWriter bw = new RFileBatchWriter(env.getHadoopConfiguration(), fs, bulkDir + "/file01.rf");
try {
TreeSet<Long> rows = new TreeSet<>();
int numRows = rand.nextInt(100000);
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
index fd01d98..f663802 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
@@ -16,6 +16,8 @@
*/
package org.apache.accumulo.testing.core.randomwalk.concurrent;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
import java.util.List;
import java.util.Properties;
import java.util.Random;
@@ -27,8 +29,6 @@ import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
import org.apache.accumulo.testing.core.randomwalk.Test;
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
public class OfflineTable extends Test {
@Override
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
index 189d743..713adaf 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.testing.core.randomwalk.concurrent;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.accumulo.core.conf.Property.MASTER_REPLICATION_SCAN_INTERVAL;
import static org.apache.accumulo.core.conf.Property.REPLICATION_NAME;
import static org.apache.accumulo.core.conf.Property.REPLICATION_PEERS;
@@ -54,8 +55,6 @@ import org.apache.accumulo.testing.core.randomwalk.Test;
import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
import org.apache.hadoop.io.Text;
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
public class Replication extends Test {
final int ROWS = 1000;
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
deleted file mode 100644
index dc2e670..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.impl.MasterClient;
-import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
-import org.apache.accumulo.core.master.thrift.MasterGoalState;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.master.state.SetGoalState;
-import org.apache.accumulo.server.AccumuloServerContext;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
-public class Shutdown extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- log.info("shutting down");
- SetGoalState.main(new String[] {MasterGoalState.CLEAN_STOP.name()});
-
- while (!env.getConnector().instanceOperations().getTabletServers().isEmpty()) {
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
-
- while (true) {
- try {
- AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()));
- Client client = MasterClient.getConnection(context);
- client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
- } catch (Exception e) {
- // assume this is due to server shutdown
- break;
- }
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
-
- log.info("servers stopped");
- sleepUninterruptibly(10, TimeUnit.SECONDS);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
deleted file mode 100644
index df30487..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.impl.MasterClient;
-import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
-import org.apache.accumulo.core.master.thrift.MasterGoalState;
-import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.master.state.SetGoalState;
-import org.apache.accumulo.server.AccumuloServerContext;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
-public class StartAll extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- log.info("Starting all servers");
- SetGoalState.main(new String[] {MasterGoalState.NORMAL.name()});
- Process exec = Runtime.getRuntime().exec(new String[] {System.getenv().get("ACCUMULO_HOME") + "/bin/start-all.sh"});
- exec.waitFor();
- while (true) {
- try {
- AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()));
- Client client = MasterClient.getConnection(context);
- MasterMonitorInfo masterStats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
- if (!masterStats.tServerInfo.isEmpty())
- break;
- } catch (Exception ex) {
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
deleted file mode 100644
index 8210dc4..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.util.AddressUtil;
-import org.apache.accumulo.core.zookeeper.ZooUtil;
-import org.apache.accumulo.fate.zookeeper.ZooReader;
-import org.apache.accumulo.server.master.state.TServerInstance;
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
-
-public class StopTabletServer extends Test {
-
- Set<TServerInstance> getTServers(Instance instance) throws KeeperException, InterruptedException {
- Set<TServerInstance> result = new HashSet<>();
- ZooReader rdr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
- String base = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
- for (String child : rdr.getChildren(base)) {
- try {
- List<String> children = rdr.getChildren(base + "/" + child);
- if (children.size() > 0) {
- Collections.sort(children);
- Stat stat = new Stat();
- byte[] data = rdr.getData(base + "/" + child + "/" + children.get(0), stat);
- if (!"master".equals(new String(data, UTF_8))) {
- result.add(new TServerInstance(AddressUtil.parseAddress(child, false), stat.getEphemeralOwner()));
- }
- }
- } catch (KeeperException.NoNodeException ex) {
- // someone beat us too it
- }
- }
- return result;
- }
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
-
- Instance instance = env.getInstance();
-
- List<TServerInstance> currentServers = new ArrayList<>(getTServers(instance));
- Collections.shuffle(currentServers);
- Runtime runtime = Runtime.getRuntime();
- if (currentServers.size() > 1) {
- TServerInstance victim = currentServers.get(0);
- log.info("Stopping " + victim.hostPort());
- Process exec = runtime.exec(new String[] {System.getenv("ACCUMULO_HOME") + "/bin/accumulo", "admin", "stop", victim.hostPort()});
- if (exec.waitFor() != 0)
- throw new RuntimeException("admin stop returned a non-zero response: " + exec.exitValue());
- Set<TServerInstance> set = getTServers(instance);
- if (set.contains(victim))
- throw new RuntimeException("Failed to stop " + victim);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
index 50a1e52..3656653 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
@@ -47,7 +47,7 @@ public class Init extends Test {
for (int i = 1; i < 10; i++)
splits.add(new Text(Utils.getBank((int) (numBanks * .1 * i))));
env.getConnector().tableOperations().addSplits((String) state.get("tableName"), splits);
- log.debug("Added splits " + splits);
+ log.info("Added splits " + splits);
ArrayList<Integer> banks = new ArrayList<>();
for (int i = 0; i < numBanks; i++)
@@ -87,8 +87,10 @@ public class Init extends Test {
acceptedCount++;
}
- log.debug("Added bank " + Utils.getBank(i) + " " + acceptedCount);
+ log.trace("Added bank " + Utils.getBank(i) + " " + acceptedCount);
}
+ log.debug("Added " + numBanks + " banks");
+
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
index b67f3d7..8d1e451 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
@@ -22,7 +22,6 @@ import java.util.Random;
import java.util.TreeSet;
import org.apache.accumulo.core.client.impl.Tables;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
@@ -55,18 +54,16 @@ public class CopyTable extends Test {
int nextId = ((Integer) state.get("nextId")).intValue();
String dstTableName = String.format("%s_%d", state.getString("tableNamePrefix"), nextId);
- String[] args = new String[8];
- args[0] = "-libjars";
- args[1] = getMapReduceJars();
- args[2] = env.getUserName();
- args[3] = env.getPassword();
- if (null == args[3]) {
- args[3] = env.getKeytab();
+ String[] args = new String[6];
+ args[0] = env.getUserName();
+ args[1] = env.getPassword();
+ if (null == args[1]) {
+ args[1] = env.getKeytab();
}
- args[4] = srcTableName;
- args[5] = env.getInstance().getInstanceName();
- args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
- args[7] = dstTableName;
+ args[2] = srcTableName;
+ args[3] = env.getInstance().getInstanceName();
+ args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS);
+ args[5] = dstTableName;
log.debug("copying " + srcTableName + " to " + dstTableName);
@@ -74,7 +71,7 @@ public class CopyTable extends Test {
env.getConnector().tableOperations().addSplits(dstTableName, splits);
- if (ToolRunner.run(CachedConfiguration.getInstance(), new CopyTool(), args) != 0) {
+ if (ToolRunner.run(env.getHadoopConfiguration(), new CopyTool(), args) != 0) {
log.error("Failed to run map/red verify");
return;
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
index 5113973..09f4dce 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
@@ -25,7 +25,6 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
@@ -37,20 +36,18 @@ public class MapRedVerify extends Test {
@Override
public void visit(State state, Environment env, Properties props) throws Exception {
- String[] args = new String[8];
- args[0] = "-libjars";
- args[1] = getMapReduceJars();
- args[2] = env.getUserName();
- args[3] = env.getPassword();
- if (null == args[3]) {
- args[3] = env.getKeytab();
+ String[] args = new String[6];
+ args[0] = env.getUserName();
+ args[1] = env.getPassword();
+ if (null == args[1]) {
+ args[1] = env.getKeytab();
}
- args[4] = state.getString("seqTableName");
- args[5] = env.getInstance().getInstanceName();
- args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
- args[7] = args[4] + "_MR";
+ args[2] = state.getString("seqTableName");
+ args[3] = env.getInstance().getInstanceName();
+ args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS);
+ args[5] = args[2] + "_MR";
- if (ToolRunner.run(CachedConfiguration.getInstance(), new MapRedVerifyTool(), args) != 0) {
+ if (ToolRunner.run(env.getHadoopConfiguration(), new MapRedVerifyTool(), args) != 0) {
log.error("Failed to run map/red verify");
return;
}
@@ -73,8 +70,8 @@ public class MapRedVerify extends Test {
log.error("Gaps in output");
}
- log.debug("Dropping table: " + args[7]);
+ log.debug("Dropping table: " + args[5]);
Connector conn = env.getConnector();
- conn.tableOperations().delete(args[7]);
+ conn.tableOperations().delete(args[5]);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
index 86afd8f..76b9ef6 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
@@ -110,7 +110,7 @@ public class BulkInsert extends Test {
int maxSplits = Integer.parseInt(props.getProperty("maxSplits"));
- Configuration conf = CachedConfiguration.getInstance();
+ Configuration conf = env.getHadoopConfiguration();
FileSystem fs = FileSystem.get(conf);
String rootDir = "/tmp/shard_bulk/" + dataTableName;
@@ -179,11 +179,7 @@ public class BulkInsert extends Test {
SortTool sortTool = new SortTool(seqFile, outputDir, workDir + "/splits.txt", splits);
- String[] args = new String[2];
- args[0] = "-libjars";
- args[1] = getMapReduceJars();
-
- if (ToolRunner.run(CachedConfiguration.getInstance(), sortTool, args) != 0) {
+ if (ToolRunner.run(env.getHadoopConfiguration(), sortTool, new String[0]) != 0) {
throw new Exception("Failed to run map/red verify");
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
index d52198b..0e4853d 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
@@ -49,7 +49,7 @@ public class ExportIndex extends Test {
String exportDir = "/tmp/shard_export/" + indexTableName;
String copyDir = "/tmp/shard_export/" + tmpIndexTableName;
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ FileSystem fs = FileSystem.get(env.getHadoopConfiguration());
fs.delete(new Path("/tmp/shard_export/" + indexTableName), true);
fs.delete(new Path("/tmp/shard_export/" + tmpIndexTableName), true);
@@ -74,7 +74,7 @@ public class ExportIndex extends Test {
while ((file = reader.readLine()) != null) {
Path src = new Path(file);
Path dest = new Path(new Path(copyDir), src.getName());
- FileUtil.copy(fs, src, fs, dest, false, true, CachedConfiguration.getInstance());
+ FileUtil.copy(fs, src, fs, dest, false, true, env.getHadoopConfiguration());
}
reader.close();
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/resources/randomwalk/modules/Concurrent.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml
index ea96b27..eb17066 100644
--- a/core/src/main/resources/randomwalk/modules/Concurrent.xml
+++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml
@@ -45,14 +45,10 @@
<edge id="ct.ChangeAuthorizations" weight="1000"/>
<edge id="ct.ChangePermissions" weight="1000"/>
<edge id="ct.CheckPermission" weight="1000"/>
- <edge id="ct.StopTabletServer" weight="100"/>
- <edge id="ct.StartAll" weight="1000"/>
- <edge id="ct.Shutdown" weight="10"/>
<edge id="ct.Config" weight="1000"/>
<edge id="ct.CreateNamespace" weight="1000"/>
<edge id="ct.DeleteNamespace" weight="100"/>
<edge id="ct.RenameNamespace" weight="100"/>
- <edge id="ct.Apocalypse" weight="10"/>
<edge id="END" weight="1"/>
</node>
@@ -146,36 +142,20 @@
<edge id="dummy.ToAll" weight="1"/>
</node>
-<node id="ct.StopTabletServer">
- <edge id="dummy.ToAll" weight="1"/>
-</node>
-
-<node id="ct.StartAll">
- <edge id="dummy.ToAll" weight="1"/>
-</node>
-
<node id="ct.Config">
<edge id="dummy.ToAll" weight="1"/>
</node>
-<node id="ct.Shutdown">
- <edge id="ct.StartAll" weight="1"/>
-</node>
-
-<node id="ct.Apocalypse">
- <edge id="ct.StartAll" weight="1"/>
-</node>
-
<node id="ct.CreateNamespace">
- <edge id="ct.StartAll" weight="1"/>
+ <edge id="dummy.ToAll" weight="1"/>
</node>
<node id="ct.DeleteNamespace">
- <edge id="ct.StartAll" weight="1"/>
+ <edge id="dummy.ToAll" weight="1"/>
</node>
<node id="ct.RenameNamespace">
- <edge id="ct.StartAll" weight="1"/>
+ <edge id="dummy.ToAll" weight="1"/>
</node>
</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2c72faf..cea25b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -40,7 +40,7 @@
<properties>
<accumulo.version>1.8.0</accumulo.version>
<hadoop.version>2.6.4</hadoop.version>
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.9</zookeeper.version>
<slf4j.version>1.7.21</slf4j.version>
<twill.version>0.9.0</twill.version>
<maven.compiler.source>1.8</maven.compiler.source>
@@ -101,6 +101,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
index e50dbb4..84d2576 100644
--- a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -22,9 +22,11 @@ import com.beust.jcommander.Parameter;
import com.google.common.base.Preconditions;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillApplication;
import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
import org.apache.twill.api.TwillRunnerService;
import org.apache.twill.api.TwillSpecification;
import org.apache.twill.ext.BundledJarRunnable;
@@ -36,15 +38,16 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Properties;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicBoolean;
public class YarnAccumuloTestRunner {
private static final Logger LOG = LoggerFactory.getLogger(YarnAccumuloTestRunner.class);
+ private static final String RUNNABLE_ID = "BundledJarRunnable";
+
private static class YarnTestApp implements TwillApplication {
private TestRunnerOpts opts;
@@ -72,7 +75,7 @@ public class YarnAccumuloTestRunner {
return TwillSpecification.Builder.with()
.setName(opts.testName)
.withRunnable()
- .add("BundledJarRunnable", new BundledJarRunnable(), resourceSpec)
+ .add(RUNNABLE_ID, new BundledJarRunnable(), resourceSpec)
.withLocalFiles()
.add(jarFile.getName(), jarFile.toURI(), false)
.add(testProps.getName(), testProps.toURI())
@@ -113,6 +116,15 @@ public class YarnAccumuloTestRunner {
Preconditions.checkState(f.canRead());
}
+ private static int getNumRunning(TwillController controller) {
+ ResourceReport report = controller.getResourceReport();
+ if (report == null) {
+ return 0;
+ }
+ Collection<TwillRunResources> resources = report.getRunnableResources(RUNNABLE_ID);
+ return resources == null ? 0 : resources.size();
+ }
+
public static void main(String[] args) throws Exception {
TestRunnerOpts opts = new TestRunnerOpts();
@@ -136,34 +148,20 @@ public class YarnAccumuloTestRunner {
zookeepers);
twillRunner.start();
- final TwillController controller = twillRunner.prepare(
+ TwillController controller = twillRunner.prepare(
new YarnTestApp(opts, props))
.addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName())
.withArguments("BundledJarRunnable", arguments.toArray())
.start();
- final AtomicBoolean done = new AtomicBoolean(false);
-
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- if (!done.get()) {
- controller.kill();
- }
- } finally {
- twillRunner.stop();
- }
- }));
-
- LOG.info("Waiting for {} to finish in YARN...", opts.testName);
- LOG.info("Press ctrl-c to kill {} in YARN", opts.testName);
-
- try {
- controller.awaitTerminated();
- done.set(true);
- } catch (ExecutionException e) {
- LOG.error("Exception during execution", e);
- throw e;
+ int numRunning = getNumRunning(controller);
+ while (numRunning != opts.numContainers) {
+ LOG.info("{} of {} containers have started in YARN.", numRunning, opts.numContainers);
+ Thread.sleep(5000);
+ numRunning = getNumRunning(controller);
}
- LOG.info("{} finished", opts.testName);
+
+ LOG.info("{} of {} containers have started in YARN", numRunning, opts.numContainers);
+ LOG.info("{} application was successfully started in YARN", opts.testName);
}
}
\ No newline at end of file