You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/09 22:12:06 UTC
[2/2] accumulo-testing git commit: ACCUMULO-4510 Updates to make old
RW tests run in YARN
ACCUMULO-4510 Updates to make old RW tests run in YARN
* Changed how MapReduce jobs are launched in tests and fixed a configuration bug
* Removed test code that expects Accumulo to be installed locally as it cannot be
expected to be on YARN node where test process is run
* Zookeeper version is now configurable in accumulo-testing-env.sh
* YarnAccumuloTestRunner now exits after application was successfully created in YARN
Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/4f822064
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/4f822064
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/4f822064
Branch: refs/heads/master
Commit: 4f8220640419d158e03625e69170a771b9a9c536
Parents: efaa737
Author: Mike Walch <mw...@apache.org>
Authored: Fri Jan 6 16:20:25 2017 -0500
Committer: Mike Walch <mw...@apache.org>
Committed: Mon Jan 9 16:27:34 2017 -0500
----------------------------------------------------------------------
README.md | 6 +-
bin/accumulo-testing | 2 +-
conf/accumulo-testing-env.sh.example | 1 +
conf/log4j.properties.example | 2 +-
core/pom.xml | 4 +
.../testing/core/randomwalk/Environment.java | 12 +++
.../accumulo/testing/core/randomwalk/Node.java | 35 --------
.../testing/core/randomwalk/bulk/Setup.java | 3 +-
.../core/randomwalk/concurrent/Apocalypse.java | 34 --------
.../core/randomwalk/concurrent/BulkImport.java | 5 +-
.../randomwalk/concurrent/OfflineTable.java | 4 +-
.../core/randomwalk/concurrent/Replication.java | 3 +-
.../core/randomwalk/concurrent/Shutdown.java | 63 ---------------
.../core/randomwalk/concurrent/StartAll.java | 58 --------------
.../randomwalk/concurrent/StopTabletServer.java | 84 --------------------
.../core/randomwalk/conditional/Init.java | 6 +-
.../core/randomwalk/multitable/CopyTable.java | 23 +++---
.../randomwalk/sequential/MapRedVerify.java | 27 +++----
.../core/randomwalk/shard/BulkInsert.java | 8 +-
.../core/randomwalk/shard/ExportIndex.java | 4 +-
.../resources/randomwalk/modules/Concurrent.xml | 26 +-----
pom.xml | 7 +-
.../testing/yarn/YarnAccumuloTestRunner.java | 50 ++++++------
23 files changed, 91 insertions(+), 376 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 49b38a9..f19b91b 100644
--- a/README.md
+++ b/README.md
@@ -46,9 +46,9 @@ walkers in 5 containers in YARN using the Image.xml module.
./bin/accumulo-testing rw-yarn 5 Image.xml
-This command will create an application in YARN and exit when test is completed. While its
-running, you can view logs for each random walker using the YARN resource manager. The YARN
-application can be killed at any time using `ctrl-c` or via the resource manager.
+This command will create an application in YARN and exit when all containers for the test have started.
+While its running, you can view logs for each random walker using the YARN resource manager. The YARN
+application can be killed at any time using the YARN resource manager or command line tool.
[modules]: core/src/main/resources/randomwalk/modules
[image]: core/src/main/resources/randomwalk/modules/Image.xml
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/bin/accumulo-testing
----------------------------------------------------------------------
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
index e5c1063..dc6f5da 100755
--- a/bin/accumulo-testing
+++ b/bin/accumulo-testing
@@ -65,7 +65,7 @@ function build_shade_jar() {
if [ ! -f "$at_shaded_jar" ]; then
echo "Building $at_shaded_jar"
cd "$at_home" || exit 1
- mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION"
+ mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION" -D zookeeper.version="$ZOOKEEPER_VERSION"
fi
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/conf/accumulo-testing-env.sh.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
index 24f8571..2e3554e 100644
--- a/conf/accumulo-testing-env.sh.example
+++ b/conf/accumulo-testing-env.sh.example
@@ -23,3 +23,4 @@ test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop
# Set the Accumulo version that should be included in the shaded jar
export ACCUMULO_VERSION=`accumulo version`
export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`
+export ZOOKEEPER_VERSION=3.4.9
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/conf/log4j.properties.example
----------------------------------------------------------------------
diff --git a/conf/log4j.properties.example b/conf/log4j.properties.example
index 4cbfd34..b03805e 100644
--- a/conf/log4j.properties.example
+++ b/conf/log4j.properties.example
@@ -19,7 +19,7 @@ log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c] %-5p: %m%n
log4j.logger.org.apache.accumulo=WARN
-log4j.logger.org.apache.accumulo.testing=DEBUG
+log4j.logger.org.apache.accumulo.testing=INFO
log4j.logger.org.apache.curator=ERROR
log4j.logger.org.apache.hadoop=WARN
log4j.logger.org.apache.hadoop.mapreduce=ERROR
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 742ff89..d5d9230 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -63,6 +63,10 @@
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
index 92a5de0..09d235e 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.testing.core.TestProps;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -122,6 +123,17 @@ public class Environment {
return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
}
+
+ public Configuration getHadoopConfiguration() {
+ Configuration config = new Configuration();
+ config.set("mapreduce.framework.name", "yarn");
+ // Setting below are required due to bundled jar breaking default config.
+ // See http://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file
+ config.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
+ config.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
+ return config;
+ }
+
/**
* Gets an authentication token based on the configured password.
*
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
index b2c2f97..296e974 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
@@ -16,7 +16,6 @@
*/
package org.apache.accumulo.testing.core.randomwalk;
-import java.io.File;
import java.util.Properties;
import org.apache.log4j.Logger;
@@ -63,38 +62,4 @@ public abstract class Node {
synchronized public long lastProgress() {
return progress;
}
-
- protected String getMapReduceJars() {
-
- String acuHome = System.getenv("ACCUMULO_HOME");
- String zkHome = System.getenv("ZOOKEEPER_HOME");
-
- if (acuHome == null || zkHome == null) {
- throw new RuntimeException("ACCUMULO or ZOOKEEPER home not set!");
- }
-
- String retval = null;
-
- File zkLib = new File(zkHome);
- String[] files = zkLib.list();
- if (files != null) {
- for (int i = 0; i < files.length; i++) {
- String f = files[i];
- if (f.matches("^zookeeper-.+jar$")) {
- if (retval == null) {
- retval = String.format("%s/%s", zkLib.getAbsolutePath(), f);
- } else {
- retval += String.format(",%s/%s", zkLib.getAbsolutePath(), f);
- }
- }
- }
- }
-
- File libdir = new File(acuHome + "/lib");
- for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math3 libthrift htrace-core".split(" ")) {
- retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
- }
-
- return retval;
- }
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
index f3c3fdf..635618f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
@@ -26,7 +26,6 @@ import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
@@ -59,7 +58,7 @@ public class Setup extends Test {
// expected if there are multiple walkers
}
state.set("rand", rand);
- state.set("fs", FileSystem.get(CachedConfiguration.getInstance()));
+ state.set("fs", FileSystem.get(env.getHadoopConfiguration()));
state.set("bulkImportSuccess", "true");
BulkPlusOne.counter.set(0l);
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
deleted file mode 100644
index cebc146..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import java.util.Properties;
-
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-
-public class Apocalypse extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- Process exec = Runtime.getRuntime().exec(new String[] {System.getenv("ACCUMULO_HOME") + "/test/system/randomwalk/bin/apocalypse.sh"});
- if (exec.waitFor() != 0)
- throw new RuntimeException("apocalypse.sh returned a non-zero response: " + exec.exitValue());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
index 55fa8d6..9c8eeb4 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
@@ -105,8 +105,7 @@ public class BulkImport extends Test {
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
- Configuration conf = CachedConfiguration.getInstance();
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = FileSystem.get(env.getHadoopConfiguration());
String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl);
@@ -114,7 +113,7 @@ public class BulkImport extends Test {
fs.mkdirs(new Path(bulkDir + "_f"));
try {
- BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf");
+ BatchWriter bw = new RFileBatchWriter(env.getHadoopConfiguration(), fs, bulkDir + "/file01.rf");
try {
TreeSet<Long> rows = new TreeSet<>();
int numRows = rand.nextInt(100000);
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
index fd01d98..f663802 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
@@ -16,6 +16,8 @@
*/
package org.apache.accumulo.testing.core.randomwalk.concurrent;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
import java.util.List;
import java.util.Properties;
import java.util.Random;
@@ -27,8 +29,6 @@ import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
import org.apache.accumulo.testing.core.randomwalk.Test;
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
public class OfflineTable extends Test {
@Override
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
index 189d743..713adaf 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.testing.core.randomwalk.concurrent;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.accumulo.core.conf.Property.MASTER_REPLICATION_SCAN_INTERVAL;
import static org.apache.accumulo.core.conf.Property.REPLICATION_NAME;
import static org.apache.accumulo.core.conf.Property.REPLICATION_PEERS;
@@ -54,8 +55,6 @@ import org.apache.accumulo.testing.core.randomwalk.Test;
import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
import org.apache.hadoop.io.Text;
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
public class Replication extends Test {
final int ROWS = 1000;
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
deleted file mode 100644
index dc2e670..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.impl.MasterClient;
-import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
-import org.apache.accumulo.core.master.thrift.MasterGoalState;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.master.state.SetGoalState;
-import org.apache.accumulo.server.AccumuloServerContext;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
-public class Shutdown extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- log.info("shutting down");
- SetGoalState.main(new String[] {MasterGoalState.CLEAN_STOP.name()});
-
- while (!env.getConnector().instanceOperations().getTabletServers().isEmpty()) {
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
-
- while (true) {
- try {
- AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()));
- Client client = MasterClient.getConnection(context);
- client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
- } catch (Exception e) {
- // assume this is due to server shutdown
- break;
- }
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
-
- log.info("servers stopped");
- sleepUninterruptibly(10, TimeUnit.SECONDS);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
deleted file mode 100644
index df30487..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.impl.MasterClient;
-import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
-import org.apache.accumulo.core.master.thrift.MasterGoalState;
-import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.master.state.SetGoalState;
-import org.apache.accumulo.server.AccumuloServerContext;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-
-public class StartAll extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- log.info("Starting all servers");
- SetGoalState.main(new String[] {MasterGoalState.NORMAL.name()});
- Process exec = Runtime.getRuntime().exec(new String[] {System.getenv().get("ACCUMULO_HOME") + "/bin/start-all.sh"});
- exec.waitFor();
- while (true) {
- try {
- AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()));
- Client client = MasterClient.getConnection(context);
- MasterMonitorInfo masterStats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
- if (!masterStats.tServerInfo.isEmpty())
- break;
- } catch (Exception ex) {
- sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
deleted file mode 100644
index 8210dc4..0000000
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.core.randomwalk.concurrent;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.util.AddressUtil;
-import org.apache.accumulo.core.zookeeper.ZooUtil;
-import org.apache.accumulo.fate.zookeeper.ZooReader;
-import org.apache.accumulo.server.master.state.TServerInstance;
-import org.apache.accumulo.testing.core.randomwalk.Environment;
-import org.apache.accumulo.testing.core.randomwalk.State;
-import org.apache.accumulo.testing.core.randomwalk.Test;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
-
-public class StopTabletServer extends Test {
-
- Set<TServerInstance> getTServers(Instance instance) throws KeeperException, InterruptedException {
- Set<TServerInstance> result = new HashSet<>();
- ZooReader rdr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
- String base = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
- for (String child : rdr.getChildren(base)) {
- try {
- List<String> children = rdr.getChildren(base + "/" + child);
- if (children.size() > 0) {
- Collections.sort(children);
- Stat stat = new Stat();
- byte[] data = rdr.getData(base + "/" + child + "/" + children.get(0), stat);
- if (!"master".equals(new String(data, UTF_8))) {
- result.add(new TServerInstance(AddressUtil.parseAddress(child, false), stat.getEphemeralOwner()));
- }
- }
- } catch (KeeperException.NoNodeException ex) {
- // someone beat us too it
- }
- }
- return result;
- }
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
-
- Instance instance = env.getInstance();
-
- List<TServerInstance> currentServers = new ArrayList<>(getTServers(instance));
- Collections.shuffle(currentServers);
- Runtime runtime = Runtime.getRuntime();
- if (currentServers.size() > 1) {
- TServerInstance victim = currentServers.get(0);
- log.info("Stopping " + victim.hostPort());
- Process exec = runtime.exec(new String[] {System.getenv("ACCUMULO_HOME") + "/bin/accumulo", "admin", "stop", victim.hostPort()});
- if (exec.waitFor() != 0)
- throw new RuntimeException("admin stop returned a non-zero response: " + exec.exitValue());
- Set<TServerInstance> set = getTServers(instance);
- if (set.contains(victim))
- throw new RuntimeException("Failed to stop " + victim);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
index 50a1e52..3656653 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
@@ -47,7 +47,7 @@ public class Init extends Test {
for (int i = 1; i < 10; i++)
splits.add(new Text(Utils.getBank((int) (numBanks * .1 * i))));
env.getConnector().tableOperations().addSplits((String) state.get("tableName"), splits);
- log.debug("Added splits " + splits);
+ log.info("Added splits " + splits);
ArrayList<Integer> banks = new ArrayList<>();
for (int i = 0; i < numBanks; i++)
@@ -87,8 +87,10 @@ public class Init extends Test {
acceptedCount++;
}
- log.debug("Added bank " + Utils.getBank(i) + " " + acceptedCount);
+ log.trace("Added bank " + Utils.getBank(i) + " " + acceptedCount);
}
+ log.debug("Added " + numBanks + " banks");
+
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
index b67f3d7..8d1e451 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
@@ -22,7 +22,6 @@ import java.util.Random;
import java.util.TreeSet;
import org.apache.accumulo.core.client.impl.Tables;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
@@ -55,18 +54,16 @@ public class CopyTable extends Test {
int nextId = ((Integer) state.get("nextId")).intValue();
String dstTableName = String.format("%s_%d", state.getString("tableNamePrefix"), nextId);
- String[] args = new String[8];
- args[0] = "-libjars";
- args[1] = getMapReduceJars();
- args[2] = env.getUserName();
- args[3] = env.getPassword();
- if (null == args[3]) {
- args[3] = env.getKeytab();
+ String[] args = new String[6];
+ args[0] = env.getUserName();
+ args[1] = env.getPassword();
+ if (null == args[1]) {
+ args[1] = env.getKeytab();
}
- args[4] = srcTableName;
- args[5] = env.getInstance().getInstanceName();
- args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
- args[7] = dstTableName;
+ args[2] = srcTableName;
+ args[3] = env.getInstance().getInstanceName();
+ args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS);
+ args[5] = dstTableName;
log.debug("copying " + srcTableName + " to " + dstTableName);
@@ -74,7 +71,7 @@ public class CopyTable extends Test {
env.getConnector().tableOperations().addSplits(dstTableName, splits);
- if (ToolRunner.run(CachedConfiguration.getInstance(), new CopyTool(), args) != 0) {
+ if (ToolRunner.run(env.getHadoopConfiguration(), new CopyTool(), args) != 0) {
log.error("Failed to run map/red verify");
return;
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
index 5113973..09f4dce 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
@@ -25,7 +25,6 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.accumulo.testing.core.randomwalk.Environment;
import org.apache.accumulo.testing.core.randomwalk.State;
@@ -37,20 +36,18 @@ public class MapRedVerify extends Test {
@Override
public void visit(State state, Environment env, Properties props) throws Exception {
- String[] args = new String[8];
- args[0] = "-libjars";
- args[1] = getMapReduceJars();
- args[2] = env.getUserName();
- args[3] = env.getPassword();
- if (null == args[3]) {
- args[3] = env.getKeytab();
+ String[] args = new String[6];
+ args[0] = env.getUserName();
+ args[1] = env.getPassword();
+ if (null == args[1]) {
+ args[1] = env.getKeytab();
}
- args[4] = state.getString("seqTableName");
- args[5] = env.getInstance().getInstanceName();
- args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
- args[7] = args[4] + "_MR";
+ args[2] = state.getString("seqTableName");
+ args[3] = env.getInstance().getInstanceName();
+ args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS);
+ args[5] = args[2] + "_MR";
- if (ToolRunner.run(CachedConfiguration.getInstance(), new MapRedVerifyTool(), args) != 0) {
+ if (ToolRunner.run(env.getHadoopConfiguration(), new MapRedVerifyTool(), args) != 0) {
log.error("Failed to run map/red verify");
return;
}
@@ -73,8 +70,8 @@ public class MapRedVerify extends Test {
log.error("Gaps in output");
}
- log.debug("Dropping table: " + args[7]);
+ log.debug("Dropping table: " + args[5]);
Connector conn = env.getConnector();
- conn.tableOperations().delete(args[7]);
+ conn.tableOperations().delete(args[5]);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
index 86afd8f..76b9ef6 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
@@ -110,7 +110,7 @@ public class BulkInsert extends Test {
int maxSplits = Integer.parseInt(props.getProperty("maxSplits"));
- Configuration conf = CachedConfiguration.getInstance();
+ Configuration conf = env.getHadoopConfiguration();
FileSystem fs = FileSystem.get(conf);
String rootDir = "/tmp/shard_bulk/" + dataTableName;
@@ -179,11 +179,7 @@ public class BulkInsert extends Test {
SortTool sortTool = new SortTool(seqFile, outputDir, workDir + "/splits.txt", splits);
- String[] args = new String[2];
- args[0] = "-libjars";
- args[1] = getMapReduceJars();
-
- if (ToolRunner.run(CachedConfiguration.getInstance(), sortTool, args) != 0) {
+ if (ToolRunner.run(env.getHadoopConfiguration(), sortTool, new String[0]) != 0) {
throw new Exception("Failed to run map/red verify");
}
}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
index d52198b..0e4853d 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
@@ -49,7 +49,7 @@ public class ExportIndex extends Test {
String exportDir = "/tmp/shard_export/" + indexTableName;
String copyDir = "/tmp/shard_export/" + tmpIndexTableName;
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ FileSystem fs = FileSystem.get(env.getHadoopConfiguration());
fs.delete(new Path("/tmp/shard_export/" + indexTableName), true);
fs.delete(new Path("/tmp/shard_export/" + tmpIndexTableName), true);
@@ -74,7 +74,7 @@ public class ExportIndex extends Test {
while ((file = reader.readLine()) != null) {
Path src = new Path(file);
Path dest = new Path(new Path(copyDir), src.getName());
- FileUtil.copy(fs, src, fs, dest, false, true, CachedConfiguration.getInstance());
+ FileUtil.copy(fs, src, fs, dest, false, true, env.getHadoopConfiguration());
}
reader.close();
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/core/src/main/resources/randomwalk/modules/Concurrent.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml
index ea96b27..eb17066 100644
--- a/core/src/main/resources/randomwalk/modules/Concurrent.xml
+++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml
@@ -45,14 +45,10 @@
<edge id="ct.ChangeAuthorizations" weight="1000"/>
<edge id="ct.ChangePermissions" weight="1000"/>
<edge id="ct.CheckPermission" weight="1000"/>
- <edge id="ct.StopTabletServer" weight="100"/>
- <edge id="ct.StartAll" weight="1000"/>
- <edge id="ct.Shutdown" weight="10"/>
<edge id="ct.Config" weight="1000"/>
<edge id="ct.CreateNamespace" weight="1000"/>
<edge id="ct.DeleteNamespace" weight="100"/>
<edge id="ct.RenameNamespace" weight="100"/>
- <edge id="ct.Apocalypse" weight="10"/>
<edge id="END" weight="1"/>
</node>
@@ -146,36 +142,20 @@
<edge id="dummy.ToAll" weight="1"/>
</node>
-<node id="ct.StopTabletServer">
- <edge id="dummy.ToAll" weight="1"/>
-</node>
-
-<node id="ct.StartAll">
- <edge id="dummy.ToAll" weight="1"/>
-</node>
-
<node id="ct.Config">
<edge id="dummy.ToAll" weight="1"/>
</node>
-<node id="ct.Shutdown">
- <edge id="ct.StartAll" weight="1"/>
-</node>
-
-<node id="ct.Apocalypse">
- <edge id="ct.StartAll" weight="1"/>
-</node>
-
<node id="ct.CreateNamespace">
- <edge id="ct.StartAll" weight="1"/>
+ <edge id="dummy.ToAll" weight="1"/>
</node>
<node id="ct.DeleteNamespace">
- <edge id="ct.StartAll" weight="1"/>
+ <edge id="dummy.ToAll" weight="1"/>
</node>
<node id="ct.RenameNamespace">
- <edge id="ct.StartAll" weight="1"/>
+ <edge id="dummy.ToAll" weight="1"/>
</node>
</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2c72faf..cea25b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -40,7 +40,7 @@
<properties>
<accumulo.version>1.8.0</accumulo.version>
<hadoop.version>2.6.4</hadoop.version>
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.9</zookeeper.version>
<slf4j.version>1.7.21</slf4j.version>
<twill.version>0.9.0</twill.version>
<maven.compiler.source>1.8</maven.compiler.source>
@@ -101,6 +101,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/4f822064/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
index e50dbb4..84d2576 100644
--- a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -22,9 +22,11 @@ import com.beust.jcommander.Parameter;
import com.google.common.base.Preconditions;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillApplication;
import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
import org.apache.twill.api.TwillRunnerService;
import org.apache.twill.api.TwillSpecification;
import org.apache.twill.ext.BundledJarRunnable;
@@ -36,15 +38,16 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Properties;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicBoolean;
public class YarnAccumuloTestRunner {
private static final Logger LOG = LoggerFactory.getLogger(YarnAccumuloTestRunner.class);
+ private static final String RUNNABLE_ID = "BundledJarRunnable";
+
private static class YarnTestApp implements TwillApplication {
private TestRunnerOpts opts;
@@ -72,7 +75,7 @@ public class YarnAccumuloTestRunner {
return TwillSpecification.Builder.with()
.setName(opts.testName)
.withRunnable()
- .add("BundledJarRunnable", new BundledJarRunnable(), resourceSpec)
+ .add(RUNNABLE_ID, new BundledJarRunnable(), resourceSpec)
.withLocalFiles()
.add(jarFile.getName(), jarFile.toURI(), false)
.add(testProps.getName(), testProps.toURI())
@@ -113,6 +116,15 @@ public class YarnAccumuloTestRunner {
Preconditions.checkState(f.canRead());
}
+ private static int getNumRunning(TwillController controller) {
+ ResourceReport report = controller.getResourceReport();
+ if (report == null) {
+ return 0;
+ }
+ Collection<TwillRunResources> resources = report.getRunnableResources(RUNNABLE_ID);
+ return resources == null ? 0 : resources.size();
+ }
+
public static void main(String[] args) throws Exception {
TestRunnerOpts opts = new TestRunnerOpts();
@@ -136,34 +148,20 @@ public class YarnAccumuloTestRunner {
zookeepers);
twillRunner.start();
- final TwillController controller = twillRunner.prepare(
+ TwillController controller = twillRunner.prepare(
new YarnTestApp(opts, props))
.addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName())
.withArguments("BundledJarRunnable", arguments.toArray())
.start();
- final AtomicBoolean done = new AtomicBoolean(false);
-
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- if (!done.get()) {
- controller.kill();
- }
- } finally {
- twillRunner.stop();
- }
- }));
-
- LOG.info("Waiting for {} to finish in YARN...", opts.testName);
- LOG.info("Press ctrl-c to kill {} in YARN", opts.testName);
-
- try {
- controller.awaitTerminated();
- done.set(true);
- } catch (ExecutionException e) {
- LOG.error("Exception during execution", e);
- throw e;
+ int numRunning = getNumRunning(controller);
+ while (numRunning != opts.numContainers) {
+ LOG.info("{} of {} containers have started in YARN.", numRunning, opts.numContainers);
+ Thread.sleep(5000);
+ numRunning = getNumRunning(controller);
}
- LOG.info("{} finished", opts.testName);
+
+ LOG.info("{} of {} containers have started in YARN", numRunning, opts.numContainers);
+ LOG.info("{} application was successfully started in YARN", opts.testName);
}
}
\ No newline at end of file